From a023c28a7a40a314a70b91c497cb7ec51f03bc88 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 19 Aug 2024 17:20:51 -0700 Subject: [PATCH 01/22] HDDS-11244. Draft PR to address multiple issues in garbage collection. Will split it into multiple PRs Change-Id: I6915139a407220d782442e28be6e540342af459b --- .../hadoop/hdds/utils/TransactionInfo.java | 11 +- .../java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../hadoop/ozone/om/helpers/SnapshotInfo.java | 63 ++- .../src/main/proto/OmClientProtocol.proto | 10 + .../apache/hadoop/ozone/om/KeyManager.java | 3 +- .../hadoop/ozone/om/KeyManagerImpl.java | 16 +- .../ozone/om/OmMetadataManagerImpl.java | 72 ++- .../hadoop/ozone/om/OmSnapshotManager.java | 36 ++ .../hadoop/ozone/om/SnapshotChainManager.java | 44 +- .../om/ratis/OzoneManagerDoubleBuffer.java | 22 +- .../ratis/utils/OzoneManagerRatisUtils.java | 3 + .../ozone/om/request/file/OMFileRequest.java | 15 + .../key/OMDirectoriesPurgeRequestWithFSO.java | 8 +- .../snapshot/OMSnapshotCreateRequest.java | 3 +- .../OMSnapshotPurgeAndMoveRequest.java | 248 ++++++++++ .../snapshot/OMSnapshotPurgeRequest.java | 1 + .../OMSnapshotSetPropertyRequest.java | 117 +++-- .../OMDirectoriesPurgeResponseWithFSO.java | 40 +- .../OMSnapshotPurgeAndMoveResponse.java | 227 +++++++++ .../OMSnapshotSetPropertyResponse.java | 18 +- .../service/AbstractKeyDeletingService.java | 95 +++- .../om/service/DirectoryDeletingService.java | 21 +- .../ozone/om/service/KeyDeletingService.java | 115 ++--- .../om/service/SnapshotDeletingService.java | 458 ++---------------- .../SnapshotDirectoryCleaningService.java | 431 +++++++++------- .../ozone/om/snapshot/SnapshotUtils.java | 89 +++- 26 files changed, 1343 insertions(+), 824 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeAndMoveRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeAndMoveResponse.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java index e7c4ec4ce3d6..68d05af0c6ff 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java @@ -162,7 +162,16 @@ public String toString() { */ public static TransactionInfo readTransactionInfo( DBStoreHAManager metadataManager) throws IOException { - return metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); + return readTransactionInfo(metadataManager, false); + } + + /** + * Return transaction info persisted in OM DB skipping cache. + */ + public static TransactionInfo readTransactionInfo( + DBStoreHAManager metadataManager, boolean skipCache) throws IOException { + return skipCache ? metadataManager.getTransactionInfoTable().getSkipCache(TRANSACTION_INFO_KEY) : + metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); } public SnapshotInfo toSnapshotInfo() { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 82030669c93a..6e1d4a5912b2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -323,6 +323,7 @@ public static boolean isReadOnly( case RenameSnapshot: case SnapshotMoveDeletedKeys: case SnapshotPurge: + case SnapshotPurgeAndMove: case RecoverLease: case SetTimes: case AbortExpiredMultiPartUploads: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index 47a48c37e8e0..2bd222ca9195 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -19,6 +19,7 @@ */ import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ByteString; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.CopyObject; @@ -36,6 +37,7 @@ import java.time.ZonedDateTime; import java.time.ZoneId; +import java.util.Arrays; import java.util.Objects; import java.util.UUID; import java.util.Map; @@ -124,6 +126,7 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) { private long exclusiveSize; private long exclusiveReplicatedSize; private boolean deepCleanedDeletedDir; + private ByteString lastTransactionInfo; private SnapshotInfo(Builder b) { this.snapshotId = b.snapshotId; @@ -145,6 +148,7 @@ private SnapshotInfo(Builder b) { this.exclusiveSize = b.exclusiveSize; this.exclusiveReplicatedSize = b.exclusiveReplicatedSize; this.deepCleanedDeletedDir = b.deepCleanedDeletedDir; + this.lastTransactionInfo = b.lastTransactionInfo; } public void setName(String name) { @@ -261,13 +265,15 @@ public SnapshotInfo.Builder toBuilder() { .setGlobalPreviousSnapshotId(globalPreviousSnapshotId) .setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) + .setDbTxSequenceNumber(dbTxSequenceNumber) .setDeepClean(deepClean) .setSstFiltered(sstFiltered) .setReferencedSize(referencedSize) .setReferencedReplicatedSize(referencedReplicatedSize) .setExclusiveSize(exclusiveSize) .setExclusiveReplicatedSize(exclusiveReplicatedSize) - .setDeepCleanedDeletedDir(deepCleanedDeletedDir); + .setDeepCleanedDeletedDir(deepCleanedDeletedDir) + .setLastTransactionInfo(lastTransactionInfo); } /** @@ -293,6 +299,7 @@ public static class Builder { private long exclusiveSize; private long exclusiveReplicatedSize; private boolean deepCleanedDeletedDir; + private ByteString lastTransactionInfo; public Builder() { // default values @@ -411,6 +418,16 @@ public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) { return this; } + public Builder setLastTransactionInfo(byte[] lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo == null? null : ByteString.copyFrom(lastTransactionInfo); + return this; + } + + public Builder setLastTransactionInfo(ByteString lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo; + return this; + } + public SnapshotInfo build() { Preconditions.checkNotNull(name); return new SnapshotInfo(this); @@ -445,6 +462,10 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() { sib.setGlobalPreviousSnapshotID(toProtobuf(globalPreviousSnapshotId)); } + if (lastTransactionInfo != null) { + sib.setLastTransactionInfo(lastTransactionInfo); + } + sib.setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) .setDbTxSequenceNumber(dbTxSequenceNumber) @@ -513,6 +534,10 @@ public static SnapshotInfo getFromProtobuf( snapshotInfoProto.getDeepCleanedDeletedDir()); } + if (snapshotInfoProto.hasLastTransactionInfo()) { + osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo().toByteArray()); + } + osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) .setCheckpointDir(snapshotInfoProto.getCheckpointDir()) .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber()); @@ -605,6 +630,14 @@ public void setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) { this.deepCleanedDeletedDir = deepCleanedDeletedDir; } + public ByteString getLastTransactionInfo() { + return lastTransactionInfo; + } + + public void setLastTransactionInfo(byte[] lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo == null ? null : ByteString.copyFrom(lastTransactionInfo); + } + /** * Generate default name of snapshot, (used if user doesn't provide one). */ @@ -673,7 +706,8 @@ public boolean equals(Object o) { referencedReplicatedSize == that.referencedReplicatedSize && exclusiveSize == that.exclusiveSize && exclusiveReplicatedSize == that.exclusiveReplicatedSize && - deepCleanedDeletedDir == that.deepCleanedDeletedDir; + deepCleanedDeletedDir == that.deepCleanedDeletedDir && + Objects.equals(lastTransactionInfo, that.lastTransactionInfo); } @Override @@ -684,7 +718,7 @@ public int hashCode() { globalPreviousSnapshotId, snapshotPath, checkpointDir, deepClean, sstFiltered, referencedSize, referencedReplicatedSize, - exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir); + exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir, lastTransactionInfo); } /** @@ -692,27 +726,7 @@ public int hashCode() { */ @Override public SnapshotInfo copyObject() { - return new Builder() - .setSnapshotId(snapshotId) - .setName(name) - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setSnapshotStatus(snapshotStatus) - .setCreationTime(creationTime) - .setDeletionTime(deletionTime) - .setPathPreviousSnapshotId(pathPreviousSnapshotId) - .setGlobalPreviousSnapshotId(globalPreviousSnapshotId) - .setSnapshotPath(snapshotPath) - .setCheckpointDir(checkpointDir) - .setDbTxSequenceNumber(dbTxSequenceNumber) - .setDeepClean(deepClean) - .setSstFiltered(sstFiltered) - .setReferencedSize(referencedSize) - .setReferencedReplicatedSize(referencedReplicatedSize) - .setExclusiveSize(exclusiveSize) - .setExclusiveReplicatedSize(exclusiveReplicatedSize) - .setDeepCleanedDeletedDir(deepCleanedDeletedDir) - .build(); + return this.toBuilder().build(); } @Override @@ -737,6 +751,7 @@ public String toString() { ", exclusiveSize: '" + exclusiveSize + '\'' + ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' + ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' + + ", lastTransactionInfo: '" + lastTransactionInfo + '\'' + '}'; } } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 32bba266080a..e5f736fb5d6b 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -149,6 +149,7 @@ enum Type { RenameSnapshot = 131; ListOpenFiles = 132; + SnapshotPurgeAndMove = 133; } enum SafeMode { @@ -287,6 +288,8 @@ message OMRequest { optional RenameSnapshotRequest RenameSnapshotRequest = 129; optional ListOpenFilesRequest ListOpenFilesRequest = 130; + repeated SetSnapshotPropertyRequest SetSnapshotPropertyRequests = 131; + optional SnapshotPurgeAndMoveRequest SnapshotPurgeAndMoveRequest = 132; } message OMResponse { @@ -867,6 +870,7 @@ message SnapshotInfo { optional uint64 exclusiveReplicatedSize = 18; // note: shared sizes can be calculated from: referenced - exclusive optional bool deepCleanedDeletedDir = 19; + optional bytes lastTransactionInfo = 20; } message SnapshotDiffJobProto { @@ -1971,6 +1975,11 @@ message SnapshotPurgeRequest { repeated string updatedSnapshotDBKey = 2 [deprecated = true]; } +message SnapshotPurgeAndMoveRequest { + required hadoop.hdds.UUID snapshotID = 1; + optional hadoop.hdds.UUID expectedNextSnapshotID = 2; +} + message SetSnapshotPropertyRequest { optional SnapshotProperty snapshotProperty = 1 [deprecated = true]; optional string snapshotKey = 2; @@ -1989,6 +1998,7 @@ message SnapshotProperty { message SnapshotSize { optional uint64 exclusiveSize = 1; optional uint64 exclusiveReplicatedSize = 2; + optional bool addSize = 3; } message DeleteTenantRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 7a3312c0685a..6b632a2d69e7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.om.fs.OzoneManagerFS; import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDirectoryCleaningService; @@ -262,7 +263,7 @@ List getPendingDeletionSubFiles(long volumeId, * Returns the instance of Directory Deleting Service. * @return Background service. */ - BackgroundService getDirDeletingService(); + DirectoryDeletingService getDirDeletingService(); /** * Returns the instance of Open Key Cleanup Service. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 2cb55135294a..99501f30c1a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -193,7 +193,7 @@ public class KeyManagerImpl implements KeyManager { private final KeyProviderCryptoExtension kmsProvider; private final boolean enableFileSystemPaths; - private BackgroundService dirDeletingService; + private DirectoryDeletingService dirDeletingService; private final OMPerformanceMetrics metrics; private BackgroundService openKeyCleanupService; @@ -710,7 +710,7 @@ public KeyDeletingService getDeletingService() { } @Override - public BackgroundService getDirDeletingService() { + public DirectoryDeletingService getDirDeletingService() { return dirDeletingService; } @@ -2034,11 +2034,7 @@ private List gatherSubDirsWithIterator(OmKeyInfo parentInfo, if (!metadataManager.getDirectoryTable().isExist(entry.getKey())) { continue; } - String dirName = OMFileRequest.getAbsolutePath(parentInfo.getKeyName(), - dirInfo.getName()); - OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo( - parentInfo.getVolumeName(), parentInfo.getBucketName(), dirInfo, - dirName); + OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo(parentInfo, dirInfo); directories.add(omKeyInfo); countEntries++; } @@ -2071,11 +2067,7 @@ public List getPendingDeletionSubFiles(long volumeId, if (!metadataManager.getFileTable().isExist(entry.getKey())) { continue; } - fileInfo.setFileName(fileInfo.getKeyName()); - String fullKeyPath = OMFileRequest.getAbsolutePath( - parentInfo.getKeyName(), fileInfo.getKeyName()); - fileInfo.setKeyName(fullKeyPath); - + OMFileRequest.setKeyNameAndFileName(parentInfo, fileInfo); files.add(fileInfo); countEntries++; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 22d2b1e50b2a..13568dcbafdd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1616,11 +1616,15 @@ public PendingKeysDeletion getPendingDeletionKeys(final int keyCount, String[] keySplit = kv.getKey().split(OM_KEY_PREFIX); String bucketKey = getBucketKey(keySplit[1], keySplit[2]); OmBucketInfo bucketInfo = getBucketTable().get(bucketKey); - + SnapshotInfo snapshotInfo = getLatestSnapshotInfo(keySplit[1], keySplit[2], omSnapshotManager); + if (snapshotInfo.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE || + !OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapshotInfo)) { + continue; + } // Get the latest snapshot in snapshot path. try (ReferenceCounted - rcLatestSnapshot = getLatestActiveSnapshot( - keySplit[1], keySplit[2], omSnapshotManager)) { + rcLatestSnapshot = getLatestSnapshot( + keySplit[1], keySplit[2], omSnapshotManager, false)) { // Multiple keys with the same path can be queued in one DB entry RepeatedOmKeyInfo infoList = kv.getValue(); @@ -1731,48 +1735,38 @@ public ReferenceCounted getLatestActiveSnapshot( String volumeName, String bucketName, OmSnapshotManager snapshotManager) throws IOException { + return getLatestSnapshot(volumeName, bucketName, snapshotManager, true); + } - String snapshotPath = volumeName + OM_KEY_PREFIX + bucketName; - Optional latestPathSnapshot = Optional.ofNullable( - snapshotChainManager.getLatestPathSnapshotId(snapshotPath)); - - Optional snapshotInfo = Optional.empty(); - - while (latestPathSnapshot.isPresent()) { - Optional snapTableKey = latestPathSnapshot - .map(uuid -> snapshotChainManager.getTableKey(uuid)); - - snapshotInfo = snapTableKey.isPresent() ? - Optional.ofNullable(getSnapshotInfoTable().get(snapTableKey.get())) : - Optional.empty(); - - if (snapshotInfo.isPresent() && snapshotInfo.get().getSnapshotStatus() == - SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { - break; - } - - // Update latestPathSnapshot if current snapshot is deleted. - if (snapshotChainManager.hasPreviousPathSnapshot(snapshotPath, - latestPathSnapshot.get())) { - latestPathSnapshot = Optional.ofNullable(snapshotChainManager - .previousPathSnapshot(snapshotPath, latestPathSnapshot.get())); - } else { - latestPathSnapshot = Optional.empty(); - } + /** + * Get the latest OmSnapshot for a snapshot path. + */ + public ReferenceCounted getLatestSnapshot(String volumeName, String bucketName, + OmSnapshotManager snapshotManager, boolean isActive) + throws IOException { + Optional snapshotInfo = Optional.ofNullable(getLatestSnapshotInfo(volumeName, bucketName, + snapshotManager)); + if (isActive && snapshotInfo.map(si -> si.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) + .orElse(false)) { + snapshotInfo = Optional.ofNullable(SnapshotUtils.getPreviousActiveSnapshot(snapshotInfo.get(), + snapshotChainManager, snapshotManager)); } - - Optional> rcOmSnapshot = - snapshotInfo.isPresent() ? - Optional.ofNullable( - snapshotManager.getSnapshot(volumeName, - bucketName, - snapshotInfo.get().getName()) - ) : - Optional.empty(); + Optional> rcOmSnapshot = snapshotInfo.isPresent() ? + Optional.ofNullable(snapshotManager.getSnapshot(volumeName, bucketName, snapshotInfo.get().getName())) : + Optional.empty(); return rcOmSnapshot.orElse(null); } + public SnapshotInfo getLatestSnapshotInfo(String volumeName, String bucketName, + OmSnapshotManager snapshotManager) throws IOException { + String snapshotPath = volumeName + OM_KEY_PREFIX + bucketName; + Optional latestPathSnapshot = Optional.ofNullable( + snapshotChainManager.getLatestPathSnapshotId(snapshotPath)); + return latestPathSnapshot.isPresent() ? + SnapshotUtils.getSnapshotInfo(snapshotChainManager, latestPathSnapshot.get(), snapshotManager) : null; + } + /** * Decide whether the open key is a multipart upload related key. * @param openKeyInfo open key related to multipart upload diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 0d17851ed1f7..308f8b3723af 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.CodecRegistry; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; @@ -673,6 +674,41 @@ private ReferenceCounted getSnapshot(String snapshotTableKey, boolea return snapshotCache.get(snapshotInfo.getSnapshotId()); } + /** + * Checks if the last transaction performed on the snapshot has been flushed to disk + * @param metadataManager Metadatamanager of Active OM. + * @param snapshotTableKey table key corresponding to snapshot in snapshotInfoTable. + * @return True if the changes have been flushed to DB otherwise false + * @throws IOException + */ + public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataManager, String snapshotTableKey) + throws IOException { + // Need this info from cache since the snapshot could have been updated only on cache and not on disk. + SnapshotInfo snapshotInfo = metadataManager.getSnapshotInfoTable().get(snapshotTableKey); + return areSnapshotChangesFlushedToDB(metadataManager, snapshotInfo); + } + + /** + * Checks if the last transaction performed on the snapshot has been flushed to disk + * @param metadataManager Metadatamanager of Active OM. + * @param snapshotInfo table key corresponding to snapshot in snapshotInfoTable, this should be a value from cache + * and not from disk. + * @return True if the changes have been flushed to DB otherwise false + * @throws IOException + */ + public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataManager, SnapshotInfo snapshotInfo) + throws IOException { + TransactionInfo snapshotTransactionInfo = null; + if (snapshotInfo != null && snapshotInfo.getLastTransactionInfo() != null) { + snapshotTransactionInfo = TransactionInfo.getCodec() + .fromPersistedFormat(snapshotInfo.getLastTransactionInfo().toByteArray()); + TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager, true); + return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; + } + return false; + } + + /** * Returns true if the snapshot is in given status. * @param key DB snapshot table key diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java index 60353590e75c..491f5e278664 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java @@ -24,8 +24,10 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.NoSuchElementException; @@ -56,6 +58,7 @@ public class SnapshotChainManager { private final ConcurrentMap snapshotIdToTableKey; private UUID latestGlobalSnapshotId; private final boolean snapshotChainCorrupted; + private UUID oldestGlobalSnapshotId; public SnapshotChainManager(OMMetadataManager metadataManager) { globalSnapshotChain = Collections.synchronizedMap(new LinkedHashMap<>()); @@ -104,6 +107,8 @@ private void addSnapshotGlobal(UUID snapshotID, UUID prevGlobalID) // On add snapshot, set previous snapshot entry nextSnapshotID = // snapshotID globalSnapshotChain.get(prevGlobalID).setNextSnapshotId(snapshotID); + } else { + oldestGlobalSnapshotId = snapshotID; } globalSnapshotChain.put(snapshotID, @@ -171,7 +176,9 @@ private boolean deleteSnapshotGlobal(UUID snapshotID) throws IOException { // for node removal UUID next = globalSnapshotChain.get(snapshotID).getNextSnapshotId(); UUID prev = globalSnapshotChain.get(snapshotID).getPreviousSnapshotId(); - + if (snapshotID.equals(oldestGlobalSnapshotId)) { + oldestGlobalSnapshotId = next; + } if (prev != null && !globalSnapshotChain.containsKey(prev)) { throw new IOException(String.format( "Global snapshot chain corruption. " + @@ -379,6 +386,41 @@ public UUID getLatestGlobalSnapshotId() throws IOException { return latestGlobalSnapshotId; } + /** + * Get oldest of global snapshot in snapshot chain. + */ + public UUID getOldestGlobalSnapshotId() throws IOException { + validateSnapshotChain(); + return oldestGlobalSnapshotId; + } + + public Iterator iterator(final boolean reverse) throws IOException { + validateSnapshotChain(); + return new Iterator() { + private UUID currentSnapshotId = reverse ? getLatestGlobalSnapshotId() : getOldestGlobalSnapshotId(); + @Override + public boolean hasNext() { + try { + return reverse ? hasPreviousGlobalSnapshot(currentSnapshotId) : hasNextGlobalSnapshot(currentSnapshotId); + } catch (IOException e) { + return false; + } + } + + @Override + public UUID next() { + try { + UUID prevSnapshotId = currentSnapshotId; + currentSnapshotId = + reverse ? previousGlobalSnapshot(currentSnapshotId) : nextGlobalSnapshot(currentSnapshotId); + return prevSnapshotId; + } catch (IOException e) { + throw new UncheckedIOException("Error while getting next snapshot for " + currentSnapshotId, e); + } + } + }; + } + /** * Get latest path snapshot in snapshot chain. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java index a6fcc40dda17..59e8861ef3b5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; @@ -36,7 +37,12 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; @@ -46,6 +52,7 @@ import org.apache.hadoop.ozone.om.codec.OMDBDefinition; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Time; @@ -426,6 +433,10 @@ private String addToBatch(Queue buffer, BatchOperation batchOperation) { * in RocksDB callback flush. If multiple operations are flushed in one * specific batch, we are not sure at the flush of which specific operation * the callback is coming. + * PurgeSnapshot is also considered a barrier, since purgeSnapshot transaction on a standalone basis is an + * idempotent operation. Once the directory gets deleted the previous transactions that have been performed on the + * snapshot would start failing on replay since those transactions have not been committed but the directory has + * been deleted. This could also lead to inconsistencies in DB if operations are not performed consciously. * There could be a possibility of race condition that is exposed to rocksDB * behaviour for the batch. * Hence, we treat createSnapshot as separate batch flush. @@ -437,17 +448,22 @@ private String addToBatch(Queue buffer, BatchOperation batchOperation) { */ private List> splitReadyBufferAtCreateSnapshot() { final List> response = new ArrayList<>(); - + final Set standaloneBatchCmdTypes = ImmutableSet.of( + OzoneManagerProtocolProtos.Type.SnapshotPurge, OzoneManagerProtocolProtos.Type.SnapshotPurgeAndMove); + final List> standaloneBatchConditions = + ImmutableList.of(OMResponse::hasCreateSnapshotResponse, + (omResponse) -> standaloneBatchCmdTypes.contains(omResponse.getCmdType())); OMResponse previousOmResponse = null; for (final Entry entry : readyBuffer) { + OMResponse prevResponse = previousOmResponse; OMResponse omResponse = entry.getResponse().getOMResponse(); // New queue gets created in three conditions: // 1. It is first element in the response, // 2. Current request is createSnapshot request. // 3. Previous request was createSnapshot request. - if (response.isEmpty() || omResponse.hasCreateSnapshotResponse() + if (response.isEmpty() || standaloneBatchConditions.stream().anyMatch(condition -> condition.apply(omResponse)) || (previousOmResponse != null && - previousOmResponse.hasCreateSnapshotResponse())) { + standaloneBatchConditions.stream().anyMatch(condition -> condition.apply(prevResponse)))) { response.add(new LinkedList<>()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 8ff59e091d88..61b3bdce04fc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -77,6 +77,7 @@ import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotDeleteRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveDeletedKeysRequest; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotPurgeAndMoveRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotPurgeRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotRenameRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotSetPropertyRequest; @@ -230,6 +231,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMSnapshotMoveDeletedKeysRequest(omRequest); case SnapshotPurge: return new OMSnapshotPurgeRequest(omRequest); + case SnapshotPurgeAndMove: + return new OMSnapshotPurgeAndMoveRequest(omRequest); case SetSnapshotProperty: return new OMSnapshotSetPropertyRequest(omRequest); case DeleteOpenKeys: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index 3e7549b176e2..6f9d2a425da3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -724,6 +724,14 @@ public static OzoneFileStatus getOMKeyInfoIfExists( return null; } + public static OmKeyInfo getOmKeyInfo(OmKeyInfo parentInfo, OmDirectoryInfo directoryInfo) { + String dirName = OMFileRequest.getAbsolutePath(parentInfo.getKeyName(), + directoryInfo.getName()); + return OMFileRequest.getOmKeyInfo( + parentInfo.getVolumeName(), parentInfo.getBucketName(), directoryInfo, + dirName); + } + /** * Prepare OmKeyInfo from OmDirectoryInfo. * @@ -773,6 +781,13 @@ public static String getAbsolutePath(String prefixName, String fileName) { return prefixName.concat(fileName); } + public static void setKeyNameAndFileName(OmKeyInfo parentInfo, OmKeyInfo omKeyInfo) { + omKeyInfo.setFileName(omKeyInfo.getKeyName()); + String fullKeyPath = OMFileRequest.getAbsolutePath( + parentInfo.getKeyName(), omKeyInfo.getKeyName()); + omKeyInfo.setKeyName(fullKeyPath); + } + /** * Build DirectoryInfo from OmKeyInfo. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index 2c182a6a5f5d..234d8b5bfc3f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -25,10 +25,12 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -150,6 +152,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } } } + if (fromSnapshotInfo != null) { + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshotInfo, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshotInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshotInfo)); + } } catch (IOException ex) { // Case of IOException for fromProtobuf will not happen // as this is created and send within OM @@ -164,7 +171,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn entry.setValue(entry.getValue().copyObject()); } } - OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 3aa4151cea32..c55539070a5e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -166,7 +167,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn ((RDBStore) omMetadataManager.getStore()).getDb() .getLatestSequenceNumber(); snapshotInfo.setDbTxSequenceNumber(dbLatestSequenceNumber); - + SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); // Snapshot referenced size should be bucket's used bytes OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeAndMoveRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeAndMoveRequest.java new file mode 100644 index 000000000000..66d573a3df23 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeAndMoveRequest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.request.snapshot; + +import com.google.common.base.Objects; +import org.apache.hadoop.hdds.HddsUtils; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotPurgeAndMoveResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotPurgeResponse; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; +import org.apache.ratis.server.protocol.TermIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.UUID; + +/** + * Handles OMSnapshotPurge Request. + * This is an OM internal request. Does not need @RequireSnapshotFeatureState. + */ +public class OMSnapshotPurgeAndMoveRequest extends OMClientRequest { + + private static final Logger LOG = LoggerFactory.getLogger(OMSnapshotPurgeAndMoveRequest.class); + + /** + * This map contains up to date snapshotInfo and works as a local cache for OMSnapshotPurgeRequest. + * Since purge and other updates happen in sequence inside validateAndUpdateCache, we can get updated snapshotInfo + * from this map rather than getting form snapshotInfoTable which creates a deep copy for every get call. + */ + private final Map updatedSnapshotInfos = new HashMap<>(); + + public OMSnapshotPurgeAndMoveRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIndex termIndex) { + OMMetrics omMetrics = ozoneManager.getMetrics(); + + final long trxnLogIndex = termIndex.getIndex(); + + OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) + ozoneManager.getMetadataManager(); + SnapshotChainManager snapshotChainManager = + omMetadataManager.getSnapshotChainManager(); + + OMClientResponse omClientResponse = null; + + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); + OzoneManagerProtocolProtos.SnapshotPurgeAndMoveRequest snapshotPurgeAndMoveRequest = getOmRequest() + .getSnapshotPurgeAndMoveRequest(); + + try { + UUID snapshotId = HddsUtils.fromProtobuf(snapshotPurgeAndMoveRequest.getSnapshotID()); + // When value is null it basically means the next expected target would be AOS. + UUID nextExpectedSnapshotId = snapshotPurgeAndMoveRequest.hasExpectedNextSnapshotID() ? + HddsUtils.fromProtobuf(snapshotPurgeAndMoveRequest.getSnapshotID()) : null; + SnapshotInfo currentSnapshot = SnapshotUtils.getSnapshotInfo(snapshotChainManager, snapshotId, omSnapshotManager); + SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(currentSnapshot, snapshotChainManager, + omSnapshotManager); + SnapshotInfo prevSnapshot = SnapshotUtils.getPreviousSnapshot(currentSnapshot, snapshotChainManager, + omSnapshotManager); + // When value is null it basically means the next target is AOS. + UUID nextSnapshotId = Optional.ofNullable(nextSnapshot).map(SnapshotInfo::getSnapshotId).orElse(null); + if (!Objects.equal(nextSnapshotId, nextExpectedSnapshotId)) { + throw new OMException("Next path snapshot for " + currentSnapshot + " expected snapshot Id: " + + nextExpectedSnapshotId + " but was " + nextSnapshotId, OMException.ResultCodes.INVALID_REQUEST); + } + + // When next snapshot is not active. The keys can be moved to the next active snapshot to avoid unnecessary hop. + if (nextSnapshot != null && nextSnapshot.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { + throw new OMException("Next path snapshot for " + currentSnapshot + " " + nextSnapshotId + " is not active.", + OMException.ResultCodes.INVALID_REQUEST); + } + + // Each snapshot purge operation does three things: + // 1. Update the deep clean flag for the next snapshot if the previous snapshot is active otherwise don't + // bother to run deepClean unnecessarily + // (So that it can be deep cleaned by the KeyDeletingService in the next run), + // 2. Update the snapshot chain, + // 3. Finally, purge the snapshot. + // There is no need to take lock for snapshot purge as of now. We can simply rely on OMStateMachine + // because it executes transaction sequentially. + // As part of the flush the keys in the deleted table, deletedDirectoryTable and renameTable would be moved to + // the next snapshot. + // Step 1: Update the deep clean flag for the next active snapshot + if (prevSnapshot == null || prevSnapshot.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { + updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, termIndex); + } + // Step 2: Update the snapshot chain. + updateSnapshotChainAndCache(omMetadataManager, currentSnapshot, trxnLogIndex); + // Remove and close snapshot's RocksDB instance from SnapshotCache. + omSnapshotManager.invalidateCacheEntry(snapshotId); + // Step 3: Purge the snapshot from SnapshotInfoTable cache. + omMetadataManager.getSnapshotInfoTable() + .addCacheEntry(new CacheKey<>(currentSnapshot.getTableKey()), CacheValue.get(trxnLogIndex)); + + omClientResponse = new OMSnapshotPurgeAndMoveResponse(omResponse.build(), currentSnapshot, nextSnapshot, + updatedSnapshotInfos); + } catch (IOException ex) { + omClientResponse = new OMSnapshotPurgeResponse( + createErrorOMResponse(omResponse, ex)); + omMetrics.incNumSnapshotPurgeFails(); + LOG.error("Failed to execute snapshotPurgeAndMoveRequest:{{}}.", snapshotPurgeAndMoveRequest, ex); + } + + return omClientResponse; + } + + private void updateSnapshotInfoAndCache(SnapshotInfo snapInfo, OmMetadataManagerImpl omMetadataManager, + TermIndex termIndex) throws IOException { + if (snapInfo != null) { + // Setting next snapshot deep clean to false, Since the + // current snapshot is deleted. We can potentially + // reclaim more keys in the next snapshot. + snapInfo.setDeepClean(false); + snapInfo.setDeepCleanedDeletedDir(false); + SnapshotUtils.setTransactionInfoInSnapshot(snapInfo, termIndex); + + // Update table cache first + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), snapInfo)); + } + } + + /** + * Removes the snapshot from the chain and updates the next snapshot's + * previousPath and previousGlobal IDs in DB cache. + * It also returns the pair of updated next path and global snapshots to + * update in DB. + */ + private void updateSnapshotChainAndCache( + OmMetadataManagerImpl metadataManager, + SnapshotInfo snapInfo, + long trxnLogIndex + ) throws IOException { + if (snapInfo == null) { + return; + } + + SnapshotChainManager snapshotChainManager = metadataManager + .getSnapshotChainManager(); + + // If the snapshot is deleted in the previous run, then the in-memory + // SnapshotChainManager might throw NoSuchElementException as the snapshot + // is removed in-memory but OMDoubleBuffer has not flushed yet. + boolean hasNextPathSnapshot; + boolean hasNextGlobalSnapshot; + try { + hasNextPathSnapshot = snapshotChainManager.hasNextPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); + hasNextGlobalSnapshot = snapshotChainManager.hasNextGlobalSnapshot( + snapInfo.getSnapshotId()); + } catch (NoSuchElementException ex) { + return; + } + + String nextPathSnapshotKey = null; + + if (hasNextPathSnapshot) { + UUID nextPathSnapshotId = snapshotChainManager.nextPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); + nextPathSnapshotKey = snapshotChainManager + .getTableKey(nextPathSnapshotId); + } + + String nextGlobalSnapshotKey = null; + if (hasNextGlobalSnapshot) { + UUID nextGlobalSnapshotId = snapshotChainManager.nextGlobalSnapshot(snapInfo.getSnapshotId()); + nextGlobalSnapshotKey = snapshotChainManager.getTableKey(nextGlobalSnapshotId); + } + + SnapshotInfo nextPathSnapInfo = + nextPathSnapshotKey != null ? getUpdatedSnapshotInfo(nextPathSnapshotKey, metadataManager) : null; + + SnapshotInfo nextGlobalSnapInfo = + nextGlobalSnapshotKey != null ? getUpdatedSnapshotInfo(nextGlobalSnapshotKey, metadataManager) : null; + + // Updates next path snapshot's previous snapshot ID + if (nextPathSnapInfo != null) { + nextPathSnapInfo.setPathPreviousSnapshotId(snapInfo.getPathPreviousSnapshotId()); + metadataManager.getSnapshotInfoTable().addCacheEntry( + new CacheKey<>(nextPathSnapInfo.getTableKey()), + CacheValue.get(trxnLogIndex, nextPathSnapInfo)); + } + + if (nextGlobalSnapInfo != null) { + nextGlobalSnapInfo.setGlobalPreviousSnapshotId( + snapInfo.getGlobalPreviousSnapshotId()); + metadataManager.getSnapshotInfoTable().addCacheEntry( + new CacheKey<>(nextGlobalSnapInfo.getTableKey()), + CacheValue.get(trxnLogIndex, nextGlobalSnapInfo)); + } + + snapshotChainManager.deleteSnapshot(snapInfo); + } + + private SnapshotInfo getUpdatedSnapshotInfo(String snapshotTableKey, OMMetadataManager omMetadataManager) + throws IOException { + SnapshotInfo snapshotInfo = updatedSnapshotInfos.get(snapshotTableKey); + + if (snapshotInfo == null) { + snapshotInfo = omMetadataManager.getSnapshotInfoTable().get(snapshotTableKey); + updatedSnapshotInfos.put(snapshotTableKey, snapshotInfo); + } + return snapshotInfo; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index 2a9cfa6baf0d..cf5ba7b1cea9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -141,6 +141,7 @@ private void updateSnapshotInfoAndCache(SnapshotInfo snapInfo, // current snapshot is deleted. We can potentially // reclaim more keys in the next snapshot. snapInfo.setDeepClean(false); + snapInfo.setDeepCleanedDeletedDir(false); // Update table cache first omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapInfo.getTableKey()), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java index 53047fd8026b..9bcb050ae8fe 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.commons.compress.utils.Lists; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -36,8 +37,16 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; /** * Updates the exclusive size of the snapshot. @@ -50,6 +59,32 @@ public OMSnapshotSetPropertyRequest(OMRequest omRequest) { super(omRequest); } + private void updateSnapshotProperty( + SnapshotInfo snapInfo, OzoneManagerProtocolProtos.SetSnapshotPropertyRequest setSnapshotPropertyRequest) { + if (setSnapshotPropertyRequest.hasDeepCleanedDeletedDir()) { + snapInfo.setDeepCleanedDeletedDir(setSnapshotPropertyRequest + .getDeepCleanedDeletedDir()); + } + + if (setSnapshotPropertyRequest.hasDeepCleanedDeletedKey()) { + snapInfo.setDeepClean(setSnapshotPropertyRequest + .getDeepCleanedDeletedKey()); + } + + if (setSnapshotPropertyRequest.hasSnapshotSize()) { + SnapshotSize snapshotSize = setSnapshotPropertyRequest + .getSnapshotSize(); + boolean addToSnapshotSize = snapshotSize.hasAddSize() && snapshotSize.getAddSize(); + long exclusiveSize = (addToSnapshotSize ? snapInfo.getExclusiveSize() : 0) + + snapshotSize.getExclusiveSize(); + long exclusiveReplicatedSize = (addToSnapshotSize ? snapInfo.getExclusiveReplicatedSize() : 0) + + snapshotSize.getExclusiveReplicatedSize(); + // Set Exclusive size. + snapInfo.setExclusiveSize(exclusiveSize); + snapInfo.setExclusiveReplicatedSize(exclusiveReplicatedSize); + } + } + @Override public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIndex termIndex) { OMMetrics omMetrics = ozoneManager.getMetrics(); @@ -59,55 +94,57 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OzoneManagerProtocolProtos.OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); - OzoneManagerProtocolProtos.SetSnapshotPropertyRequest - setSnapshotPropertyRequest = getOmRequest() - .getSetSnapshotPropertyRequest(); - - String snapshotKey = setSnapshotPropertyRequest.getSnapshotKey(); - + List setSnapshotPropertyRequests = Lists.newArrayList(); + if (getOmRequest().hasSetSnapshotPropertyRequest()) { + setSnapshotPropertyRequests.add(getOmRequest().getSetSnapshotPropertyRequest()); + } + setSnapshotPropertyRequests.addAll(getOmRequest().getSetSnapshotPropertyRequestsList()); + Set snapshotKeys = new HashSet<>(); + Map snapshotInfoMap = new HashMap<>(); try { - SnapshotInfo updatedSnapInfo = metadataManager.getSnapshotInfoTable().get(snapshotKey); - if (updatedSnapInfo == null) { - LOG.error("Snapshot: '{}' doesn't not exist in snapshot table.", snapshotKey); - throw new OMException("Snapshot: '{" + snapshotKey + "}' doesn't not exist in snapshot table.", FILE_NOT_FOUND); + for (OzoneManagerProtocolProtos.SetSnapshotPropertyRequest setSnapshotPropertyRequest : + setSnapshotPropertyRequests) { + String snapshotKey = setSnapshotPropertyRequest.getSnapshotKey(); + if (snapshotKeys.contains(snapshotKey)) { + throw new OMException("Snapshot with snapshot key: " + snapshotKey + " added multiple times in the request. " + + "Request: " + setSnapshotPropertyRequests, INVALID_REQUEST); + } + snapshotKeys.add(snapshotKey); + SnapshotInfo updatedSnapInfo = snapshotInfoMap.computeIfAbsent(snapshotKey, + (k) -> { + try { + return metadataManager.getSnapshotInfoTable().get(k); + } catch (IOException e) { + throw new UncheckedIOException("Exception while getting key " + k, e); + } + }); + if (updatedSnapInfo == null) { + LOG.error("Snapshot: '{}' doesn't not exist in snapshot table.", snapshotKey); + throw new OMException("Snapshot: '{}' doesn't not exist in snapshot table." + snapshotKey + + "Request: " + setSnapshotPropertyRequests, FILE_NOT_FOUND); + } + updateSnapshotProperty(updatedSnapInfo, setSnapshotPropertyRequest); } - - if (setSnapshotPropertyRequest.hasDeepCleanedDeletedDir()) { - updatedSnapInfo.setDeepCleanedDeletedDir(setSnapshotPropertyRequest - .getDeepCleanedDeletedDir()); + if (snapshotInfoMap.isEmpty()) { + throw new OMException("Snapshots: " + snapshotKeys + " don't not exist in snapshot table.", + FILE_NOT_FOUND); } - - if (setSnapshotPropertyRequest.hasDeepCleanedDeletedKey()) { - updatedSnapInfo.setDeepClean(setSnapshotPropertyRequest - .getDeepCleanedDeletedKey()); + // Update Table Cache + for (Map.Entry snapshot : snapshotInfoMap.entrySet()) { + metadataManager.getSnapshotInfoTable().addCacheEntry( + new CacheKey<>(snapshot.getKey()), + CacheValue.get(termIndex.getIndex(), snapshot.getValue())); + omMetrics.incNumSnapshotSetProperties(); } - if (setSnapshotPropertyRequest.hasSnapshotSize()) { - SnapshotSize snapshotSize = setSnapshotPropertyRequest - .getSnapshotSize(); - long exclusiveSize = updatedSnapInfo.getExclusiveSize() + - snapshotSize.getExclusiveSize(); - long exclusiveReplicatedSize = updatedSnapInfo - .getExclusiveReplicatedSize() + snapshotSize - .getExclusiveReplicatedSize(); - // Set Exclusive size. - updatedSnapInfo.setExclusiveSize(exclusiveSize); - updatedSnapInfo.setExclusiveReplicatedSize(exclusiveReplicatedSize); - } - // Update Table Cache - metadataManager.getSnapshotInfoTable().addCacheEntry( - new CacheKey<>(snapshotKey), - CacheValue.get(termIndex.getIndex(), updatedSnapInfo)); - omClientResponse = new OMSnapshotSetPropertyResponse( - omResponse.build(), updatedSnapInfo); - omMetrics.incNumSnapshotSetProperties(); - LOG.info("Successfully executed snapshotSetPropertyRequest: {{}}.", setSnapshotPropertyRequest); - } catch (IOException ex) { + omClientResponse = new OMSnapshotSetPropertyResponse(omResponse.build(), snapshotInfoMap.values()); + LOG.info("Successfully executed snapshotSetPropertyRequest: {{}}.", setSnapshotPropertyRequests); + } catch (UncheckedIOException | IOException ex) { omClientResponse = new OMSnapshotSetPropertyResponse( createErrorOMResponse(omResponse, ex)); omMetrics.incNumSnapshotSetPropertyFails(); - LOG.error("Failed to execute snapshotSetPropertyRequest: {{}}.", setSnapshotPropertyRequest, ex); + LOG.error("Failed to execute snapshotSetPropertyRequest: {{}}.", setSnapshotPropertyRequests, ex); } return omClientResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index edb13f8cf984..b889eb0f218a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -48,12 +48,13 @@ import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; /** * Response for {@link OMDirectoriesPurgeRequestWithFSO} request. */ @CleanupTableInfo(cleanupTables = {DELETED_TABLE, DELETED_DIR_TABLE, - DIRECTORY_TABLE, FILE_TABLE}) + DIRECTORY_TABLE, FILE_TABLE, SNAPSHOT_INFO_TABLE}) public class OMDirectoriesPurgeResponseWithFSO extends OmKeyResponse { private static final Logger LOG = LoggerFactory.getLogger(OMDirectoriesPurgeResponseWithFSO.class); @@ -96,12 +97,13 @@ public void addToDBBatch(OMMetadataManager metadataManager, // Init Batch Operation for snapshot db. try (BatchOperation writeBatch = fromSnapshotStore.initBatchOperation()) { - processPaths(fromSnapshot.getMetadataManager(), writeBatch); + processPaths(metadataManager, fromSnapshot.getMetadataManager(), batchOp, writeBatch); fromSnapshotStore.commitBatchOperation(writeBatch); } } + metadataManager.getSnapshotInfoTable().putWithBatch(batchOp, fromSnapshotInfo.getTableKey(), fromSnapshotInfo); } else { - processPaths(metadataManager, batchOp); + processPaths(metadataManager, metadataManager, batchOp, batchOp); } // update bucket quota in active db @@ -112,8 +114,10 @@ public void addToDBBatch(OMMetadataManager metadataManager, } } - public void processPaths(OMMetadataManager omMetadataManager, - BatchOperation batchOperation) throws IOException { + public void processPaths(OMMetadataManager keySpaceOmMetadataManager, + OMMetadataManager deletedSpaceOmMetadataManager, + BatchOperation keySpaceBatchOperation, + BatchOperation deletedSpaceBatchOperation) throws IOException { for (OzoneManagerProtocolProtos.PurgePathRequest path : paths) { final long volumeId = path.getVolumeId(); final long bucketId = path.getBucketId(); @@ -126,14 +130,14 @@ public void processPaths(OMMetadataManager omMetadataManager, // Add all sub-directories to deleted directory table. for (OzoneManagerProtocolProtos.KeyInfo key : markDeletedSubDirsList) { OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key); - String ozoneDbKey = omMetadataManager.getOzonePathKey(volumeId, + String ozoneDbKey = keySpaceOmMetadataManager.getOzonePathKey(volumeId, bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName()); - String ozoneDeleteKey = omMetadataManager.getOzoneDeletePathKey( + String ozoneDeleteKey = deletedSpaceOmMetadataManager.getOzoneDeletePathKey( key.getObjectID(), ozoneDbKey); - omMetadataManager.getDeletedDirTable().putWithBatch(batchOperation, + deletedSpaceOmMetadataManager.getDeletedDirTable().putWithBatch(deletedSpaceBatchOperation, ozoneDeleteKey, keyInfo); - omMetadataManager.getDirectoryTable().deleteWithBatch(batchOperation, + keySpaceOmMetadataManager.getDirectoryTable().deleteWithBatch(keySpaceBatchOperation, ozoneDbKey); if (LOG.isDebugEnabled()) { @@ -144,10 +148,10 @@ public void processPaths(OMMetadataManager omMetadataManager, for (OzoneManagerProtocolProtos.KeyInfo key : deletedSubFilesList) { OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key); - String ozoneDbKey = omMetadataManager.getOzonePathKey(volumeId, + String ozoneDbKey = keySpaceOmMetadataManager.getOzonePathKey(volumeId, bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName()); - omMetadataManager.getKeyTable(getBucketLayout()) - .deleteWithBatch(batchOperation, ozoneDbKey); + keySpaceOmMetadataManager.getKeyTable(getBucketLayout()) + .deleteWithBatch(keySpaceBatchOperation, ozoneDbKey); if (LOG.isDebugEnabled()) { LOG.info("Move keyName:{} to DeletedTable DBKey: {}", @@ -157,26 +161,26 @@ public void processPaths(OMMetadataManager omMetadataManager, RepeatedOmKeyInfo repeatedOmKeyInfo = OmUtils.prepareKeyForDelete( keyInfo, keyInfo.getUpdateID(), isRatisEnabled); - String deletedKey = omMetadataManager + String deletedKey = keySpaceOmMetadataManager .getOzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName()); - deletedKey = omMetadataManager.getOzoneDeletePathKey( + deletedKey = deletedSpaceOmMetadataManager.getOzoneDeletePathKey( keyInfo.getObjectID(), deletedKey); - omMetadataManager.getDeletedTable().putWithBatch(batchOperation, + deletedSpaceOmMetadataManager.getDeletedTable().putWithBatch(deletedSpaceBatchOperation, deletedKey, repeatedOmKeyInfo); } if (!openKeyInfoMap.isEmpty()) { for (Map.Entry entry : openKeyInfoMap.entrySet()) { - omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( - batchOperation, entry.getKey(), entry.getValue()); + keySpaceOmMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch( + keySpaceBatchOperation, entry.getKey(), entry.getValue()); } } // Delete the visited directory from deleted directory table if (path.hasDeletedDir()) { - omMetadataManager.getDeletedDirTable().deleteWithBatch(batchOperation, + deletedSpaceOmMetadataManager.getDeletedDirTable().deleteWithBatch(deletedSpaceBatchOperation, path.getDeletedDir()); if (LOG.isDebugEnabled()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeAndMoveResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeAndMoveResponse.java new file mode 100644 index 000000000000..727bd4ef92c4 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeAndMoveResponse.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.response.snapshot; + +import jakarta.annotation.Nonnull; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.Stat; +import org.apache.hadoop.hdds.StringUtils; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.lock.OMLockDetails; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; +import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.SNAPSHOT_LOCK; + +/** + * Response for OMSnapshotMoveDeletedKeysRequest. + */ +@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) +public class OMSnapshotPurgeAndMoveResponse extends OMClientResponse { + private static final Logger LOG = + LoggerFactory.getLogger(OMSnapshotPurgeAndMoveResponse.class); + + private SnapshotInfo purgedSnapshot; + private SnapshotInfo nextSnapshot; + Map updatedSnapInfos; + + Path purgedSnapshotPath; + private File statePath; + + enum State { + STARTING, RENAME_TABLE_WRITE, RENAME_TABLE_LOAD, DELETED_TABLE_WRITE, DELETED_TABLE_LOAD, DELETED_DIR_TABLE_WRITE, + DELETED_DIR_TABLE_LOAD + } + + + public OMSnapshotPurgeAndMoveResponse(@Nonnull OMResponse omResponse, + @Nonnull SnapshotInfo purgedSnapshot, + @Nonnull SnapshotInfo nextSnapshot, + Map updatedSnapInfos) { + super(omResponse); + this.purgedSnapshot = purgedSnapshot; + this.nextSnapshot = nextSnapshot; + this.updatedSnapInfos = updatedSnapInfos; + } + + private void writeState(State currentState) throws IOException { + try (FileWriter writer = new FileWriter(statePath)) { + writer.write(currentState.toString()); + } + } + + private State readState() throws IOException { + return State.valueOf(StringUtils.bytes2String(Files.readAllBytes(statePath.toPath()))); + } + + private State writeAndLoadTable(OmSnapshotManager sourceSnapshotManager, OMMetadataManager targetMetadataManager, + State currentState, State writeState, State loadState, + String tableName, String prefix) throws IOException { + + File sstFile = new File(purgedSnapshotPath + "_" + tableName + ".sst"); + if (currentState != writeState) { + if (sstFile.exists()) { + sstFile.delete(); + } + try (ReferenceCounted snapshotReference = sourceSnapshotManager.getSnapshot( + purgedSnapshot.getVolumeName(), purgedSnapshot.getBucketName(), purgedSnapshot.getName())) { + OmSnapshot omSnapshot = snapshotReference.get(); + (omSnapshot.getMetadataManager().getTable(tableName)).dumpToFileWithPrefix(sstFile, prefix); + } + currentState = writeState; + writeState(currentState); + } + if (!sstFile.exists()) { + throw new IOException("Sst file created: "+ sstFile.getAbsolutePath() +" doesn't exist"); + } + targetMetadataManager.getTable(tableName).loadFromFile(sstFile); + currentState = loadState; + writeState(currentState); + sstFile.delete(); + return currentState; + } + + /** + * For when the request is not successful. + * For a successful request, the other constructor should be used. + */ + public OMSnapshotPurgeAndMoveResponse(@Nonnull OMResponse omResponse) { + super(omResponse); + checkStatusNotOK(); + } + + @Override + protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { + //Loading Deleted Table, RenameTable & DeletedDirectory table by creating a temporary sst file and ingesting it + // in the rocksDB directly to avoid memory pressure on rocksdb which would have been the case if a batch would + // have been happened. Here the entries would be directly written to a file and rocksdb would just add the file + // in its meta. + purgedSnapshotPath = OmSnapshotManager.getSnapshotPath(omMetadataManager, purgedSnapshot); + statePath = new File( purgedSnapshotPath + "_state"); + OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) omMetadataManager) + .getOzoneManager().getOmSnapshotManager(); + String dbBucketKey = omMetadataManager.getBucketKey(purgedSnapshot.getVolumeName(), + purgedSnapshot.getBucketName()); + long volumeId = omMetadataManager.getVolumeId(purgedSnapshot.getVolumeName()); + long bucketId = omMetadataManager.getBucketTable().get(dbBucketKey).getObjectID(); + String dbBucketKeyForDir = omMetadataManager + .getBucketKey(Long.toString(volumeId), Long.toString(bucketId)) + OM_KEY_PREFIX; + String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; + + State currentState = null; + if (!statePath.exists() && purgedSnapshotPath.toFile().exists()) { + currentState = State.STARTING; + } else if (statePath.exists()) { + currentState = readState(); + } + ReferenceCounted rcOmNextSnapshot = null; + OMMetadataManager nextTableMetadataManager = omMetadataManager; + if (nextSnapshot != null) { + rcOmNextSnapshot = omSnapshotManager.getSnapshot(nextSnapshot.getVolumeName(), nextSnapshot.getBucketName(), + nextSnapshot.getName()); + nextTableMetadataManager = rcOmNextSnapshot.get().getMetadataManager(); + } + try { + switch (currentState) { + case STARTING: + case RENAME_TABLE_WRITE: + currentState = writeAndLoadTable(omSnapshotManager, nextTableMetadataManager, currentState, + State.RENAME_TABLE_WRITE, State.RENAME_TABLE_LOAD, OmMetadataManagerImpl.SNAPSHOT_RENAMED_TABLE, + snapshotBucketKey); + case RENAME_TABLE_LOAD: + case DELETED_TABLE_WRITE: + currentState = writeAndLoadTable(omSnapshotManager, nextTableMetadataManager, currentState, + State.DELETED_TABLE_WRITE, State.DELETED_TABLE_LOAD, OmMetadataManagerImpl.DELETED_TABLE, + snapshotBucketKey); + case DELETED_TABLE_LOAD: + case DELETED_DIR_TABLE_WRITE: + currentState = writeAndLoadTable(omSnapshotManager, nextTableMetadataManager, currentState, + State.DELETED_DIR_TABLE_WRITE, State.DELETED_DIR_TABLE_LOAD, OmMetadataManagerImpl.DELETED_DIR_TABLE, + dbBucketKeyForDir); + } + deleteCheckpointDirectory(omMetadataManager, purgedSnapshot); + statePath.delete(); + } finally { + IOUtils.closeQuietly(rcOmNextSnapshot); + } + for (Map.Entry updatedSnapshotInfo : updatedSnapInfos.entrySet()) { + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, updatedSnapshotInfo.getKey(), + updatedSnapshotInfo.getValue()); + } + omMetadataManager.getSnapshotInfoTable().deleteWithBatch(batchOperation, purgedSnapshot.getTableKey()); + } + + /** + * Deletes the checkpoint directory for a snapshot. + */ + private void deleteCheckpointDirectory(OMMetadataManager omMetadataManager, + SnapshotInfo snapshotInfo) { + // Acquiring write lock to avoid race condition with sst filtering service which creates a sst filtered file + // inside the snapshot directory. Any operation apart which doesn't create/delete files under this snapshot + // directory can run in parallel along with this operation. + OMLockDetails omLockDetails = omMetadataManager.getLock() + .acquireWriteLock(SNAPSHOT_LOCK, snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), + snapshotInfo.getName()); + boolean acquiredSnapshotLock = omLockDetails.isLockAcquired(); + if (acquiredSnapshotLock) { + Path snapshotDirPath = OmSnapshotManager.getSnapshotPath(omMetadataManager, snapshotInfo); + try { + FileUtils.deleteDirectory(snapshotDirPath.toFile()); + } catch (IOException ex) { + LOG.error("Failed to delete snapshot directory {} for snapshot {}", + snapshotDirPath, snapshotInfo.getTableKey(), ex); + } finally { + omMetadataManager.getLock().releaseWriteLock(SNAPSHOT_LOCK, snapshotInfo.getVolumeName(), + snapshotInfo.getBucketName(), snapshotInfo.getName()); + } + } + } + + +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java index c018de78cb88..e0a179889bec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java @@ -27,6 +27,9 @@ import jakarta.annotation.Nonnull; import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Set; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; @@ -35,26 +38,29 @@ */ @CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) public class OMSnapshotSetPropertyResponse extends OMClientResponse { - private final SnapshotInfo updatedSnapInfo; + private final Collection updatedSnapInfos; public OMSnapshotSetPropertyResponse( @Nonnull OMResponse omResponse, - @Nonnull SnapshotInfo updatedSnapInfo) { + @Nonnull Collection updatedSnapInfos) { super(omResponse); - this.updatedSnapInfo = updatedSnapInfo; + this.updatedSnapInfos = updatedSnapInfos; } public OMSnapshotSetPropertyResponse(@Nonnull OMResponse omResponse) { super(omResponse); checkStatusNotOK(); - this.updatedSnapInfo = null; + this.updatedSnapInfos = null; } @Override protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, - updatedSnapInfo.getTableKey(), updatedSnapInfo); + for (SnapshotInfo updatedSnapInfo : updatedSnapInfos) { + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, + updatedSnapInfo.getTableKey(), updatedSnapInfo); + } + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index 429e286287c1..fef7b4552963 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -36,9 +36,11 @@ import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeletedKeys; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -57,6 +59,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -292,8 +295,8 @@ private void addToMap(Map, List> map, String object map.get(volumeBucketPair).add(objectKey); } - protected void submitPurgePaths(List requests, - String snapTableKey) { + protected OzoneManagerProtocolProtos.OMResponse submitPurgePaths( + List requests, String snapTableKey) { OzoneManagerProtocolProtos.PurgeDirectoriesRequest.Builder purgeDirRequest = OzoneManagerProtocolProtos.PurgeDirectoriesRequest.newBuilder(); @@ -314,15 +317,30 @@ protected void submitPurgePaths(List requests, if (isRatisEnabled()) { RaftClientRequest raftClientRequest = createRaftClientRequestForPurge(omRequest); - ozoneManager.getOmRatisServer().submitRequest(omRequest, + return ozoneManager.getOmRatisServer().submitRequest(omRequest, raftClientRequest); } else { - getOzoneManager().getOmServerProtocol() + return getOzoneManager().getOmServerProtocol() .submitRequest(null, omRequest); } } catch (ServiceException e) { LOG.error("PurgePaths request failed. Will retry at next run."); } + return null; + } + + protected OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( + final long volumeId, final long bucketId, final List purgeDeletedFiles) { + // Put all keys to be purged in a list + PurgePathRequest.Builder purgePathsRequest = PurgePathRequest.newBuilder(); + purgePathsRequest.setVolumeId(volumeId); + purgePathsRequest.setBucketId(bucketId); + + for (OzoneManagerProtocolProtos.KeyInfo purgeFile : purgeDeletedFiles) { + purgePathsRequest.addDeletedSubFiles(purgeFile); + } + + return purgePathsRequest.build(); } private OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( @@ -408,12 +426,10 @@ protected PurgePathRequest prepareDeleteDirRequest( } @SuppressWarnings("checkstyle:ParameterNumber") - public long optimizeDirDeletesAndSubmitRequest(long remainNum, - long dirNum, long subDirNum, long subFileNum, - List> allSubDirList, - List purgePathRequestList, - String snapTableKey, long startTime, - int remainingBufLimit, KeyManager keyManager) { + public Pair> optimizeDirDeletesAndSubmitRequest( + long remainNum, long dirNum, long subDirNum, long subFileNum, List> allSubDirList, + List purgePathRequestList, String snapTableKey, long startTime, int remainingBufLimit, + KeyManager keyManager) { // Optimization to handle delete sub-dir and keys to remove quickly // This case will be useful to handle when depth of directory is high @@ -451,9 +467,9 @@ public long optimizeDirDeletesAndSubmitRequest(long remainNum, break; } } - + OzoneManagerProtocolProtos.OMResponse response = null; if (!purgePathRequestList.isEmpty()) { - submitPurgePaths(purgePathRequestList, snapTableKey); + response = submitPurgePaths(purgePathRequestList, snapTableKey); } if (dirNum != 0 || subDirNum != 0 || subFileNum != 0) { @@ -468,7 +484,7 @@ public long optimizeDirDeletesAndSubmitRequest(long remainNum, dirNum, subdirDelNum, subFileNum, (subDirNum - subdirDelNum), Time.monotonicNow() - startTime, getRunCount()); } - return remainNum; + return Pair.of(remainNum, Optional.ofNullable(response)); } /** @@ -596,6 +612,20 @@ protected SnapshotInfo getPreviousActiveSnapshot(SnapshotInfo snapInfo, return null; } + protected SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo, + SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager) + throws IOException { + SnapshotInfo currSnapInfo = snapInfo; + if (chainManager.hasPreviousPathSnapshot(currSnapInfo.getSnapshotPath(), currSnapInfo.getSnapshotId())) { + + UUID prevPathSnapshot = chainManager.previousPathSnapshot( + currSnapInfo.getSnapshotPath(), currSnapInfo.getSnapshotId()); + String tableKey = chainManager.getTableKey(prevPathSnapshot); + return omSnapshotManager.getSnapshotInfo(tableKey); + } + return null; + } + protected boolean isKeyReclaimable( Table previousKeyTable, Table renamedTable, @@ -662,6 +692,45 @@ protected boolean isKeyReclaimable( return !isBlockLocationInfoSame(prevKeyInfo, deletedKeyInfo); } + protected boolean isDirReclaimable( + Table.KeyValue deletedDir, + Table previousDirTable, + Table renamedTable) throws IOException { + + if (previousDirTable == null) { + return true; + } + + String deletedDirDbKey = deletedDir.getKey(); + OmKeyInfo deletedDirInfo = deletedDir.getValue(); + String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey( + deletedDirInfo.getVolumeName(), deletedDirInfo.getBucketName(), + deletedDirInfo.getObjectID()); + + /* + snapshotRenamedTable: /volumeName/bucketName/objectID -> + /volumeId/bucketId/parentId/dirName + */ + String dbKeyBeforeRename = renamedTable.getIfExist(dbRenameKey); + String prevDbKey = null; + + if (dbKeyBeforeRename != null) { + prevDbKey = dbKeyBeforeRename; + } else { + // In OMKeyDeleteResponseWithFSO OzonePathKey is converted to + // OzoneDeletePathKey. Changing it back to check the previous DirTable. + prevDbKey = ozoneManager.getMetadataManager() + .getOzoneDeletePathDirKey(deletedDirDbKey); + } + + OmDirectoryInfo prevDirectoryInfo = previousDirTable.get(prevDbKey); + if (prevDirectoryInfo == null) { + return true; + } + + return prevDirectoryInfo.getObjectID() != deletedDirInfo.getObjectID(); + } + public boolean isRatisEnabled() { if (ozoneManager == null) { return false; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index c8703c3c4c62..fd4c2d3686c9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; import org.apache.hadoop.util.Time; @@ -82,6 +83,7 @@ public class DirectoryDeletingService extends AbstractKeyDeletingService { private final long pathLimitPerTask; private final int ratisByteLimit; private final AtomicBoolean suspended; + private boolean runningOnAOS; public DirectoryDeletingService(long interval, TimeUnit unit, long serviceTimeout, OzoneManager ozoneManager, @@ -98,6 +100,7 @@ public DirectoryDeletingService(long interval, TimeUnit unit, // always go to 90% of max limit for request as other header will be added this.ratisByteLimit = (int) (limit * 0.9); this.suspended = new AtomicBoolean(false); + this.runningOnAOS = false; } private boolean shouldRun() { @@ -124,6 +127,11 @@ public void resume() { suspended.set(false); } + public boolean isRunningOnAOS() { + return runningOnAOS; + } + + @Override public BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); @@ -141,6 +149,7 @@ public int getPriority() { @Override public BackgroundTaskResult call() { if (shouldRun()) { + runningOnAOS = true; if (LOG.isDebugEnabled()) { LOG.debug("Running DirectoryDeletingService"); } @@ -210,6 +219,7 @@ public BackgroundTaskResult call() { LOG.error("Error while running delete directories and files " + "background task. Will retry at next run.", e); } + runningOnAOS = false; } // place holder by returning empty results of this call back. @@ -224,12 +234,17 @@ private boolean previousSnapshotHasDir( getOzoneManager().getOmSnapshotManager(); OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) getOzoneManager().getMetadataManager(); - + SnapshotInfo snapshotInfo = metadataManager.getLatestSnapshotInfo(deletedDirInfo.getVolumeName(), + deletedDirInfo.getBucketName(), omSnapshotManager); + if (snapshotInfo.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE || + !OmSnapshotManager.areSnapshotChangesFlushedToDB(metadataManager, snapshotInfo)) { + return true; + } try (ReferenceCounted rcLatestSnapshot = - metadataManager.getLatestActiveSnapshot( + metadataManager.getLatestSnapshot( deletedDirInfo.getVolumeName(), deletedDirInfo.getBucketName(), - omSnapshotManager)) { + omSnapshotManager, false)) { if (rcLatestSnapshot != null) { String dbRenameKey = metadataManager diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index c42854828720..13233dc0a543 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -19,11 +19,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -31,6 +29,7 @@ import com.google.common.base.Preconditions; import com.google.protobuf.ServiceException; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; @@ -38,6 +37,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; @@ -56,7 +56,6 @@ import com.google.common.annotations.VisibleForTesting; -import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT; @@ -88,12 +87,13 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private static final int KEY_DELETING_CORE_POOL_SIZE = 1; private final KeyManager manager; + + private boolean runningOnAOS; private int keyLimitPerTask; private final AtomicLong deletedKeyCount; private final AtomicBoolean suspended; private final Map exclusiveSizeMap; private final Map exclusiveReplicatedSizeMap; - private final Set completedExclusiveSizeSet; private final Map snapshotSeekMap; public KeyDeletingService(OzoneManager ozoneManager, @@ -112,8 +112,8 @@ public KeyDeletingService(OzoneManager ozoneManager, this.suspended = new AtomicBoolean(false); this.exclusiveSizeMap = new HashMap<>(); this.exclusiveReplicatedSizeMap = new HashMap<>(); - this.completedExclusiveSizeSet = new HashSet<>(); this.snapshotSeekMap = new HashMap<>(); + this.runningOnAOS = false; } /** @@ -165,6 +165,10 @@ public void setKeyLimitPerTask(int keyLimitPerTask) { this.keyLimitPerTask = keyLimitPerTask; } + public boolean isRunningOnAOS() { + return runningOnAOS; + } + /** * A key deleting task scans OM DB and looking for a certain number of * pending-deletion keys, sends these keys along with their associated blocks @@ -184,6 +188,7 @@ public BackgroundTaskResult call() { // Check if this is the Leader OM. If not leader, no need to execute this // task. if (shouldRun()) { + runningOnAOS = true; final long run = getRunCount().incrementAndGet(); LOG.debug("Running KeyDeletingService {}", run); @@ -194,7 +199,6 @@ public BackgroundTaskResult call() { // doesn't have enough entries left. // OM would have to keep track of which snapshot the key is coming // from if the above would be done inside getPendingDeletionKeys(). - PendingKeysDeletion pendingKeysDeletion = manager .getPendingDeletionKeys(getKeyLimitPerTask()); List keyBlocksList = pendingKeysDeletion @@ -205,6 +209,7 @@ public BackgroundTaskResult call() { pendingKeysDeletion.getKeysToModify(), null); deletedKeyCount.addAndGet(delCount); } + runningOnAOS = false; } catch (IOException e) { LOG.error("Error while running delete keys background task. Will " + "retry at next run.", e); @@ -218,7 +223,6 @@ public BackgroundTaskResult call() { LOG.error("Error while running deep clean on snapshots. Will " + "retry at next run.", e); } - } // By design, no one cares about the results of this call back. return EmptyTaskResult.newResult(); @@ -235,7 +239,7 @@ private void processSnapshotDeepClean(int delCount) .getSnapshotChainManager(); Table snapshotInfoTable = getOzoneManager().getMetadataManager().getSnapshotInfoTable(); - List deepCleanedSnapshots = new ArrayList<>(); + Map> deepCleanedSnapshots = new HashMap<>(); try (TableIterator> iterator = snapshotInfoTable.iterator()) { @@ -244,10 +248,11 @@ private void processSnapshotDeepClean(int delCount) HashMap keysToModify = new HashMap<>(); SnapshotInfo currSnapInfo = iterator.next().getValue(); - // Deep clean only on active snapshot. Deleted Snapshots will be - // cleaned up by SnapshotDeletingService. - if (currSnapInfo.getSnapshotStatus() != SNAPSHOT_ACTIVE || - currSnapInfo.getDeepClean()) { + // Deep clean only on snapshots whose deleted directory table has been deep cleaned. This is to avoid + // unnecessary additional iteration on the deleted table after SnapshotDirectoryDeletingService adds + // entries in deleted table. We should wait for all changes on the snapshot to be flushed to the db. + if (!currSnapInfo.getDeepCleanedDeletedDir() || currSnapInfo.getDeepClean() + || !OmSnapshotManager.areSnapshotChangesFlushedToDB(metadataManager, currSnapInfo.getTableKey())) { continue; } @@ -279,12 +284,12 @@ private void processSnapshotDeepClean(int delCount) } String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; - SnapshotInfo previousSnapshot = getPreviousActiveSnapshot( + SnapshotInfo previousSnapshot = getPreviousSnapshot( currSnapInfo, snapChainManager, omSnapshotManager); SnapshotInfo previousToPrevSnapshot = null; if (previousSnapshot != null) { - previousToPrevSnapshot = getPreviousActiveSnapshot( + previousToPrevSnapshot = getPreviousSnapshot( previousSnapshot, snapChainManager, omSnapshotManager); } @@ -392,16 +397,14 @@ private void processSnapshotDeepClean(int delCount) if (delCount < keyLimitPerTask) { // Deep clean is completed, we can update the SnapInfo. - deepCleanedSnapshots.add(currSnapInfo.getTableKey()); + deepCleanedSnapshots.put(currSnapInfo.getTableKey(), Optional.empty()); // exclusiveSizeList contains check is used to prevent // case where there is no entry in deletedTable, this // will throw NPE when we submit request. if (previousSnapshot != null && exclusiveSizeMap .containsKey(previousSnapshot.getTableKey())) { - completedExclusiveSizeSet.add( - previousSnapshot.getTableKey()); + deepCleanedSnapshots.put(currSnapInfo.getTableKey(), Optional.of(previousSnapshot.getTableKey())); } - snapshotSeekMap.remove(currSnapInfo.getTableKey()); } else { // There are keys that still needs processing @@ -423,61 +426,39 @@ private void processSnapshotDeepClean(int delCount) } } - updateDeepCleanedSnapshots(deepCleanedSnapshots); - updateSnapshotExclusiveSize(); } - private void updateSnapshotExclusiveSize() { - - if (completedExclusiveSizeSet.isEmpty()) { - return; - } - - Iterator completedSnapshotIterator = - completedExclusiveSizeSet.iterator(); - while (completedSnapshotIterator.hasNext()) { - ClientId clientId = ClientId.randomId(); - String dbKey = completedSnapshotIterator.next(); - SnapshotSize snapshotSize = SnapshotSize.newBuilder() - .setExclusiveSize(exclusiveSizeMap.getOrDefault(dbKey, 0L)) - .setExclusiveReplicatedSize( - exclusiveReplicatedSizeMap.getOrDefault(dbKey, 0L)) - .build(); - SetSnapshotPropertyRequest setSnapshotPropertyRequest = - SetSnapshotPropertyRequest.newBuilder() - .setSnapshotKey(dbKey) - .setSnapshotSize(snapshotSize) - .build(); - - OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(Type.SetSnapshotProperty) - .setSetSnapshotPropertyRequest(setSnapshotPropertyRequest) - .setClientId(clientId.toString()) - .build(); - submitRequest(omRequest, clientId); - exclusiveSizeMap.remove(dbKey); - exclusiveReplicatedSizeMap.remove(dbKey); - completedSnapshotIterator.remove(); - } + private SnapshotSize getSnapshotSize(String snapshotDBKey) { + return SnapshotSize.newBuilder() + .setExclusiveSize(exclusiveSizeMap.getOrDefault(snapshotDBKey, 0L)) + .setExclusiveReplicatedSize( + exclusiveReplicatedSizeMap.getOrDefault(snapshotDBKey, 0L)) + .setAddSize(true) + .build(); } - private void updateDeepCleanedSnapshots(List deepCleanedSnapshots) { - for (String deepCleanedSnapshot: deepCleanedSnapshots) { + private void updateDeepCleanedSnapshots(Map> deepCleanedSnapshots) { + for (Map.Entry> deepCleanedSnapshot : deepCleanedSnapshots.entrySet()) { ClientId clientId = ClientId.randomId(); - SetSnapshotPropertyRequest setSnapshotPropertyRequest = - SetSnapshotPropertyRequest.newBuilder() - .setSnapshotKey(deepCleanedSnapshot) - .setDeepCleanedDeletedKey(true) - .build(); - - OMRequest omRequest = OMRequest.newBuilder() + String snapshotDBKey = deepCleanedSnapshot.getKey(); + Optional previousSnapshotDBKey = deepCleanedSnapshot.getValue(); + SetSnapshotPropertyRequest.Builder setSnapshotPropertyRequest = SetSnapshotPropertyRequest.newBuilder() + .setSnapshotKey(snapshotDBKey) + .setDeepCleanedDeletedKey(true); + OMRequest.Builder omRequest = OMRequest.newBuilder() .setCmdType(Type.SetSnapshotProperty) - .setSetSnapshotPropertyRequest(setSnapshotPropertyRequest) - .setClientId(clientId.toString()) - .build(); - - submitRequest(omRequest, clientId); + .addSetSnapshotPropertyRequests(setSnapshotPropertyRequest) + .setClientId(clientId.toString()); + if (previousSnapshotDBKey.isPresent()) { + SetSnapshotPropertyRequest setPreviousSnapshotPropertyRequest = + SetSnapshotPropertyRequest.newBuilder() + .setSnapshotKey(previousSnapshotDBKey.get()) + .setSnapshotSize(getSnapshotSize(previousSnapshotDBKey.get())) + .build(); + omRequest.addSetSnapshotPropertyRequests(setPreviousSnapshotPropertyRequest); + } + submitRequest(omRequest.build(), clientId); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 99e3903447d4..a673d846e15d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ServiceException; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -30,9 +31,6 @@ import org.apache.hadoop.hdds.utils.BackgroundTaskResult; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -41,7 +39,6 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -49,12 +46,10 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; -import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; @@ -65,12 +60,13 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK; @@ -97,8 +93,6 @@ public class SnapshotDeletingService extends AbstractKeyDeletingService { private final OzoneConfiguration conf; private final AtomicLong successRunCount; private final long snapshotDeletionPerTask; - private final int keyLimitPerSnapshot; - private final int ratisByteLimit; public SnapshotDeletingService(long interval, long serviceTimeout, OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient) @@ -117,15 +111,6 @@ public SnapshotDeletingService(long interval, long serviceTimeout, this.snapshotDeletionPerTask = conf .getLong(SNAPSHOT_DELETING_LIMIT_PER_TASK, SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT); - int limit = (int) conf.getStorageSize( - OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, - OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT, - StorageUnit.BYTES); - // always go to 90% of max limit for request as other header will be added - this.ratisByteLimit = (int) (limit * 0.9); - this.keyLimitPerSnapshot = conf.getInt( - OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK, - OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); } private class SnapshotDeletingTask implements BackgroundTask { @@ -138,421 +123,55 @@ public BackgroundTaskResult call() throws InterruptedException { } getRunCount().incrementAndGet(); + SnapshotChainManager snapshotChainManager = + ((OmMetadataManagerImpl)ozoneManager.getMetadataManager()).getSnapshotChainManager(); - ReferenceCounted rcOmSnapshot = null; - ReferenceCounted rcOmPreviousSnapshot = null; - - Table snapshotInfoTable = - ozoneManager.getMetadataManager().getSnapshotInfoTable(); - List purgeSnapshotKeys = new ArrayList<>(); - try (TableIterator> iterator = snapshotInfoTable.iterator()) { - + try { + Iterator iterator = snapshotChainManager.iterator(true); long snapshotLimit = snapshotDeletionPerTask; - while (iterator.hasNext() && snapshotLimit > 0) { - SnapshotInfo snapInfo = iterator.next().getValue(); - - // Only Iterate in deleted snapshot + SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(chainManager, iterator.next(), omSnapshotManager); + // Only Iterate in deleted snapshot & only if all the changes have been flushed into disk. if (shouldIgnoreSnapshot(snapInfo)) { continue; } - // Note: Can refactor this to use try-with-resources. - // Handling RC decrements manually for now to minimize conflicts. - rcOmSnapshot = omSnapshotManager.getSnapshot( - snapInfo.getVolumeName(), - snapInfo.getBucketName(), - snapInfo.getName()); - OmSnapshot omSnapshot = rcOmSnapshot.get(); - - Table snapshotDeletedTable = - omSnapshot.getMetadataManager().getDeletedTable(); - Table snapshotDeletedDirTable = - omSnapshot.getMetadataManager().getDeletedDirTable(); - - Table renamedTable = - omSnapshot.getMetadataManager().getSnapshotRenamedTable(); - - long volumeId = ozoneManager.getMetadataManager() - .getVolumeId(snapInfo.getVolumeName()); - // Get bucketInfo for the snapshot bucket to get bucket layout. - String dbBucketKey = ozoneManager.getMetadataManager().getBucketKey( - snapInfo.getVolumeName(), snapInfo.getBucketName()); - OmBucketInfo bucketInfo = ozoneManager.getMetadataManager() - .getBucketTable().get(dbBucketKey); - - if (bucketInfo == null) { - // Decrement ref count - rcOmSnapshot.close(); - rcOmSnapshot = null; - throw new IllegalStateException("Bucket " + "/" + - snapInfo.getVolumeName() + "/" + snapInfo.getBucketName() + - " is not found. BucketInfo should not be null for snapshotted" + - " bucket. The OM is in unexpected state."); - } - - String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; - String dbBucketKeyForDir = ozoneManager.getMetadataManager() - .getBucketKey(Long.toString(volumeId), - Long.toString(bucketInfo.getObjectID())) + OM_KEY_PREFIX; - - if (isSnapshotReclaimable(snapshotDeletedTable, - snapshotDeletedDirTable, snapshotBucketKey, dbBucketKeyForDir)) { - purgeSnapshotKeys.add(snapInfo.getTableKey()); - // Decrement ref count - rcOmSnapshot.close(); - rcOmSnapshot = null; + SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(snapInfo, chainManager, omSnapshotManager); + // Continue if the next snapshot is not active. This is to avoid unnecessary copies from one snapshot to + // another. + if (nextSnapshot != null && nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { continue; } - //TODO: [SNAPSHOT] Add lock to deletedTable and Active DB. - SnapshotInfo previousSnapshot = getPreviousActiveSnapshot( - snapInfo, chainManager, omSnapshotManager); - Table previousKeyTable = null; - Table previousDirTable = null; - OmSnapshot omPreviousSnapshot = null; - - // Split RepeatedOmKeyInfo and update current snapshot deletedKeyTable - // and next snapshot deletedKeyTable. - if (previousSnapshot != null) { - rcOmPreviousSnapshot = omSnapshotManager.getSnapshot( - previousSnapshot.getVolumeName(), - previousSnapshot.getBucketName(), - previousSnapshot.getName()); - omPreviousSnapshot = rcOmPreviousSnapshot.get(); - - previousKeyTable = omPreviousSnapshot - .getMetadataManager().getKeyTable(bucketInfo.getBucketLayout()); - previousDirTable = omPreviousSnapshot - .getMetadataManager().getDirectoryTable(); - } - - // Move key to either next non deleted snapshot's deletedTable - // or keep it in current snapshot deleted table. - List toReclaimList = new ArrayList<>(); - List toNextDBList = new ArrayList<>(); - // A list of renamed keys/files/dirs - List renamedList = new ArrayList<>(); - List dirsToMove = new ArrayList<>(); - - long remainNum = handleDirectoryCleanUp(snapshotDeletedDirTable, - previousDirTable, renamedTable, dbBucketKeyForDir, snapInfo, - omSnapshot, dirsToMove, renamedList); - int deletionCount = 0; - - try (TableIterator> deletedIterator = snapshotDeletedTable - .iterator()) { - - List keysToPurge = new ArrayList<>(); - deletedIterator.seek(snapshotBucketKey); - - while (deletedIterator.hasNext() && - deletionCount < remainNum) { - Table.KeyValue - deletedKeyValue = deletedIterator.next(); - String deletedKey = deletedKeyValue.getKey(); - - // Exit if it is out of the bucket scope. - if (!deletedKey.startsWith(snapshotBucketKey)) { - // If snapshot deletedKeyTable doesn't have any - // entry in the snapshot scope it can be reclaimed - break; - } - - RepeatedOmKeyInfo repeatedOmKeyInfo = deletedKeyValue.getValue(); - - SnapshotMoveKeyInfos.Builder toReclaim = SnapshotMoveKeyInfos - .newBuilder() - .setKey(deletedKey); - SnapshotMoveKeyInfos.Builder toNextDb = SnapshotMoveKeyInfos - .newBuilder() - .setKey(deletedKey); - HddsProtos.KeyValue.Builder renamedKey = HddsProtos.KeyValue - .newBuilder(); - - for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) { - splitRepeatedOmKeyInfo(toReclaim, toNextDb, renamedKey, - keyInfo, previousKeyTable, renamedTable, - bucketInfo, volumeId); - } - - // If all the KeyInfos are reclaimable in RepeatedOmKeyInfo - // then no need to update current snapshot deletedKeyTable. - if (!(toReclaim.getKeyInfosCount() == - repeatedOmKeyInfo.getOmKeyInfoList().size())) { - toReclaimList.add(toReclaim.build()); - toNextDBList.add(toNextDb.build()); - } else { - // The key can be reclaimed here. - List blocksForKeyDelete = omSnapshot - .getMetadataManager() - .getBlocksForKeyDelete(deletedKey); - if (blocksForKeyDelete != null) { - keysToPurge.addAll(blocksForKeyDelete); - } - } - - if (renamedKey.hasKey() && renamedKey.hasValue()) { - renamedList.add(renamedKey.build()); - } - deletionCount++; - } - - // Delete keys From deletedTable - processKeyDeletes(keysToPurge, omSnapshot.getKeyManager(), - null, snapInfo.getTableKey()); - successRunCount.incrementAndGet(); - } catch (IOException ex) { - LOG.error("Error while running Snapshot Deleting Service for " + - "snapshot " + snapInfo.getTableKey() + " with snapshotId " + - snapInfo.getSnapshotId() + ". Processed " + deletionCount + - " keys and " + (keyLimitPerSnapshot - remainNum) + - " directories and files", ex); + // Wait for the next iteration if the next snapshot is still running deep cleaning or AOS is running + // cleaning, since purge transaction will add entries and it could be processed by mistake. + if ((nextSnapshot == null && isDeepCleaningRunningOnAOS()) || + (nextSnapshot != null && isSnapshotDeepCleaningServiceRunning(nextSnapshot))) { + continue; } + successRunCount.incrementAndGet(); + submitSnapshotPurgeAndMoveRequest(snapInfo, nextSnapshot); snapshotLimit--; - // Submit Move request to OM. - submitSnapshotMoveDeletedKeys(snapInfo, toReclaimList, - toNextDBList, renamedList, dirsToMove); - - // Properly decrement ref count for rcOmPreviousSnapshot - if (rcOmPreviousSnapshot != null) { - rcOmPreviousSnapshot.close(); - rcOmPreviousSnapshot = null; - } } } catch (IOException e) { LOG.error("Error while running Snapshot Deleting Service", e); - } finally { - // Decrement ref counts - if (rcOmPreviousSnapshot != null) { - rcOmPreviousSnapshot.close(); - } - if (rcOmSnapshot != null) { - rcOmSnapshot.close(); - } } - submitSnapshotPurgeRequest(purgeSnapshotKeys); - return BackgroundTaskResult.EmptyTaskResult.newResult(); } - private boolean isSnapshotReclaimable( - Table snapshotDeletedTable, - Table snapshotDeletedDirTable, - String snapshotBucketKey, String dbBucketKeyForDir) throws IOException { - - boolean isDirTableCleanedUp = false; - boolean isKeyTableCleanedUp = false; - try (TableIterator> iterator = snapshotDeletedTable.iterator();) { - iterator.seek(snapshotBucketKey); - // If the next entry doesn't start with snapshotBucketKey then - // deletedKeyTable is already cleaned up. - isKeyTableCleanedUp = !iterator.hasNext() || !iterator.next().getKey() - .startsWith(snapshotBucketKey); - } - - try (TableIterator> - iterator = snapshotDeletedDirTable.iterator()) { - iterator.seek(dbBucketKeyForDir); - // If the next entry doesn't start with dbBucketKeyForDir then - // deletedDirTable is already cleaned up. - isDirTableCleanedUp = !iterator.hasNext() || !iterator.next().getKey() - .startsWith(dbBucketKeyForDir); - } + public void submitSnapshotPurgeAndMoveRequest(SnapshotInfo snapInfo, SnapshotInfo nextSnapshotInfo) { - return (isDirTableCleanedUp || snapshotDeletedDirTable.isEmpty()) && - (isKeyTableCleanedUp || snapshotDeletedTable.isEmpty()); - } - - @SuppressWarnings("checkstyle:ParameterNumber") - private long handleDirectoryCleanUp( - Table snapshotDeletedDirTable, - Table previousDirTable, - Table renamedTable, - String dbBucketKeyForDir, SnapshotInfo snapInfo, - OmSnapshot omSnapshot, List dirsToMove, - List renamedList) { - - long dirNum = 0L; - long subDirNum = 0L; - long subFileNum = 0L; - long remainNum = keyLimitPerSnapshot; - int consumedSize = 0; - List purgePathRequestList = new ArrayList<>(); - List> allSubDirList - = new ArrayList<>(keyLimitPerSnapshot); - try (TableIterator> deletedDirIterator = - snapshotDeletedDirTable.iterator()) { - - long startTime = Time.monotonicNow(); - deletedDirIterator.seek(dbBucketKeyForDir); - - while (deletedDirIterator.hasNext()) { - Table.KeyValue deletedDir = - deletedDirIterator.next(); - String deletedDirKey = deletedDir.getKey(); - - // Exit for dirs out of snapshot scope. - if (!deletedDirKey.startsWith(dbBucketKeyForDir)) { - break; - } + OzoneManagerProtocolProtos.SnapshotPurgeAndMoveRequest.Builder requestBuilder = + OzoneManagerProtocolProtos.SnapshotPurgeAndMoveRequest.newBuilder().setSnapshotID( + HddsUtils.toProtobuf(snapInfo.getSnapshotId())); - if (isDirReclaimable(deletedDir, previousDirTable, - renamedTable, renamedList)) { - // Reclaim here - PurgePathRequest request = prepareDeleteDirRequest( - remainNum, deletedDir.getValue(), deletedDir.getKey(), - allSubDirList, omSnapshot.getKeyManager()); - if (isBufferLimitCrossed(ratisByteLimit, consumedSize, - request.getSerializedSize())) { - if (purgePathRequestList.size() != 0) { - // if message buffer reaches max limit, avoid sending further - remainNum = 0; - break; - } - // if directory itself is having a lot of keys / files, - // reduce capacity to minimum level - remainNum = MIN_ERR_LIMIT_PER_TASK; - request = prepareDeleteDirRequest( - remainNum, deletedDir.getValue(), deletedDir.getKey(), - allSubDirList, omSnapshot.getKeyManager()); - } - consumedSize += request.getSerializedSize(); - purgePathRequestList.add(request); - remainNum = remainNum - request.getDeletedSubFilesCount(); - remainNum = remainNum - request.getMarkDeletedSubDirsCount(); - // Count up the purgeDeletedDir, subDirs and subFiles - if (request.getDeletedDir() != null - && !request.getDeletedDir().isEmpty()) { - dirNum++; - } - subDirNum += request.getMarkDeletedSubDirsCount(); - subFileNum += request.getDeletedSubFilesCount(); - } else { - dirsToMove.add(deletedDir.getKey()); - } - } - - remainNum = optimizeDirDeletesAndSubmitRequest(remainNum, dirNum, - subDirNum, subFileNum, allSubDirList, purgePathRequestList, - snapInfo.getTableKey(), startTime, ratisByteLimit - consumedSize, - omSnapshot.getKeyManager()); - } catch (IOException e) { - LOG.error("Error while running delete directories and files for " + - "snapshot " + snapInfo.getTableKey() + " in snapshot deleting " + - "background task. Will retry at next run.", e); + // When null it means next target is the AOS. + if (nextSnapshotInfo != null) { + requestBuilder.setExpectedNextSnapshotID(HddsUtils.toProtobuf(nextSnapshotInfo.getSnapshotId())); } - - return remainNum; - } - - private void submitSnapshotPurgeRequest(List purgeSnapshotKeys) { - if (!purgeSnapshotKeys.isEmpty()) { - SnapshotPurgeRequest snapshotPurgeRequest = SnapshotPurgeRequest - .newBuilder() - .addAllSnapshotDBKeys(purgeSnapshotKeys) - .build(); - - OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(Type.SnapshotPurge) - .setSnapshotPurgeRequest(snapshotPurgeRequest) - .setClientId(clientId.toString()) - .build(); - - submitRequest(omRequest); - } - } - - @SuppressWarnings("checkstyle:ParameterNumber") - private void splitRepeatedOmKeyInfo(SnapshotMoveKeyInfos.Builder toReclaim, - SnapshotMoveKeyInfos.Builder toNextDb, - HddsProtos.KeyValue.Builder renamedKey, OmKeyInfo keyInfo, - Table previousKeyTable, - Table renamedTable, - OmBucketInfo bucketInfo, long volumeId) throws IOException { - - if (isKeyReclaimable(previousKeyTable, renamedTable, - keyInfo, bucketInfo, volumeId, renamedKey)) { - // Update in current db's deletedKeyTable - toReclaim.addKeyInfos(keyInfo - .getProtobuf(ClientVersion.CURRENT_VERSION)); - } else { - // Move to next non deleted snapshot's deleted table - toNextDb.addKeyInfos(keyInfo.getProtobuf( - ClientVersion.CURRENT_VERSION)); - } - } - - private boolean isDirReclaimable( - Table.KeyValue deletedDir, - Table previousDirTable, - Table renamedTable, - List renamedList) throws IOException { - - if (previousDirTable == null) { - return true; - } - - String deletedDirDbKey = deletedDir.getKey(); - OmKeyInfo deletedDirInfo = deletedDir.getValue(); - String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey( - deletedDirInfo.getVolumeName(), deletedDirInfo.getBucketName(), - deletedDirInfo.getObjectID()); - - /* - snapshotRenamedTable: /volumeName/bucketName/objectID -> - /volumeId/bucketId/parentId/dirName - */ - String dbKeyBeforeRename = renamedTable.getIfExist(dbRenameKey); - String prevDbKey = null; - - if (dbKeyBeforeRename != null) { - prevDbKey = dbKeyBeforeRename; - HddsProtos.KeyValue renamedDir = HddsProtos.KeyValue - .newBuilder() - .setKey(dbRenameKey) - .setValue(dbKeyBeforeRename) - .build(); - renamedList.add(renamedDir); - } else { - // In OMKeyDeleteResponseWithFSO OzonePathKey is converted to - // OzoneDeletePathKey. Changing it back to check the previous DirTable. - prevDbKey = ozoneManager.getMetadataManager() - .getOzoneDeletePathDirKey(deletedDirDbKey); - } - - OmDirectoryInfo prevDirectoryInfo = previousDirTable.get(prevDbKey); - if (prevDirectoryInfo == null) { - return true; - } - - return prevDirectoryInfo.getObjectID() != deletedDirInfo.getObjectID(); - } - - public void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, - List toReclaimList, - List toNextDBList, - List renamedList, - List dirsToMove) throws InterruptedException { - - SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder = - SnapshotMoveDeletedKeysRequest.newBuilder() - .setFromSnapshot(snapInfo.getProtobuf()); - - SnapshotMoveDeletedKeysRequest moveDeletedKeys = moveDeletedKeysBuilder - .addAllReclaimKeys(toReclaimList) - .addAllNextDBKeys(toNextDBList) - .addAllRenamedKeys(renamedList) - .addAllDeletedDirsToMove(dirsToMove) - .build(); - OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(Type.SnapshotMoveDeletedKeys) - .setSnapshotMoveDeletedKeysRequest(moveDeletedKeys) + .setCmdType(Type.SnapshotPurgeAndMove) + .setSnapshotPurgeAndMoveRequest(requestBuilder.build()) .setClientId(clientId.toString()) .build(); @@ -588,11 +207,22 @@ public void submitRequest(OMRequest omRequest) { } @VisibleForTesting - boolean shouldIgnoreSnapshot(SnapshotInfo snapInfo) { + boolean shouldIgnoreSnapshot(SnapshotInfo snapInfo) throws IOException { SnapshotInfo.SnapshotStatus snapshotStatus = snapInfo.getSnapshotStatus(); - return snapshotStatus != SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED; + return snapshotStatus != SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED || + OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapInfo) + && isSnapshotDeepCleaningServiceRunning(snapInfo); + } + private boolean isSnapshotDeepCleaningServiceRunning(SnapshotInfo snapInfo) { + return (!snapInfo.getDeepClean() || !snapInfo.getDeepCleanedDeletedDir()); } + private boolean isDeepCleaningRunningOnAOS() { + return getOzoneManager().getKeyManager().getDeletingService().isRunningOnAOS() || + getOzoneManager().getKeyManager().getDirDeletingService().isRunningOnAOS(); + } + + // TODO: Move this util class. public static boolean isBlockLocationInfoSame(OmKeyInfo prevKeyInfo, OmKeyInfo deletedKeyInfo) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java index fe0f6e111ed3..89f78d8bb247 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java @@ -19,7 +19,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ServiceException; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hdds.client.BlockID; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; @@ -27,7 +28,9 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.common.BlockGroup; +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; @@ -38,15 +41,15 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientRequest; @@ -59,10 +62,15 @@ import java.util.Stack; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; -import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT; import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getDirectoryInfo; +import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getOmKeyInfo; +import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.setKeyNameAndFileName; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getOzonePathKeyForFso; /** @@ -75,10 +83,12 @@ public class SnapshotDirectoryCleaningService // or write to same tables and can send deletion requests for same key // multiple times. private static final int SNAPSHOT_DIR_CORE_POOL_SIZE = 1; - + private static final int MIN_ERR_LIMIT_PER_TASK = 1000; private final AtomicBoolean suspended; private final Map exclusiveSizeMap; private final Map exclusiveReplicatedSizeMap; + private final long keyLimitPerSnapshot; + private final int ratisByteLimit; public SnapshotDirectoryCleaningService(long interval, TimeUnit unit, long serviceTimeout, @@ -90,6 +100,15 @@ public SnapshotDirectoryCleaningService(long interval, TimeUnit unit, this.suspended = new AtomicBoolean(false); this.exclusiveSizeMap = new HashMap<>(); this.exclusiveReplicatedSizeMap = new HashMap<>(); + this.keyLimitPerSnapshot = ozoneManager.getConfiguration().getLong( + OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK, + OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); + int limit = (int) ozoneManager.getConfiguration().getStorageSize( + OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, + OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT, + StorageUnit.BYTES); + // always go to 90% of max limit for request as other header will be added + this.ratisByteLimit = (int) (limit * 0.9); } private boolean shouldRun() { @@ -119,11 +138,90 @@ public void resume() { @Override public BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); - queue.add(new SnapshotDirectoryCleaningService.SnapshotDirTask()); + queue.add(new SnapshotDirectoryCleaningService.SnapshotAOSDirCleanupTask()); return queue; } - private class SnapshotDirTask implements BackgroundTask { + private class SnapshotAOSDirCleanupTask implements BackgroundTask { + + //Expands deleted directory from active AOS if it is not referenced in the previous and breaks the dfs iteration. + private boolean expandDirectoryAndPurgeIfDirNotReferenced( + Table.KeyValue deletedDir, + Table previousDirTable, + Table renamedTable, + AtomicLong remainNum, AtomicInteger consumedSize, AtomicLong dirNum, + AtomicLong subDirNum, AtomicLong subFileNum, + List purgePathRequestList, + List> allSubDirList, + KeyManager keyManager) throws IOException { + if (isDirReclaimable(deletedDir, previousDirTable, renamedTable)) { + OzoneManagerProtocolProtos.PurgePathRequest request = prepareDeleteDirRequest( + remainNum.get(), deletedDir.getValue(), deletedDir.getKey(), + allSubDirList, keyManager); + if (isBufferLimitCrossed(ratisByteLimit, consumedSize.get(), + request.getSerializedSize())) { + if (purgePathRequestList.size() != 0) { + // if message buffer reaches max limit, avoid sending further + remainNum.set(0); + return false; + } + // if directory itself is having a lot of keys / files, + // reduce capacity to minimum level + remainNum.set(MIN_ERR_LIMIT_PER_TASK); + request = prepareDeleteDirRequest( + remainNum.get(), deletedDir.getValue(), deletedDir.getKey(), + allSubDirList, keyManager); + } + consumedSize.addAndGet(request.getSerializedSize()); + purgePathRequestList.add(request); + remainNum.addAndGet(-1 * request.getDeletedSubFilesCount()); + remainNum.addAndGet(-1 * request.getMarkDeletedSubDirsCount()); + // Count up the purgeDeletedDir, subDirs and subFiles + boolean dirDeleted = false; + if (request.getDeletedDir() != null + && !request.getDeletedDir().isEmpty()) { + dirNum.incrementAndGet(); + dirDeleted = true; + } + subDirNum.addAndGet(request.getMarkDeletedSubDirsCount()); + subFileNum.addAndGet(request.getDeletedSubFilesCount()); + // returning true if there are only files and all files along with the directory is removed. Otherwise this + // directory needs another iteration to cleanup. + return request.getMarkDeletedSubDirsCount() == 0 && dirDeleted; + } + return true; + } + + //Removes deleted file from AOS and moves if it is not referenced in the previous snapshot. + // Returns + private boolean deleteKeyIfNotReferencedInPreviousSnapshot( + long volumeId, OmBucketInfo bucketInfo, + Table.KeyValue deletedKeyInfoEntry, + SnapshotInfo prevSnapshotInfo, SnapshotInfo prevPrevSnapshotInfo, + Table renamedTable, Table prevRenamedTable, + Table previousKeyTable, Table previousPrevKeyTable, + List deletedKeyInfos, + AtomicLong remainNum, AtomicInteger consumedSize) throws IOException { + OmKeyInfo deletedKeyInfo = deletedKeyInfoEntry.getValue(); + if (isKeyReclaimable(previousKeyTable, renamedTable, deletedKeyInfo, bucketInfo, volumeId, null)) { + OzoneManagerProtocolProtos.KeyInfo keyInfo = deletedKeyInfo.getProtobuf(true, + ClientVersion.CURRENT_VERSION); + if (isBufferLimitCrossed(ratisByteLimit, consumedSize.get(), + keyInfo.getSerializedSize()) && consumedSize.get() > 0) { + // if message buffer reaches max limit, avoid sending further + remainNum.set(0); + return false; + } + deletedKeyInfos.add(keyInfo); + remainNum.decrementAndGet(); + consumedSize.addAndGet(keyInfo.getSerializedSize()); + } else { + calculateExclusiveSize(prevSnapshotInfo, prevPrevSnapshotInfo, deletedKeyInfo, bucketInfo, volumeId, + renamedTable, previousKeyTable, prevRenamedTable, previousPrevKeyTable, exclusiveSizeMap, + exclusiveReplicatedSizeMap); + } + return true; + } @Override public BackgroundTaskResult call() { @@ -142,16 +240,22 @@ public BackgroundTaskResult call() { SnapshotChainManager snapChainManager = metadataManager .getSnapshotChainManager(); + try (TableIterator> iterator = snapshotInfoTable.iterator()) { - while (iterator.hasNext()) { + AtomicLong dirNum = new AtomicLong(0); + AtomicLong subDirNum = new AtomicLong(0); + AtomicLong subFileNum = new AtomicLong(0); + AtomicLong remainNum = new AtomicLong(keyLimitPerSnapshot); + AtomicInteger consumedSize = new AtomicInteger(0); + // Getting the Snapshot info from cache. SnapshotInfo currSnapInfo = iterator.next().getValue(); - - // Expand deleted dirs only on active snapshot. Deleted Snapshots - // will be cleaned up by SnapshotDeletingService. - if (currSnapInfo.getSnapshotStatus() != SNAPSHOT_ACTIVE || - currSnapInfo.getDeepCleanedDeletedDir()) { + // Checking if all the transactions corresponding to the snapshot has been flushed by comparing the active + // om's transaction. Since here we are just iterating through snapshots only on disk. There is a + // possibility of missing deletedDirectory entries added from the previous run which has not been flushed. + if (currSnapInfo == null || currSnapInfo.getDeepCleanedDeletedDir() || + !OmSnapshotManager.areSnapshotChangesFlushedToDB(metadataManager, currSnapInfo.getTableKey())) { continue; } @@ -176,12 +280,15 @@ public BackgroundTaskResult call() { "unexpected state."); } - SnapshotInfo previousSnapshot = getPreviousActiveSnapshot( + SnapshotInfo previousSnapshot = getPreviousSnapshot( currSnapInfo, snapChainManager, omSnapshotManager); - SnapshotInfo previousToPrevSnapshot = null; + final SnapshotInfo previousToPrevSnapshot; - Table previousKeyTable = null; - Table prevRenamedTable = null; + final Table previousKeyTable; + final Table previousDirTable; + final Table prevRenamedTable; + + final Table previousToPrevKeyTable; if (previousSnapshot != null) { rcPrevOmSnapshot = omSnapshotManager.getActiveSnapshot( @@ -194,11 +301,17 @@ public BackgroundTaskResult call() { .getKeyTable(bucketInfo.getBucketLayout()); prevRenamedTable = omPreviousSnapshot .getMetadataManager().getSnapshotRenamedTable(); - previousToPrevSnapshot = getPreviousActiveSnapshot( + previousDirTable = omPreviousSnapshot.getMetadataManager().getDirectoryTable(); + previousToPrevSnapshot = getPreviousSnapshot( previousSnapshot, snapChainManager, omSnapshotManager); + } else { + previousKeyTable = null; + previousDirTable = null; + prevRenamedTable = null; + previousToPrevSnapshot = null; } - Table previousToPrevKeyTable = null; + if (previousToPrevSnapshot != null) { rcPrevToPrevOmSnapshot = omSnapshotManager.getActiveSnapshot( previousToPrevSnapshot.getVolumeName(), @@ -209,12 +322,13 @@ public BackgroundTaskResult call() { previousToPrevKeyTable = omPreviousToPrevSnapshot .getMetadataManager() .getKeyTable(bucketInfo.getBucketLayout()); + } else { + previousToPrevKeyTable = null; } String dbBucketKeyForDir = getOzonePathKeyForFso(metadataManager, currSnapInfo.getVolumeName(), currSnapInfo.getBucketName()); - try (ReferenceCounted - rcCurrOmSnapshot = omSnapshotManager.getActiveSnapshot( + try (ReferenceCounted rcCurrOmSnapshot = omSnapshotManager.getSnapshot( currSnapInfo.getVolumeName(), currSnapInfo.getBucketName(), currSnapInfo.getName())) { @@ -222,30 +336,61 @@ public BackgroundTaskResult call() { OmSnapshot currOmSnapshot = rcCurrOmSnapshot.get(); Table snapDeletedDirTable = currOmSnapshot.getMetadataManager().getDeletedDirTable(); + Table snapRenameTable = + currOmSnapshot.getMetadataManager().getSnapshotRenamedTable(); try (TableIterator> deletedDirIterator = snapDeletedDirTable .iterator(dbBucketKeyForDir)) { - + long startTime = Time.monotonicNow(); + List purgePathRequestList = new ArrayList<>(); + List> allSubDirList = new ArrayList<>(); + List deletedSubKeyInfos = new ArrayList<>(); + Operation, Boolean> operationOnDirectory = + deletedDirectoryEntry -> expandDirectoryAndPurgeIfDirNotReferenced(deletedDirectoryEntry, previousDirTable, + snapRenameTable, remainNum, consumedSize, dirNum, subDirNum, subFileNum, + purgePathRequestList, allSubDirList, getOzoneManager().getKeyManager());; + + Operation, Boolean> operationOnFile = + deletedKeyInfo -> deleteKeyIfNotReferencedInPreviousSnapshot(volumeId, bucketInfo, deletedKeyInfo, + previousSnapshot, previousToPrevSnapshot, snapRenameTable, prevRenamedTable, + previousKeyTable, previousToPrevKeyTable, deletedSubKeyInfos, remainNum, consumedSize);; + Supplier breakConditionSupplier = () -> remainNum.get() > 0; + boolean retVal = true; while (deletedDirIterator.hasNext()) { Table.KeyValue deletedDirInfo = deletedDirIterator.next(); - + if (breakConditionSupplier.get()) { + retVal = false; + break; + } // For each deleted directory we do an in-memory DFS and - // do a deep clean and exclusive size calculation. - iterateDirectoryTree(deletedDirInfo, volumeId, bucketInfo, - previousSnapshot, previousToPrevSnapshot, - currOmSnapshot, previousKeyTable, prevRenamedTable, - previousToPrevKeyTable, dbBucketKeyForDir); + // do a deep clean based on the previous snapshot. + retVal = retVal && iterateDirectoryTree(deletedDirInfo, volumeId, bucketInfo, + operationOnFile, operationOnDirectory, + breakConditionSupplier, metadataManager); } - updateDeepCleanSnapshotDir(currSnapInfo.getTableKey()); - if (previousSnapshot != null) { - updateExclusiveSize(previousSnapshot.getTableKey()); + if (deletedSubKeyInfos.size() > 0) { + purgePathRequestList.add(wrapPurgeRequest(volumeId, bucketInfo.getObjectID(), deletedSubKeyInfos)); + } + retVal = retVal && optimizeDirDeletesAndSubmitRequest(0, dirNum.get(), subDirNum.get(), + subFileNum.get(), + allSubDirList, purgePathRequestList, currSnapInfo.getTableKey(), startTime, 0, + getOzoneManager().getKeyManager()).getRight().map(OzoneManagerProtocolProtos.OMResponse::getSuccess) + .orElse(false); + List setSnapshotPropertyRequests = new ArrayList<>(); + if (retVal && subDirNum.get() == 0) { + if (previousSnapshot != null) { + setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(previousSnapshot.getTableKey())); + } + setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir( + currSnapInfo.getTableKey())); + submitSetSnapshotRequest(setSnapshotPropertyRequests); } } } } finally { - IOUtils.closeQuietly(rcPrevOmSnapshot, rcPrevToPrevOmSnapshot); + IOUtils.closeQuietly(rcPrevOmSnapshot); } } } catch (IOException ex) { @@ -256,55 +401,80 @@ public BackgroundTaskResult call() { } } - @SuppressWarnings("checkstyle:ParameterNumber") - private void iterateDirectoryTree( - Table.KeyValue deletedDirInfo, long volumeId, - OmBucketInfo bucketInfo, - SnapshotInfo previousSnapshot, - SnapshotInfo previousToPrevSnapshot, - OmSnapshot currOmSnapshot, - Table previousKeyTable, - Table prevRenamedTable, - Table previousToPrevKeyTable, - String dbBucketKeyForDir) throws IOException { - - Table snapDirTable = - currOmSnapshot.getMetadataManager().getDirectoryTable(); - Table snapRenamedTable = - currOmSnapshot.getMetadataManager().getSnapshotRenamedTable(); - + /** + * Performs a DFS iteration on a given deleted directory and performs operation on the sub deleted directories and + * subfiles in the tree. + * @param deletedDirInfo + * @param volumeId + * @param bucketInfo + * @param operationOnFile + * @param operationOnDirectory + * @param breakConditionSupplier + * @param metadataManager + * @return True if complete directory was iterated and all operations returned a true otherwise false. + * middle of iteration. + * @throws IOException + */ + private boolean iterateDirectoryTree( + Table.KeyValue deletedDirInfo, + long volumeId, OmBucketInfo bucketInfo, Operation, Boolean> operationOnFile, + Operation, Boolean> operationOnDirectory, + Supplier breakConditionSupplier, + OMMetadataManager metadataManager) throws IOException { + Table dirTable = metadataManager.getDirectoryTable(); + Table fileTable = metadataManager.getFileTable(); Stack stackNodes = new Stack<>(); - OmDirectoryInfo omDeletedDirectoryInfo = - getDirectoryInfo(deletedDirInfo.getValue()); - String dirPathDbKey = currOmSnapshot.getMetadataManager() - .getOzonePathKey(volumeId, bucketInfo.getObjectID(), - omDeletedDirectoryInfo); - // Stack Init + OmDirectoryInfo omDeletedDirectoryInfo = getDirectoryInfo(deletedDirInfo.getValue()); + String dirPathDbKey = metadataManager.getOzonePathKey(volumeId, bucketInfo.getObjectID(), + omDeletedDirectoryInfo); StackNode topLevelDir = new StackNode(); topLevelDir.setDirKey(dirPathDbKey); - topLevelDir.setDirValue(omDeletedDirectoryInfo); + topLevelDir.setDirValue(deletedDirInfo.getValue()); stackNodes.push(topLevelDir); - + String bucketPrefixKeyFSO = getOzonePathKeyForFso(metadataManager, bucketInfo.getVolumeName(), + bucketInfo.getBucketName()); + boolean retVal = true; try (TableIterator> - directoryIterator = snapDirTable.iterator(dbBucketKeyForDir)) { - + directoryIterator = dirTable.iterator(bucketPrefixKeyFSO); + TableIterator> fileIterator = + fileTable.iterator(bucketPrefixKeyFSO)) { while (!stackNodes.isEmpty()) { StackNode stackTop = stackNodes.peek(); - // First process all the files in the current directory - // and then do a DFS for directory. + // We are doing a pre order traversal here meaning, first process the current directory and all the files + // and then do a DFS for directory. This is so that we can exit early avoiding unnecessary traversal. if (StringUtils.isEmpty(stackTop.getSubDirSeek())) { - processFilesUnderDir(previousSnapshot, - previousToPrevSnapshot, - volumeId, - bucketInfo, - stackTop.getDirValue(), - currOmSnapshot.getMetadataManager(), - snapRenamedTable, - previousKeyTable, - prevRenamedTable, - previousToPrevKeyTable); + boolean directoryOpReturnValue = operationOnDirectory.apply(Table.newKeyValue(stackTop.getDirKey(), + stackTop.getDirValue())); + if (!directoryOpReturnValue) { + retVal = false; + stackNodes.pop(); + } + if (breakConditionSupplier.get()) { + return retVal; + } + String subFileSeekValue = metadataManager.getOzonePathKey(volumeId, + bucketInfo.getObjectID(), + stackTop.getDirValue().getObjectID(), ""); + fileIterator.seek(subFileSeekValue); + while (fileIterator.hasNext()) { + Table.KeyValue fileEntry = fileIterator.next(); + OmKeyInfo omKeyInfo = fileEntry.getValue(); + if (!OMFileRequest.isImmediateChild(omKeyInfo.getParentObjectID(), + stackTop.getDirValue().getObjectID())) { + break; + } + setKeyNameAndFileName(stackTop.getDirValue(), omKeyInfo); + if(!operationOnFile.apply(fileEntry)) { + retVal = false; + stackNodes.pop(); + break; + } + if (breakConditionSupplier.get()) { + return retVal; + } + } // Format : /volId/bucketId/parentId/ - String seekDirInDB = currOmSnapshot.getMetadataManager() + String seekDirInDB = metadataManager .getOzonePathKey(volumeId, bucketInfo.getObjectID(), stackTop.getDirValue().getObjectID(), ""); stackTop.setSubDirSeek(seekDirInDB); @@ -312,11 +482,9 @@ private void iterateDirectoryTree( // Adding \0 to seek the next greater element. directoryIterator.seek(stackTop.getSubDirSeek() + "\0"); if (directoryIterator.hasNext()) { - Table.KeyValue deletedSubDirInfo = directoryIterator.next(); String deletedSubDirKey = deletedSubDirInfo.getKey(); - String prefixCheck = currOmSnapshot.getMetadataManager() - .getOzoneDeletePathDirKey(stackTop.getSubDirSeek()); + String prefixCheck = metadataManager.getOzoneDeletePathDirKey(stackTop.getSubDirSeek()); // Exit if it is out of the sub dir prefix scope. if (!deletedSubDirKey.startsWith(prefixCheck)) { stackNodes.pop(); @@ -324,7 +492,7 @@ private void iterateDirectoryTree( stackTop.setSubDirSeek(deletedSubDirKey); StackNode nextSubDir = new StackNode(); nextSubDir.setDirKey(deletedSubDirInfo.getKey()); - nextSubDir.setDirValue(deletedSubDirInfo.getValue()); + nextSubDir.setDirValue(getOmKeyInfo(stackTop.getDirValue(), deletedSubDirInfo.getValue())); stackNodes.push(nextSubDir); } } else { @@ -333,10 +501,10 @@ private void iterateDirectoryTree( } } } + return retVal; } - private void updateExclusiveSize(String prevSnapshotKeyTable) { - ClientId clientId = ClientId.randomId(); + private SetSnapshotPropertyRequest getSetSnapshotRequestUpdatingExclusiveSize(String prevSnapshotKeyTable) { SnapshotSize snapshotSize = SnapshotSize.newBuilder() .setExclusiveSize( exclusiveSizeMap.getOrDefault(prevSnapshotKeyTable, 0L)) @@ -346,93 +514,27 @@ private void updateExclusiveSize(String prevSnapshotKeyTable) { .build(); exclusiveSizeMap.remove(prevSnapshotKeyTable); exclusiveReplicatedSizeMap.remove(prevSnapshotKeyTable); - SetSnapshotPropertyRequest - setSnapshotPropertyRequest = - SetSnapshotPropertyRequest.newBuilder() - .setSnapshotKey(prevSnapshotKeyTable) - .setSnapshotSize(snapshotSize) - .build(); - - OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(Type.SetSnapshotProperty) - .setSetSnapshotPropertyRequest(setSnapshotPropertyRequest) - .setClientId(clientId.toString()) - .build(); - submitRequest(omRequest, clientId); + return SetSnapshotPropertyRequest.newBuilder() + .setSnapshotKey(prevSnapshotKeyTable) + .setSnapshotSize(snapshotSize) + .build(); } - @SuppressWarnings("checkstyle:ParameterNumber") - private void processFilesUnderDir( - SnapshotInfo previousSnapshot, - SnapshotInfo previousToPrevSnapshot, - long volumeId, - OmBucketInfo bucketInfo, - OmDirectoryInfo parentInfo, - OMMetadataManager metadataManager, - Table snapRenamedTable, - Table previousKeyTable, - Table prevRenamedTable, - Table previousToPrevKeyTable) - throws IOException { - String seekFileInDB = metadataManager.getOzonePathKey(volumeId, - bucketInfo.getObjectID(), - parentInfo.getObjectID(), ""); - List blocksForKeyDelete = new ArrayList<>(); - - Table fileTable = metadataManager.getFileTable(); - try (TableIterator> - iterator = fileTable.iterator(seekFileInDB)) { - - while (iterator.hasNext()) { - Table.KeyValue entry = iterator.next(); - OmKeyInfo fileInfo = entry.getValue(); - if (!OMFileRequest.isImmediateChild(fileInfo.getParentObjectID(), - parentInfo.getObjectID())) { - break; - } - - String ozoneDeletePathKey = metadataManager - .getOzoneDeletePathKey(fileInfo.getObjectID(), entry.getKey()); - if (isKeyReclaimable(previousKeyTable, snapRenamedTable, - fileInfo, bucketInfo, volumeId, null)) { - for (OmKeyLocationInfoGroup keyLocations : - fileInfo.getKeyLocationVersions()) { - List item = keyLocations.getLocationList().stream() - .map(b -> new BlockID(b.getContainerID(), b.getLocalID())) - .collect(Collectors.toList()); - BlockGroup keyBlocks = BlockGroup.newBuilder() - .setKeyName(ozoneDeletePathKey) - .addAllBlockIDs(item) - .build(); - blocksForKeyDelete.add(keyBlocks); - } - // TODO: Add Retry mechanism. - getScmClient().deleteKeyBlocks(blocksForKeyDelete); - } else if (previousSnapshot != null) { - calculateExclusiveSize(previousSnapshot, previousToPrevSnapshot, - fileInfo, bucketInfo, volumeId, snapRenamedTable, - previousKeyTable, prevRenamedTable, previousToPrevKeyTable, - exclusiveSizeMap, exclusiveReplicatedSizeMap); - } - } - } - } - - private void updateDeepCleanSnapshotDir(String snapshotKeyTable) { - ClientId clientId = ClientId.randomId(); - SetSnapshotPropertyRequest setSnapshotPropertyRequest = - SetSnapshotPropertyRequest.newBuilder() + private SetSnapshotPropertyRequest getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir(String snapshotKeyTable) { + return SetSnapshotPropertyRequest.newBuilder() .setSnapshotKey(snapshotKeyTable) .setDeepCleanedDeletedDir(true) .build(); + } + private void submitSetSnapshotRequest(List setSnapshotPropertyRequests) { + ClientId clientId = ClientId.randomId(); OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(Type.SetSnapshotProperty) - .setSetSnapshotPropertyRequest(setSnapshotPropertyRequest) - .setClientId(clientId.toString()) - .build(); - + .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) + .addAllSetSnapshotPropertyRequests(setSnapshotPropertyRequests) + .setClientId(clientId.toString()) + .build(); submitRequest(omRequest, clientId); } @@ -468,7 +570,7 @@ public void submitRequest(OMRequest omRequest, ClientId clientId) { */ private static class StackNode { private String dirKey; - private OmDirectoryInfo dirValue; + private OmKeyInfo dirValue; private String subDirSeek; public String getDirKey() { @@ -479,11 +581,11 @@ public void setDirKey(String dirKey) { this.dirKey = dirKey; } - public OmDirectoryInfo getDirValue() { + public OmKeyInfo getDirValue() { return dirValue; } - public void setDirValue(OmDirectoryInfo dirValue) { + public void setDirValue(OmKeyInfo dirValue) { this.dirValue = dirValue; } @@ -504,4 +606,9 @@ public String toString() { '}'; } } + + private interface Operation { + R apply(T t) throws IOException; + + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index 2041fa791a76..4cbef9812713 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.snapshot; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -27,6 +28,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; +import org.apache.ratis.server.protocol.TermIndex; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.slf4j.Logger; @@ -139,38 +141,89 @@ public static void checkSnapshotActive(SnapshotInfo snapInfo, } } + public static SnapshotInfo getSnapshotInfo(SnapshotChainManager chainManager, UUID snapshotId, + OmSnapshotManager omSnapshotManager) throws IOException { + String tableKey = chainManager.getTableKey(snapshotId); + return omSnapshotManager.getSnapshotInfo(tableKey); + } + /** * Get the next non deleted snapshot in the snapshot chain. */ public static SnapshotInfo getNextActiveSnapshot(SnapshotInfo snapInfo, - SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager) - throws IOException { + SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager) throws IOException { + do { + snapInfo = getNextSnapshot(snapInfo, chainManager, omSnapshotManager); + } while (snapInfo != null && !snapInfo.getSnapshotStatus().equals(SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)); + return snapInfo; + } + + /** + * Set Sanspho + */ + public static void setTransactionInfoInSnapshot(SnapshotInfo snapshot, TermIndex termIndex) throws IOException { + TransactionInfo transactionInfo = TransactionInfo.valueOf(termIndex); + snapshot.setLastTransactionInfo(TransactionInfo.getCodec().toPersistedFormat(transactionInfo)); + } + /** + * Get the previous non deleted snapshot in the snapshot chain. + */ + public static SnapshotInfo getPreviousActiveSnapshot(SnapshotInfo snapInfo, + SnapshotChainManager chainManager, + OmSnapshotManager omSnapshotManager) throws IOException { + do { + snapInfo = getPreviousSnapshot(snapInfo, chainManager, omSnapshotManager); + } while (snapInfo != null && !snapInfo.getSnapshotStatus().equals(SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)); + return snapInfo; + } + + /** + * Get the next in the snapshot chain. + */ + public static SnapshotInfo getNextSnapshot(SnapshotInfo snapInfo, + SnapshotChainManager chainManager, + OmSnapshotManager omSnapshotManager) + throws IOException { // If the snapshot is deleted in the previous run, then the in-memory // SnapshotChainManager might throw NoSuchElementException as the snapshot // is removed in-memory but OMDoubleBuffer has not flushed yet. if (snapInfo == null) { throw new OMException("Snapshot Info is null. Cannot get the next snapshot", INVALID_SNAPSHOT_ERROR); } - try { - while (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), + if (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId())) { + UUID nextPathSnapshot = chainManager.nextPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); + return getSnapshotInfo(chainManager, nextPathSnapshot, omSnapshotManager); + } + } catch (NoSuchElementException ex) { + LOG.error("The snapshot {} is not longer in snapshot chain, It " + + "maybe removed in the previous Snapshot purge request.", + snapInfo.getTableKey()); + } + return null; + } - UUID nextPathSnapshot = - chainManager.nextPathSnapshot( - snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); - - String tableKey = chainManager.getTableKey(nextPathSnapshot); - SnapshotInfo nextSnapshotInfo = - omSnapshotManager.getSnapshotInfo(tableKey); - - if (nextSnapshotInfo.getSnapshotStatus().equals( - SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)) { - return nextSnapshotInfo; - } - - snapInfo = nextSnapshotInfo; + /** + * Get the previous in the snapshot chain. + */ + public static SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo, + SnapshotChainManager chainManager, + OmSnapshotManager omSnapshotManager) + throws IOException { + // If the snapshot is deleted in the previous run, then the in-memory + // SnapshotChainManager might throw NoSuchElementException as the snapshot + // is removed in-memory but OMDoubleBuffer has not flushed yet. + if (snapInfo == null) { + throw new OMException("Snapshot Info is null. Cannot get the next snapshot", INVALID_SNAPSHOT_ERROR); + } + try { + if (chainManager.hasPreviousPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotId())) { + UUID previousPathSnapshot = chainManager.previousPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotId()); + return getSnapshotInfo(chainManager, previousPathSnapshot, omSnapshotManager); } } catch (NoSuchElementException ex) { LOG.error("The snapshot {} is not longer in snapshot chain, It " + From c4e78d9ad42c438d1cd840696cc279da1e2442f4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 5 Sep 2024 06:48:06 -0700 Subject: [PATCH 02/22] Safety commit Change-Id: Ib03867d243cdbc9b6f9e36eab23fd5455aa747f1 --- .../hdds/scm/exceptions/SCMException.java | 3 + .../ozone/util/CheckExceptionOperation.java | 9 + .../scm/server/SCMBlockProtocolServer.java | 13 +- .../ozone/om/lock/OzoneManagerLock.java | 3 +- .../src/main/proto/OmClientProtocol.proto | 13 +- .../hadoop/ozone/om/OMMetadataManager.java | 21 + .../apache/hadoop/ozone/om/KeyManager.java | 47 +- .../hadoop/ozone/om/KeyManagerImpl.java | 142 ++++- .../ozone/om/OmMetadataManagerImpl.java | 209 +------ .../hadoop/ozone/om/PendingKeysDeletion.java | 15 +- .../ratis/utils/OzoneManagerRatisUtils.java | 3 - .../key/OMDirectoriesPurgeRequestWithFSO.java | 46 +- .../om/request/key/OMKeyPurgeRequest.java | 72 ++- .../snapshot/OMSnapshotCreateRequest.java | 24 +- .../OMSnapshotMoveDeletedKeysRequest.java | 68 ++- .../OMSnapshotPurgeAndMoveRequest.java | 248 -------- .../snapshot/OMSnapshotPurgeRequest.java | 29 +- .../om/response/key/OMKeyPurgeResponse.java | 7 + .../OMSnapshotMoveDeletedKeysResponse.java | 116 ++-- .../OMSnapshotPurgeAndMoveResponse.java | 227 -------- .../service/AbstractKeyDeletingService.java | 539 ++++++++++++++++-- .../om/service/DirectoryDeletingService.java | 311 ++++++---- .../ozone/om/service/KeyDeletingService.java | 44 +- .../SnapshotDirectoryCleaningService.java | 294 +++------- .../ozone/om/snapshot/SnapshotUtils.java | 111 ++-- .../ozone/om/request/OMRequestTestUtils.java | 10 + .../key/TestOMKeyPurgeRequestAndResponse.java | 58 +- 27 files changed, 1417 insertions(+), 1265 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckExceptionOperation.java delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeAndMoveRequest.java delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeAndMoveResponse.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java index b1dffa5e5135..6a4b20a4553d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java @@ -112,6 +112,9 @@ public enum ResultCodes { FAILED_TO_FIND_CONTAINER, FAILED_TO_FIND_CONTAINER_WITH_SPACE, BLOCK_EXISTS, + //OM can send a deleteBlockRequest for key delete more than once. Thus, it shouldn't + // fail the request since SCM already deleted the block in the previous run. We should never use this enum. + @Deprecated FAILED_TO_FIND_BLOCK, IO_EXCEPTION, UNEXPECTED_CONTAINER_STATE, diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckExceptionOperation.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckExceptionOperation.java new file mode 100644 index 000000000000..73e939014af2 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckExceptionOperation.java @@ -0,0 +1,9 @@ +package org.apache.hadoop.ozone.util; + +public interface CheckExceptionOperation { + R apply(T t) throws E; + + default CheckExceptionOperation andThen(CheckExceptionOperation operation) throws E { + return (T t) -> operation.apply(this.apply(t)); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 7ed23b110c78..9f7f0b0a93b3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -282,18 +282,11 @@ public List deleteKeyBlocks( e = ioe; perfMetrics.updateDeleteKeyFailureStats(startNanos); LOG.warn("Fail to delete {} keys", keyBlocksInfoList.size(), ioe); - switch (ioe instanceof SCMException ? ((SCMException) ioe).getResult() : - IO_EXCEPTION) { - case SAFE_MODE_EXCEPTION: + if ((ioe instanceof SCMException ? ((SCMException) ioe).getResult() : + IO_EXCEPTION) == SCMException.ResultCodes.SAFE_MODE_EXCEPTION) { resultCode = ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result.safeMode; - break; - case FAILED_TO_FIND_BLOCK: - resultCode = - ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result. - errorNotFound; - break; - default: + } else { resultCode = ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result. unknownFailure; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java index aadb6c02c763..a0a13b984b4b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java @@ -457,7 +457,8 @@ public enum Resource { S3_SECRET_LOCK((byte) 4, "S3_SECRET_LOCK"), // 31 KEY_PATH_LOCK((byte) 5, "KEY_PATH_LOCK"), //63 PREFIX_LOCK((byte) 6, "PREFIX_LOCK"), //127 - SNAPSHOT_LOCK((byte) 7, "SNAPSHOT_LOCK"); // = 255 + SNAPSHOT_LOCK((byte) 7, "SNAPSHOT_LOCK"), // = 255 + SNAPSHOT_GC_LOCK((byte) 8, "SNAPSHOT_GC_LOCK"); // level of the resource private byte lockLevel; diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index e5f736fb5d6b..37cc8852abc1 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -149,7 +149,6 @@ enum Type { RenameSnapshot = 131; ListOpenFiles = 132; - SnapshotPurgeAndMove = 133; } enum SafeMode { @@ -1356,6 +1355,8 @@ message DeletedKeys { required string volumeName = 1; required string bucketName = 2; repeated string keys = 3; + optional hadoop.hdds.UUID expectedPathPreviousSnapshotUUID = 4; + repeated string renamedKeys = 5; } message PurgeKeysRequest { @@ -1397,6 +1398,9 @@ message PurgePathRequest { optional string deletedDir = 3; repeated KeyInfo deletedSubFiles = 4; repeated KeyInfo markDeletedSubDirs = 5; + optional hadoop.hdds.UUID expectedPathPreviousSnapshotUUID = 6; + optional string volumeName = 7; + optional string bucketName = 8; } message DeleteOpenKeysRequest { @@ -1958,11 +1962,13 @@ message PrintCompactionLogDagRequest { } message SnapshotMoveDeletedKeysRequest { - optional SnapshotInfo fromSnapshot = 1; + optional SnapshotInfo fromSnapshot = 1 [deprecated = true]; repeated SnapshotMoveKeyInfos nextDBKeys = 2; - repeated SnapshotMoveKeyInfos reclaimKeys = 3; + repeated SnapshotMoveKeyInfos reclaimKeys = 3 [deprecated = true]; repeated hadoop.hdds.KeyValue renamedKeys = 4; repeated string deletedDirsToMove = 5; + optional bool purgeMovedKeys = 6; + optional hadoop.hdds.UUID fromSnapshotID = 7; } message SnapshotMoveKeyInfos { @@ -1986,6 +1992,7 @@ message SetSnapshotPropertyRequest { optional SnapshotSize snapshotSize = 3; optional bool deepCleanedDeletedDir = 4; optional bool deepCleanedDeletedKey = 5; + optional SnapshotSize snapshotDirSize = 6; } // SnapshotProperty in entirely deprecated, Keeping it here for proto.lock compatibility diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index fb34d19a8bb1..48c63126afce 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -116,6 +116,22 @@ public interface OMMetadataManager extends DBStoreHAManager { */ String getBucketKey(String volume, String bucket); + /** + * Given a volume and bucket, return the corresponding DB key prefix. + * + * @param volume - User name + * @param bucket - Bucket name + */ + String getBucketKeyPrefix(String volume, String bucket); + + /** + * Given a volume and bucket, return the corresponding DB key prefix. + * + * @param volume - User name + * @param bucket - Bucket name + */ + String getBucketKeyPrefixFSO(String volume, String bucket) throws IOException; + /** * Given a volume, bucket and a key, return the corresponding DB key. * @@ -619,6 +635,11 @@ default String getOpenFileName(long volumeId, long bucketId, long parentObjectId */ String getRenameKey(String volume, String bucket, long objectID); + /** + * Given renameKey, return the volume, bucket & objectID from the key. + */ + String[] splitRenameKey(String renameKey); + /** * Returns the DB key name of a multipart upload key in OM metadata store * for FSO-enabled buckets. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 6b632a2d69e7..b7df7b2763e5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -18,6 +18,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -34,9 +35,11 @@ import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDirectoryCleaningService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket; +import org.apache.hadoop.ozone.util.CheckExceptionOperation; import java.io.IOException; import java.time.Duration; +import java.util.Iterator; import java.util.List; /** @@ -137,7 +140,24 @@ List listTrash(String volumeName, String bucketName, * and a hashmap for key-value pair to be updated in the deletedTable. * @throws IOException */ - PendingKeysDeletion getPendingDeletionKeys(int count) throws IOException; + PendingKeysDeletion getPendingDeletionKeys( + CheckExceptionOperation, Boolean, IOException> filter, int count) + throws IOException; + /** + * Returns a PendingKeysDeletion. It has a list of pending deletion key info + * that ups to the given count.Each entry is a {@link BlockGroup}, which + * contains the info about the key name and all its associated block IDs. + * Second is a Mapping of Key-Value pair which is updated in the deletedTable. + * + * @param count max number of keys to return. + * @return a Pair of list of {@link BlockGroup} representing keys and blocks, + * and a hashmap for key-value pair to be updated in the deletedTable. + * @throws IOException + */ + PendingKeysDeletion getPendingDeletionKeys( + String volume, String bucket, String startKey, + CheckExceptionOperation, Boolean, IOException> filter, int count) + throws IOException; /** * Returns the names of up to {@code count} open keys whose age is @@ -236,6 +256,15 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, */ Table.KeyValue getPendingDeletionDir() throws IOException; + /** + * Returns an iterator for pending deleted directories. + * @throws IOException + */ + TableIterator> getPendingDeletionDirs() throws IOException; + + TableIterator> getPendingDeletionDirs(String volume, String bucket) + throws IOException; + /** * Returns all sub directories under the given parent directory. * @@ -244,8 +273,12 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, * @return list of dirs * @throws IOException */ - List getPendingDeletionSubDirs(long volumeId, long bucketId, - OmKeyInfo parentInfo, long numEntries) throws IOException; + List getPendingDeletionSubDirs( + long volumeId, long bucketId, OmKeyInfo parentInfo, + CheckExceptionOperation, Boolean, IOException> filter, + long numEntries) throws IOException; + + /** * Returns all sub files under the given parent directory. @@ -255,9 +288,10 @@ List getPendingDeletionSubDirs(long volumeId, long bucketId, * @return list of files * @throws IOException */ - List getPendingDeletionSubFiles(long volumeId, - long bucketId, OmKeyInfo parentInfo, long numEntries) - throws IOException; + List getPendingDeletionSubFiles(long volumeId, long bucketId, OmKeyInfo parentInfo, + CheckExceptionOperation, Boolean, + IOException> filter, + long numEntries) throws IOException; /** * Returns the instance of Directory Deleting Service. @@ -294,4 +328,5 @@ List getPendingDeletionSubFiles(long volumeId, * @return Background service. */ SnapshotDirectoryCleaningService getSnapshotDirectoryService(); + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 99501f30c1a5..07e85c123c0f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -40,12 +40,14 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.google.common.collect.Lists; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdds.DFSConfigKeysLegacy; +import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -66,6 +68,10 @@ import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.TableMapping; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.common.BlockGroup; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.util.CheckExceptionOperation; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; @@ -133,6 +139,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.ETAG; +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT; @@ -158,6 +165,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.SCM_GET_PIPELINE_EXCEPTION; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; +import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.KEY; @@ -675,13 +683,71 @@ public List listTrash(String volumeName, startKeyName, keyPrefix, maxKeys); } + + @Override - public PendingKeysDeletion getPendingDeletionKeys(final int count) - throws IOException { - OmMetadataManagerImpl omMetadataManager = - (OmMetadataManagerImpl) metadataManager; - return omMetadataManager - .getPendingDeletionKeys(count, ozoneManager.getOmSnapshotManager()); + public PendingKeysDeletion getPendingDeletionKeys(CheckExceptionOperation, Boolean, + IOException> filter, int count) throws IOException { + return getPendingDeletionKeys(null, null, null, filter, count); + } + + @Override + public PendingKeysDeletion getPendingDeletionKeys( + String volume, String bucket, String startKey, + CheckExceptionOperation, Boolean, IOException> filter, + int count) throws IOException { + List keyBlocksList = Lists.newArrayList(); + Map keysToModify = new HashMap<>(); + final String nextPageStartKey; + // Bucket prefix would be empty if volume is empty i.e. either null or "". + Optional bucketPrefix = Optional.ofNullable(volume).map(vol -> vol.isEmpty() ? null : vol) + .map(vol -> metadataManager.getBucketKeyPrefix(vol, bucket)); + try (TableIterator> + delKeyIter = metadataManager.getDeletedTable().iterator(bucketPrefix.orElse(""))) { + + /* Seeking to the start key if it not null. The next key picked up would be ensured to start with the bucket + prefix, {@link org.apache.hadoop.hdds.utils.db.Table#iterator(bucketPrefix)} would ensure this. + */ + if (startKey != null) { + delKeyIter.seek(startKey); + } + int currentCount = 0; + while (delKeyIter.hasNext() && currentCount < count) { + RepeatedOmKeyInfo notReclaimableKeyInfo = new RepeatedOmKeyInfo(); + Table.KeyValue kv = delKeyIter.next(); + if (kv != null) { + List blockGroupList = Lists.newArrayList(); + // Multiple keys with the same path can be queued in one DB entry + RepeatedOmKeyInfo infoList = kv.getValue(); + for (OmKeyInfo info : infoList.cloneOmKeyInfoList()) { + + // Skip the key if the filter doesn't allow the file to be deleted. + if (filter.apply(Table.newKeyValue(kv.getKey(), info))) { + List blockIDS = info.getKeyLocationVersions().stream() + .flatMap(versionLocations -> versionLocations.getLocationList().stream() + .map(b -> new BlockID(b.getContainerID(), b.getLocalID()))).collect(Collectors.toList()); + BlockGroup keyBlocks = BlockGroup.newBuilder().setKeyName(kv.getKey()) + .addAllBlockIDs(blockIDS).build(); + blockGroupList.add(keyBlocks); + currentCount++; + } else { + notReclaimableKeyInfo.addOmKeyInfo(info); + } + } + + List notReclaimableKeyInfoList = notReclaimableKeyInfo.getOmKeyInfoList(); + + // If all the versions are not reclaimable, then modify key by just purging the key that can be purged. + if (notReclaimableKeyInfoList.size() > 0 && + notReclaimableKeyInfoList.size() != infoList.getOmKeyInfoList().size()) { + keysToModify.put(kv.getKey(), notReclaimableKeyInfo); + } + keyBlocksList.addAll(blockGroupList); + } + } + nextPageStartKey = delKeyIter.hasNext() ? delKeyIter.next().getKey() : null; + } + return new PendingKeysDeletion(keyBlocksList, keysToModify, nextPageStartKey); } @Override @@ -1999,27 +2065,46 @@ public Table.KeyValue getPendingDeletionDir() } @Override - public List getPendingDeletionSubDirs(long volumeId, long bucketId, - OmKeyInfo parentInfo, long numEntries) throws IOException { + public TableIterator> getPendingDeletionDirs() throws IOException { + return this.getPendingDeletionDirs(null, null); + } + + @Override + public TableIterator> getPendingDeletionDirs(String volume, + String bucket) + throws IOException { + + // Either both volume & bucket should be null or none of them should be null. + if (!StringUtils.isBlank(volume) && StringUtils.isBlank(bucket) || + StringUtils.isBlank(volume) && !StringUtils.isBlank(bucket)) { + throw new IOException("One of volume : " + volume + ", bucket: " + bucket + " is empty. Either both should be " + + "empty or none of the arguments should be empty"); + } + return StringUtils.isBlank(volume) ? metadataManager.getDeletedDirTable().iterator() : + metadataManager.getDeletedDirTable().iterator(metadataManager.getBucketKeyPrefixFSO(volume, bucket)); + } + + @Override + public List getPendingDeletionSubDirs( + long volumeId, long bucketId, OmKeyInfo parentInfo, + CheckExceptionOperation, Boolean, IOException> filter, + long numEntries) throws IOException { String seekDirInDB = metadataManager.getOzonePathKey(volumeId, bucketId, parentInfo.getObjectID(), ""); long countEntries = 0; Table dirTable = metadataManager.getDirectoryTable(); - try (TableIterator> + try (TableIterator> iterator = dirTable.iterator()) { return gatherSubDirsWithIterator(parentInfo, numEntries, - seekDirInDB, countEntries, iterator); + seekDirInDB, filter, countEntries, iterator); } - } private List gatherSubDirsWithIterator(OmKeyInfo parentInfo, long numEntries, String seekDirInDB, - long countEntries, - TableIterator> iterator) + CheckExceptionOperation, Boolean, IOException> filter, + long countEntries, TableIterator> iterator) throws IOException { List directories = new ArrayList<>(); iterator.seek(seekDirInDB); @@ -2035,29 +2120,33 @@ private List gatherSubDirsWithIterator(OmKeyInfo parentInfo, continue; } OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo(parentInfo, dirInfo); - directories.add(omKeyInfo); - countEntries++; + if (filter.apply(Table.newKeyValue(metadataManager.getOzoneDeletePathKey(omKeyInfo.getObjectID(), entry.getKey()), + omKeyInfo))) { + directories.add(omKeyInfo); + countEntries++;; + } + } return directories; } @Override - public List getPendingDeletionSubFiles(long volumeId, - long bucketId, OmKeyInfo parentInfo, long numEntries) - throws IOException { + public List getPendingDeletionSubFiles( + long volumeId, long bucketId, OmKeyInfo parentInfo, + CheckExceptionOperation, Boolean, IOException> filter, long numEntries) + throws IOException { List files = new ArrayList<>(); String seekFileInDB = metadataManager.getOzonePathKey(volumeId, bucketId, parentInfo.getObjectID(), ""); - long countEntries = 0; - Table fileTable = metadataManager.getFileTable(); + Table fileTable = metadataManager.getFileTable(); try (TableIterator> iterator = fileTable.iterator()) { iterator.seek(seekFileInDB); - while (iterator.hasNext() && numEntries - countEntries > 0) { + while (iterator.hasNext() && numEntries > 0) { Table.KeyValue entry = iterator.next(); OmKeyInfo fileInfo = entry.getValue(); if (!OMFileRequest.isImmediateChild(fileInfo.getParentObjectID(), @@ -2068,8 +2157,11 @@ public List getPendingDeletionSubFiles(long volumeId, continue; } OMFileRequest.setKeyNameAndFileName(parentInfo, fileInfo); - files.add(fileInfo); - countEntries++; + if (filter.apply(Table.newKeyValue(metadataManager.getOzoneDeletePathKey(fileInfo.getObjectID(), entry.getKey()) + , fileInfo))) { + files.add(fileInfo); + numEntries--; + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 13568dcbafdd..f35faec7f9c4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -34,7 +34,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; -import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -115,7 +114,6 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; -import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotDirExist; import org.apache.hadoop.util.Time; @@ -838,6 +836,28 @@ public String getBucketKey(String volume, String bucket) { return builder.toString(); } + /** + * Given a volume and bucket, return the corresponding DB key prefix. + * + * @param volume - User name + * @param bucket - Bucket name + */ + @Override + public String getBucketKeyPrefix(String volume, String bucket) { + return OzoneFSUtils.addTrailingSlashIfNeeded(getBucketKey(volume, bucket)); + } + + /** + * Given a volume and bucket, return the corresponding DB key prefix. + * + * @param volume - User name + * @param bucket - Bucket name + */ + @Override + public String getBucketKeyPrefixFSO(String volume, String bucket) throws IOException { + return OzoneFSUtils.addTrailingSlashIfNeeded(getOzoneKeyFSO(volume, bucket, "")); + } + @Override public String getOzoneKey(String volume, String bucket, String key) { StringBuilder builder = new StringBuilder() @@ -1589,184 +1609,6 @@ private PersistedUserVolumeInfo getVolumesByUser(String userNameKey) } } - /** - * Returns a list of pending deletion key info up to the limit. - * Each entry is a {@link BlockGroup}, which contains the info about the key - * name and all its associated block IDs. - * - * @param keyCount max number of keys to return. - * @param omSnapshotManager SnapshotManager - * @return a list of {@link BlockGroup} represent keys and blocks. - * @throws IOException - */ - public PendingKeysDeletion getPendingDeletionKeys(final int keyCount, - OmSnapshotManager omSnapshotManager) - throws IOException { - List keyBlocksList = Lists.newArrayList(); - HashMap keysToModify = new HashMap<>(); - try (TableIterator> - keyIter = getDeletedTable().iterator()) { - int currentCount = 0; - while (keyIter.hasNext() && currentCount < keyCount) { - RepeatedOmKeyInfo notReclaimableKeyInfo = new RepeatedOmKeyInfo(); - KeyValue kv = keyIter.next(); - if (kv != null) { - List blockGroupList = Lists.newArrayList(); - // Get volume name and bucket name - String[] keySplit = kv.getKey().split(OM_KEY_PREFIX); - String bucketKey = getBucketKey(keySplit[1], keySplit[2]); - OmBucketInfo bucketInfo = getBucketTable().get(bucketKey); - SnapshotInfo snapshotInfo = getLatestSnapshotInfo(keySplit[1], keySplit[2], omSnapshotManager); - if (snapshotInfo.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE || - !OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapshotInfo)) { - continue; - } - // Get the latest snapshot in snapshot path. - try (ReferenceCounted - rcLatestSnapshot = getLatestSnapshot( - keySplit[1], keySplit[2], omSnapshotManager, false)) { - - // Multiple keys with the same path can be queued in one DB entry - RepeatedOmKeyInfo infoList = kv.getValue(); - for (OmKeyInfo info : infoList.cloneOmKeyInfoList()) { - // Skip the key if it exists in the previous snapshot (of the same - // scope) as in this case its blocks should not be reclaimed - - // If the last snapshot is deleted and the keys renamed in between - // the snapshots will be cleaned up by KDS. So we need to check - // in the renamedTable as well. - String dbRenameKey = getRenameKey(info.getVolumeName(), - info.getBucketName(), info.getObjectID()); - - if (rcLatestSnapshot != null) { - Table prevKeyTable = - rcLatestSnapshot.get() - .getMetadataManager() - .getKeyTable(bucketInfo.getBucketLayout()); - - Table prevDeletedTable = - rcLatestSnapshot.get().getMetadataManager().getDeletedTable(); - String prevKeyTableDBKey = getSnapshotRenamedTable() - .get(dbRenameKey); - String prevDelTableDBKey = getOzoneKey(info.getVolumeName(), - info.getBucketName(), info.getKeyName()); - // format: /volName/bucketName/keyName/objId - prevDelTableDBKey = getOzoneDeletePathKey(info.getObjectID(), - prevDelTableDBKey); - - if (prevKeyTableDBKey == null && - bucketInfo.getBucketLayout().isFileSystemOptimized()) { - long volumeId = getVolumeId(info.getVolumeName()); - prevKeyTableDBKey = getOzonePathKey(volumeId, - bucketInfo.getObjectID(), - info.getParentObjectID(), - info.getFileName()); - } else if (prevKeyTableDBKey == null) { - prevKeyTableDBKey = getOzoneKey(info.getVolumeName(), - info.getBucketName(), - info.getKeyName()); - } - - OmKeyInfo omKeyInfo = prevKeyTable.get(prevKeyTableDBKey); - // When key is deleted it is no longer in keyTable, we also - // have to check deletedTable of previous snapshot - RepeatedOmKeyInfo delOmKeyInfo = - prevDeletedTable.get(prevDelTableDBKey); - if (versionExistsInPreviousSnapshot(omKeyInfo, - info, delOmKeyInfo)) { - // If the infoList size is 1, there is nothing to split. - // We either delete it or skip it. - if (!(infoList.getOmKeyInfoList().size() == 1)) { - notReclaimableKeyInfo.addOmKeyInfo(info); - } - continue; - } - } - - // Add all blocks from all versions of the key to the deletion - // list - for (OmKeyLocationInfoGroup keyLocations : - info.getKeyLocationVersions()) { - List item = keyLocations.getLocationList().stream() - .map(b -> new BlockID(b.getContainerID(), b.getLocalID())) - .collect(Collectors.toList()); - BlockGroup keyBlocks = BlockGroup.newBuilder() - .setKeyName(kv.getKey()) - .addAllBlockIDs(item) - .build(); - blockGroupList.add(keyBlocks); - } - currentCount++; - } - - List notReclaimableKeyInfoList = - notReclaimableKeyInfo.getOmKeyInfoList(); - - // If all the versions are not reclaimable, then do nothing. - if (notReclaimableKeyInfoList.size() > 0 && - notReclaimableKeyInfoList.size() != - infoList.getOmKeyInfoList().size()) { - keysToModify.put(kv.getKey(), notReclaimableKeyInfo); - } - - if (notReclaimableKeyInfoList.size() != - infoList.getOmKeyInfoList().size()) { - keyBlocksList.addAll(blockGroupList); - } - } - } - } - } - return new PendingKeysDeletion(keyBlocksList, keysToModify); - } - - private boolean versionExistsInPreviousSnapshot(OmKeyInfo omKeyInfo, - OmKeyInfo info, RepeatedOmKeyInfo delOmKeyInfo) { - return (omKeyInfo != null && - info.getObjectID() == omKeyInfo.getObjectID() && - isBlockLocationInfoSame(omKeyInfo, info)) || - delOmKeyInfo != null; - } - - /** - * Get the latest OmSnapshot for a snapshot path. - */ - public ReferenceCounted getLatestActiveSnapshot( - String volumeName, String bucketName, - OmSnapshotManager snapshotManager) - throws IOException { - return getLatestSnapshot(volumeName, bucketName, snapshotManager, true); - } - - /** - * Get the latest OmSnapshot for a snapshot path. - */ - public ReferenceCounted getLatestSnapshot(String volumeName, String bucketName, - OmSnapshotManager snapshotManager, boolean isActive) - throws IOException { - Optional snapshotInfo = Optional.ofNullable(getLatestSnapshotInfo(volumeName, bucketName, - snapshotManager)); - if (isActive && snapshotInfo.map(si -> si.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) - .orElse(false)) { - snapshotInfo = Optional.ofNullable(SnapshotUtils.getPreviousActiveSnapshot(snapshotInfo.get(), - snapshotChainManager, snapshotManager)); - } - Optional> rcOmSnapshot = snapshotInfo.isPresent() ? - Optional.ofNullable(snapshotManager.getSnapshot(volumeName, bucketName, snapshotInfo.get().getName())) : - Optional.empty(); - - return rcOmSnapshot.orElse(null); - } - - public SnapshotInfo getLatestSnapshotInfo(String volumeName, String bucketName, - OmSnapshotManager snapshotManager) throws IOException { - String snapshotPath = volumeName + OM_KEY_PREFIX + bucketName; - Optional latestPathSnapshot = Optional.ofNullable( - snapshotChainManager.getLatestPathSnapshotId(snapshotPath)); - return latestPathSnapshot.isPresent() ? - SnapshotUtils.getSnapshotInfo(snapshotChainManager, latestPathSnapshot.get(), snapshotManager) : null; - } - /** * Decide whether the open key is a multipart upload related key. * @param openKeyInfo open key related to multipart upload @@ -2160,6 +2002,13 @@ public String getRenameKey(String volumeName, String bucketName, renameKey.append(OM_KEY_PREFIX).append(objectID); return renameKey.toString(); } + + @Override + public String[] splitRenameKey(String renameKey) { + String[] splitVals = renameKey.split(OM_KEY_PREFIX); + return new String[]{splitVals[1], splitVals[2], splitVals[3]}; + } + @Override public String getMultipartKey(long volumeId, long bucketId, long parentID, String fileName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PendingKeysDeletion.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PendingKeysDeletion.java index f2d73aaf411b..bba44aee7193 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PendingKeysDeletion.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PendingKeysDeletion.java @@ -23,26 +23,35 @@ import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Return class for OMMetadataManager#getPendingDeletionKeys. */ public class PendingKeysDeletion { - private HashMap keysToModify; + private Map keysToModify; private List keyBlocksList; + private String nextPageStartKey; + public PendingKeysDeletion(List keyBlocksList, - HashMap keysToModify) { + Map keysToModify, + String nextPageStartKey) { this.keysToModify = keysToModify; this.keyBlocksList = keyBlocksList; + this.nextPageStartKey = nextPageStartKey; } - public HashMap getKeysToModify() { + public Map getKeysToModify() { return keysToModify; } public List getKeyBlocksList() { return keyBlocksList; } + + public String getNextPageStartKey() { + return nextPageStartKey; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 61b3bdce04fc..8ff59e091d88 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -77,7 +77,6 @@ import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotDeleteRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveDeletedKeysRequest; -import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotPurgeAndMoveRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotPurgeRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotRenameRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotSetPropertyRequest; @@ -231,8 +230,6 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMSnapshotMoveDeletedKeysRequest(omRequest); case SnapshotPurge: return new OMSnapshotPurgeRequest(omRequest); - case SnapshotPurgeAndMove: - return new OMSnapshotPurgeAndMoveRequest(omRequest); case SetSnapshotProperty: return new OMSnapshotSetPropertyRequest(omRequest); case DeleteOpenKeys: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index 234d8b5bfc3f..cba44386c56a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -23,13 +23,21 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.UUID; + +import com.google.common.collect.Maps; +import org.apache.commons.compress.utils.Lists; import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -45,8 +53,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.validatePreviousSnapshotId; /** * Handles purging of keys from OM DB. @@ -66,22 +76,38 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn List purgeRequests = purgeDirsRequest.getDeletedPathList(); - - SnapshotInfo fromSnapshotInfo = null; + List validPurgePathRequests = Lists.newArrayList(); + final SnapshotInfo fromSnapshotInfo; Set> lockSet = new HashSet<>(); Map, OmBucketInfo> volBucketInfoMap = new HashMap<>(); - OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); Map openKeyInfoMap = new HashMap<>(); OMMetrics omMetrics = ozoneManager.getMetrics(); try { - if (fromSnapshot != null) { - fromSnapshotInfo = ozoneManager.getMetadataManager() - .getSnapshotInfoTable() - .get(fromSnapshot); - } + fromSnapshotInfo = fromSnapshot != null ? SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot) : null; + + Map, Boolean> previousSnapshotValidatedMap = Maps.newHashMap(); + for (OzoneManagerProtocolProtos.PurgePathRequest path : purgeRequests) { + // Validating previous snapshot since while purging deletes, a snapshot create request could make this purge + // directory request invalid on AOS since the deletedDirectory would be in the newly created snapshot. Adding + // subdirectories could lead to not being able to reclaim sub-files and subdirectories since the + // file/directory would be present in the newly created snapshot. + // Validating previous snapshot can ensure the chain hasn't changed. + boolean isPreviousSnapshotValid = previousSnapshotValidatedMap.computeIfAbsent(Pair.of(path.getVolumeName(), + path.getBucketName()), + (volumeBucketPair) -> validatePreviousSnapshotId(fromSnapshotInfo, volumeBucketPair.getLeft(), + volumeBucketPair.getRight(), omMetadataManager.getSnapshotChainManager(), + path.hasExpectedPathPreviousSnapshotUUID() + ? fromProtobuf(path.getExpectedPathPreviousSnapshotUUID()) : null)); + + if (!isPreviousSnapshotValid) { + continue; + } + + validPurgePathRequests.add(path); for (OzoneManagerProtocolProtos.KeyInfo key : path.getMarkDeletedSubDirsList()) { OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key); @@ -174,7 +200,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO( - omResponse.build(), purgeRequests, ozoneManager.isRatisEnabled(), + omResponse.build(), validPurgePathRequests, ozoneManager.isRatisEnabled(), getBucketLayout(), volBucketInfoMap, fromSnapshotInfo, openKeyInfoMap); return omClientResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java index 9ed921839683..cd7862e5362f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java @@ -21,6 +21,13 @@ import java.io.IOException; import java.util.ArrayList; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -37,6 +44,10 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.validatePreviousSnapshotId; /** * Handles purging of keys from OM DB. @@ -53,38 +64,63 @@ public OMKeyPurgeRequest(OMRequest omRequest) { @Override public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIndex termIndex) { PurgeKeysRequest purgeKeysRequest = getOmRequest().getPurgeKeysRequest(); - List bucketDeletedKeysList = purgeKeysRequest - .getDeletedKeysList(); - List keysToUpdateList = purgeKeysRequest - .getKeysToUpdateList(); + List bucketDeletedKeysList = purgeKeysRequest.getDeletedKeysList(); + List keysToUpdateList = purgeKeysRequest.getKeysToUpdateList(); String fromSnapshot = purgeKeysRequest.hasSnapshotTableKey() ? purgeKeysRequest.getSnapshotTableKey() : null; - List keysToBePurgedList = new ArrayList<>(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); - OMClientResponse omClientResponse = null; + final SnapshotInfo fromSnapshotInfo; + + try { + fromSnapshotInfo = fromSnapshot != null ? null : SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot); + } catch (IOException ex) { + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, ex)); + } + List keysToBePurgedList = new ArrayList<>(); + List renamedKeysToBePurged = new ArrayList<>(); + + Map, Boolean> previousSnapshotValidatedMap = Maps.newHashMap(); for (DeletedKeys bucketWithDeleteKeys : bucketDeletedKeysList) { - for (String deletedKey : bucketWithDeleteKeys.getKeysList()) { - keysToBePurgedList.add(deletedKey); + // Validating previous snapshot since while purging rename keys, a snapshot create request could make this purge + // rename entry invalid on AOS since the key could be now present on the newly created snapshot since a rename + // request could have come through after creation of a snapshot. The purged deletedKey would not be even + // present + boolean isPreviousSnapshotValid = previousSnapshotValidatedMap.computeIfAbsent( + Pair.of(bucketWithDeleteKeys.getVolumeName(), bucketWithDeleteKeys.getBucketName()), + (volumeBucketPair) -> validatePreviousSnapshotId(fromSnapshotInfo, volumeBucketPair.getLeft(), + volumeBucketPair.getRight(), omMetadataManager.getSnapshotChainManager(), + bucketWithDeleteKeys.hasExpectedPathPreviousSnapshotUUID() + ? fromProtobuf(bucketWithDeleteKeys.getExpectedPathPreviousSnapshotUUID()) : null)); + if (isPreviousSnapshotValid) { + renamedKeysToBePurged.addAll(bucketWithDeleteKeys.getRenamedKeysList()); + keysToBePurgedList.addAll(bucketWithDeleteKeys.getKeysList()); } } + if (keysToBePurgedList.isEmpty() && renamedKeysToBePurged.isEmpty()) { + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, + new OMException("None of the keys can be purged be purged since a new snapshot was created for all the " + + "buckets, making this request invalid", OMException.ResultCodes.KEY_DELETION_ERROR))); + } + + // Setting transaction info for snapshot, this is to prevent duplicate purge requests to OM from background + // services. try { - SnapshotInfo fromSnapshotInfo = null; - if (fromSnapshot != null) { - fromSnapshotInfo = ozoneManager.getMetadataManager() - .getSnapshotInfoTable().get(fromSnapshot); + if (fromSnapshotInfo != null) { + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshotInfo, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshotInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshotInfo)); } - omClientResponse = new OMKeyPurgeResponse(omResponse.build(), - keysToBePurgedList, fromSnapshotInfo, keysToUpdateList); - } catch (IOException ex) { - omClientResponse = new OMKeyPurgeResponse( - createErrorOMResponse(omResponse, ex)); + } catch (IOException e) { + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, e)); } - return omClientResponse; + return new OMKeyPurgeResponse(omResponse.build(), + keysToBePurgedList, renamedKeysToBePurged, fromSnapshotInfo, keysToUpdateList); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index c55539070a5e..002e25b7637d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.RDBStore; @@ -58,11 +59,13 @@ import java.io.IOException; import java.nio.file.InvalidPathException; +import java.util.Optional; import java.util.UUID; import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; import static org.apache.hadoop.hdds.HddsUtils.toProtobuf; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_ALREADY_EXISTS; +import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.SNAPSHOT_LOCK; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.FILESYSTEM_SNAPSHOT; @@ -183,8 +186,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn // TODO: [SNAPSHOT] Assign actual data size once we have the // pre-replicated key size counter in OmBucketInfo. snapshotInfo.setReferencedSize(estimateBucketDataSize(omBucketInfo)); - - addSnapshotInfoToSnapshotChainAndCache(omMetadataManager, termIndex.getIndex()); + SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); + addSnapshotInfoToSnapshotChainAndCache(ozoneManager.getOmSnapshotManager(), omMetadataManager, + termIndex.getIndex()); omResponse.setCreateSnapshotResponse( CreateSnapshotResponse.newBuilder() @@ -246,6 +250,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * it was removed at T-5. */ private void addSnapshotInfoToSnapshotChainAndCache( + OmSnapshotManager omSnapshotManager, OmMetadataManagerImpl omMetadataManager, long transactionLogIndex ) throws IOException { @@ -264,6 +269,21 @@ private void addSnapshotInfoToSnapshotChainAndCache( snapshotInfo.setPathPreviousSnapshotId(latestPathSnapshot); snapshotInfo.setGlobalPreviousSnapshotId(latestGlobalSnapshot); + Optional previousSnapshot = Optional.ofNullable(SnapshotUtils.getLatestSnapshotInfo( + snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), + snapshotChainManager, omSnapshotManager)); + Optional previousPrevSnapshot = previousSnapshot.isPresent() ? + Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(previousSnapshot.get(), snapshotChainManager, + omSnapshotManager)) : Optional.empty(); + + // Reset the deep clean flag for the next active snapshot if and only if the last 2 snapshots in the + // chain are active, otherwise set it to prevent deep cleaning from running till the deleted snapshots don't + // get purged. + boolean deepCleanFlag = + !(previousSnapshot.map(SnapshotInfo::getSnapshotStatus).orElse(SNAPSHOT_ACTIVE) == SNAPSHOT_ACTIVE && + previousPrevSnapshot.map(SnapshotInfo::getSnapshotStatus).orElse(SNAPSHOT_ACTIVE) == SNAPSHOT_ACTIVE); + snapshotInfo.setDeepCleanedDeletedDir(deepCleanFlag); + snapshotInfo.setDeepClean(deepCleanFlag); try { snapshotChainManager.addSnapshot(snapshotInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index 122108ad65fc..4479e07d144a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -19,7 +19,11 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshotManager; @@ -42,6 +46,7 @@ import java.io.IOException; import java.util.List; +import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.FILESYSTEM_SNAPSHOT; /** @@ -61,43 +66,54 @@ public OMSnapshotMoveDeletedKeysRequest(OMRequest omRequest) { @DisallowedUntilLayoutVersion(FILESYSTEM_SNAPSHOT) public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIndex termIndex) { OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); - OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) - ozoneManager.getMetadataManager(); - SnapshotChainManager snapshotChainManager = - omMetadataManager.getSnapshotChainManager(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); + SnapshotChainManager snapshotChainManager = omMetadataManager.getSnapshotChainManager(); - SnapshotMoveDeletedKeysRequest moveDeletedKeysRequest = - getOmRequest().getSnapshotMoveDeletedKeysRequest(); - SnapshotInfo fromSnapshot = SnapshotInfo.getFromProtobuf( - moveDeletedKeysRequest.getFromSnapshot()); + SnapshotMoveDeletedKeysRequest moveDeletedKeysRequest = getOmRequest().getSnapshotMoveDeletedKeysRequest(); + SnapshotInfo fromSnapshot = null; // If there is no Non-Deleted Snapshot move the // keys to Active Object Store. SnapshotInfo nextSnapshot = null; OMClientResponse omClientResponse = null; - OzoneManagerProtocolProtos.OMResponse.Builder omResponse = - OmResponseUtil.getOMResponseBuilder(getOmRequest()); + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); try { - nextSnapshot = SnapshotUtils.getNextActiveSnapshot(fromSnapshot, - snapshotChainManager, omSnapshotManager); + fromSnapshot = moveDeletedKeysRequest.hasFromSnapshotID() ? SnapshotUtils.getSnapshotInfo(snapshotChainManager, + fromProtobuf(moveDeletedKeysRequest.getFromSnapshotID()), omSnapshotManager) : + SnapshotInfo.getFromProtobuf(moveDeletedKeysRequest.getFromSnapshot()); + nextSnapshot = SnapshotUtils.getNextSnapshot(fromSnapshot, snapshotChainManager, omSnapshotManager); + + // Only skip the move operation for newer requests. Newer requests would always have the purgeMovedKeys in the + // request. Skip move operation if the next snapshot is not active. + if (moveDeletedKeysRequest.hasPurgeMovedKeys() && nextSnapshot != null && + nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { + throw new OMException("Next snapshot : " + nextSnapshot + " in chain is not active.", + OMException.ResultCodes.INVALID_REQUEST); + } // Get next non-deleted snapshot. - List nextDBKeysList = - moveDeletedKeysRequest.getNextDBKeysList(); - List reclaimKeysList = - moveDeletedKeysRequest.getReclaimKeysList(); - List renamedKeysList = - moveDeletedKeysRequest.getRenamedKeysList(); - List movedDirs = - moveDeletedKeysRequest.getDeletedDirsToMoveList(); - - omClientResponse = new OMSnapshotMoveDeletedKeysResponse( - omResponse.build(), fromSnapshot, nextSnapshot, - nextDBKeysList, reclaimKeysList, renamedKeysList, movedDirs); + List nextDBKeysList = moveDeletedKeysRequest.getNextDBKeysList(); + List reclaimKeysList = moveDeletedKeysRequest.getReclaimKeysList(); + List renamedKeysList = moveDeletedKeysRequest.getRenamedKeysList(); + List movedDirs = moveDeletedKeysRequest.getDeletedDirsToMoveList(); + + // Update lastTransactionInfo for fromSnapshot and the nextSnapshot. + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshot, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshot)); + if (nextSnapshot != null) { + SnapshotUtils.setTransactionInfoInSnapshot(nextSnapshot, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), nextSnapshot)); + } + + + omClientResponse = new OMSnapshotMoveDeletedKeysResponse(omResponse.build(), fromSnapshot, nextSnapshot, + nextDBKeysList, reclaimKeysList, renamedKeysList, movedDirs, + moveDeletedKeysRequest.hasPurgeMovedKeys() && moveDeletedKeysRequest.getPurgeMovedKeys()); } catch (IOException ex) { - omClientResponse = new OMSnapshotMoveDeletedKeysResponse( - createErrorOMResponse(omResponse, ex)); + omClientResponse = new OMSnapshotMoveDeletedKeysResponse(createErrorOMResponse(omResponse, ex)); } return omClientResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeAndMoveRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeAndMoveRequest.java deleted file mode 100644 index 66d573a3df23..000000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeAndMoveRequest.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.ozone.om.request.snapshot; - -import com.google.common.base.Objects; -import org.apache.hadoop.hdds.HddsUtils; -import org.apache.hadoop.hdds.utils.db.cache.CacheKey; -import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OMMetrics; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshotManager; -import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.SnapshotChainManager; -import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.hadoop.ozone.om.request.OMClientRequest; -import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotPurgeAndMoveResponse; -import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotPurgeResponse; -import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; -import org.apache.ratis.server.protocol.TermIndex; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.UUID; - -/** - * Handles OMSnapshotPurge Request. - * This is an OM internal request. Does not need @RequireSnapshotFeatureState. - */ -public class OMSnapshotPurgeAndMoveRequest extends OMClientRequest { - - private static final Logger LOG = LoggerFactory.getLogger(OMSnapshotPurgeAndMoveRequest.class); - - /** - * This map contains up to date snapshotInfo and works as a local cache for OMSnapshotPurgeRequest. - * Since purge and other updates happen in sequence inside validateAndUpdateCache, we can get updated snapshotInfo - * from this map rather than getting form snapshotInfoTable which creates a deep copy for every get call. - */ - private final Map updatedSnapshotInfos = new HashMap<>(); - - public OMSnapshotPurgeAndMoveRequest(OMRequest omRequest) { - super(omRequest); - } - - @Override - public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIndex termIndex) { - OMMetrics omMetrics = ozoneManager.getMetrics(); - - final long trxnLogIndex = termIndex.getIndex(); - - OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); - OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) - ozoneManager.getMetadataManager(); - SnapshotChainManager snapshotChainManager = - omMetadataManager.getSnapshotChainManager(); - - OMClientResponse omClientResponse = null; - - OzoneManagerProtocolProtos.OMResponse.Builder omResponse = - OmResponseUtil.getOMResponseBuilder(getOmRequest()); - OzoneManagerProtocolProtos.SnapshotPurgeAndMoveRequest snapshotPurgeAndMoveRequest = getOmRequest() - .getSnapshotPurgeAndMoveRequest(); - - try { - UUID snapshotId = HddsUtils.fromProtobuf(snapshotPurgeAndMoveRequest.getSnapshotID()); - // When value is null it basically means the next expected target would be AOS. - UUID nextExpectedSnapshotId = snapshotPurgeAndMoveRequest.hasExpectedNextSnapshotID() ? - HddsUtils.fromProtobuf(snapshotPurgeAndMoveRequest.getSnapshotID()) : null; - SnapshotInfo currentSnapshot = SnapshotUtils.getSnapshotInfo(snapshotChainManager, snapshotId, omSnapshotManager); - SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(currentSnapshot, snapshotChainManager, - omSnapshotManager); - SnapshotInfo prevSnapshot = SnapshotUtils.getPreviousSnapshot(currentSnapshot, snapshotChainManager, - omSnapshotManager); - // When value is null it basically means the next target is AOS. - UUID nextSnapshotId = Optional.ofNullable(nextSnapshot).map(SnapshotInfo::getSnapshotId).orElse(null); - if (!Objects.equal(nextSnapshotId, nextExpectedSnapshotId)) { - throw new OMException("Next path snapshot for " + currentSnapshot + " expected snapshot Id: " - + nextExpectedSnapshotId + " but was " + nextSnapshotId, OMException.ResultCodes.INVALID_REQUEST); - } - - // When next snapshot is not active. The keys can be moved to the next active snapshot to avoid unnecessary hop. - if (nextSnapshot != null && nextSnapshot.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { - throw new OMException("Next path snapshot for " + currentSnapshot + " " + nextSnapshotId + " is not active.", - OMException.ResultCodes.INVALID_REQUEST); - } - - // Each snapshot purge operation does three things: - // 1. Update the deep clean flag for the next snapshot if the previous snapshot is active otherwise don't - // bother to run deepClean unnecessarily - // (So that it can be deep cleaned by the KeyDeletingService in the next run), - // 2. Update the snapshot chain, - // 3. Finally, purge the snapshot. - // There is no need to take lock for snapshot purge as of now. We can simply rely on OMStateMachine - // because it executes transaction sequentially. - // As part of the flush the keys in the deleted table, deletedDirectoryTable and renameTable would be moved to - // the next snapshot. - // Step 1: Update the deep clean flag for the next active snapshot - if (prevSnapshot == null || prevSnapshot.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { - updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, termIndex); - } - // Step 2: Update the snapshot chain. - updateSnapshotChainAndCache(omMetadataManager, currentSnapshot, trxnLogIndex); - // Remove and close snapshot's RocksDB instance from SnapshotCache. - omSnapshotManager.invalidateCacheEntry(snapshotId); - // Step 3: Purge the snapshot from SnapshotInfoTable cache. - omMetadataManager.getSnapshotInfoTable() - .addCacheEntry(new CacheKey<>(currentSnapshot.getTableKey()), CacheValue.get(trxnLogIndex)); - - omClientResponse = new OMSnapshotPurgeAndMoveResponse(omResponse.build(), currentSnapshot, nextSnapshot, - updatedSnapshotInfos); - } catch (IOException ex) { - omClientResponse = new OMSnapshotPurgeResponse( - createErrorOMResponse(omResponse, ex)); - omMetrics.incNumSnapshotPurgeFails(); - LOG.error("Failed to execute snapshotPurgeAndMoveRequest:{{}}.", snapshotPurgeAndMoveRequest, ex); - } - - return omClientResponse; - } - - private void updateSnapshotInfoAndCache(SnapshotInfo snapInfo, OmMetadataManagerImpl omMetadataManager, - TermIndex termIndex) throws IOException { - if (snapInfo != null) { - // Setting next snapshot deep clean to false, Since the - // current snapshot is deleted. We can potentially - // reclaim more keys in the next snapshot. - snapInfo.setDeepClean(false); - snapInfo.setDeepCleanedDeletedDir(false); - SnapshotUtils.setTransactionInfoInSnapshot(snapInfo, termIndex); - - // Update table cache first - omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapInfo.getTableKey()), - CacheValue.get(termIndex.getIndex(), snapInfo)); - } - } - - /** - * Removes the snapshot from the chain and updates the next snapshot's - * previousPath and previousGlobal IDs in DB cache. - * It also returns the pair of updated next path and global snapshots to - * update in DB. - */ - private void updateSnapshotChainAndCache( - OmMetadataManagerImpl metadataManager, - SnapshotInfo snapInfo, - long trxnLogIndex - ) throws IOException { - if (snapInfo == null) { - return; - } - - SnapshotChainManager snapshotChainManager = metadataManager - .getSnapshotChainManager(); - - // If the snapshot is deleted in the previous run, then the in-memory - // SnapshotChainManager might throw NoSuchElementException as the snapshot - // is removed in-memory but OMDoubleBuffer has not flushed yet. - boolean hasNextPathSnapshot; - boolean hasNextGlobalSnapshot; - try { - hasNextPathSnapshot = snapshotChainManager.hasNextPathSnapshot( - snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); - hasNextGlobalSnapshot = snapshotChainManager.hasNextGlobalSnapshot( - snapInfo.getSnapshotId()); - } catch (NoSuchElementException ex) { - return; - } - - String nextPathSnapshotKey = null; - - if (hasNextPathSnapshot) { - UUID nextPathSnapshotId = snapshotChainManager.nextPathSnapshot( - snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); - nextPathSnapshotKey = snapshotChainManager - .getTableKey(nextPathSnapshotId); - } - - String nextGlobalSnapshotKey = null; - if (hasNextGlobalSnapshot) { - UUID nextGlobalSnapshotId = snapshotChainManager.nextGlobalSnapshot(snapInfo.getSnapshotId()); - nextGlobalSnapshotKey = snapshotChainManager.getTableKey(nextGlobalSnapshotId); - } - - SnapshotInfo nextPathSnapInfo = - nextPathSnapshotKey != null ? getUpdatedSnapshotInfo(nextPathSnapshotKey, metadataManager) : null; - - SnapshotInfo nextGlobalSnapInfo = - nextGlobalSnapshotKey != null ? getUpdatedSnapshotInfo(nextGlobalSnapshotKey, metadataManager) : null; - - // Updates next path snapshot's previous snapshot ID - if (nextPathSnapInfo != null) { - nextPathSnapInfo.setPathPreviousSnapshotId(snapInfo.getPathPreviousSnapshotId()); - metadataManager.getSnapshotInfoTable().addCacheEntry( - new CacheKey<>(nextPathSnapInfo.getTableKey()), - CacheValue.get(trxnLogIndex, nextPathSnapInfo)); - } - - if (nextGlobalSnapInfo != null) { - nextGlobalSnapInfo.setGlobalPreviousSnapshotId( - snapInfo.getGlobalPreviousSnapshotId()); - metadataManager.getSnapshotInfoTable().addCacheEntry( - new CacheKey<>(nextGlobalSnapInfo.getTableKey()), - CacheValue.get(trxnLogIndex, nextGlobalSnapInfo)); - } - - snapshotChainManager.deleteSnapshot(snapInfo); - } - - private SnapshotInfo getUpdatedSnapshotInfo(String snapshotTableKey, OMMetadataManager omMetadataManager) - throws IOException { - SnapshotInfo snapshotInfo = updatedSnapshotInfos.get(snapshotTableKey); - - if (snapshotInfo == null) { - snapshotInfo = omMetadataManager.getSnapshotInfoTable().get(snapshotTableKey); - updatedSnapshotInfos.put(snapshotTableKey, snapshotInfo); - } - return snapshotInfo; - } -} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index cf5ba7b1cea9..0beddff6b2ff 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -44,8 +44,11 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Optional; import java.util.UUID; +import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; + /** * Handles OMSnapshotPurge Request. * This is an OM internal request. Does not need @RequireSnapshotFeatureState. @@ -101,10 +104,20 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } SnapshotInfo nextSnapshot = - SnapshotUtils.getNextActiveSnapshot(fromSnapshot, snapshotChainManager, omSnapshotManager); + SnapshotUtils.getNextSnapshot(fromSnapshot, snapshotChainManager, omSnapshotManager); + Optional previousSnapshot = Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(fromSnapshot, + snapshotChainManager, omSnapshotManager)); + Optional previousPrevSnapshot = previousSnapshot.isPresent() ? + Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(previousSnapshot.get(), snapshotChainManager, + omSnapshotManager)) : Optional.empty(); + // Step 1: Reset the deep clean flag for the next active snapshot if and only if the last 2 snapshots in the + // chain are active, otherwise set it to prevent deep cleaning from running till the deleted snapshots don't + // get purged. + updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, trxnLogIndex, updatedSnapInfos, + !(previousSnapshot.map(SnapshotInfo::getSnapshotStatus).orElse(SNAPSHOT_ACTIVE) == SNAPSHOT_ACTIVE && + previousPrevSnapshot.map(SnapshotInfo::getSnapshotStatus).orElse(SNAPSHOT_ACTIVE) == SNAPSHOT_ACTIVE)); + - // Step 1: Update the deep clean flag for the next active snapshot - updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, trxnLogIndex, updatedSnapInfos); // Step 2: Update the snapshot chain. updateSnapshotChainAndCache(omMetadataManager, fromSnapshot, trxnLogIndex, updatedPathPreviousAndGlobalSnapshots); @@ -133,15 +146,15 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn return omClientResponse; } - private void updateSnapshotInfoAndCache(SnapshotInfo snapInfo, - OmMetadataManagerImpl omMetadataManager, long trxnLogIndex, - Map updatedSnapInfos) throws IOException { + private void updateSnapshotInfoAndCache(SnapshotInfo snapInfo, OmMetadataManagerImpl omMetadataManager, + long trxnLogIndex, Map updatedSnapInfos, + boolean flagValue) throws IOException { if (snapInfo != null) { // Setting next snapshot deep clean to false, Since the // current snapshot is deleted. We can potentially // reclaim more keys in the next snapshot. - snapInfo.setDeepClean(false); - snapInfo.setDeepCleanedDeletedDir(false); + snapInfo.setDeepClean(flagValue); + snapInfo.setDeepCleanedDeletedDir(flagValue); // Update table cache first omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapInfo.getTableKey()), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java index b16ba95d78f6..f9f11eaf75b0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java @@ -47,15 +47,18 @@ @CleanupTableInfo(cleanupTables = {DELETED_TABLE}) public class OMKeyPurgeResponse extends OmKeyResponse { private List purgeKeyList; + private List renamedList; private SnapshotInfo fromSnapshot; private List keysToUpdateList; public OMKeyPurgeResponse(@Nonnull OMResponse omResponse, @Nonnull List keyList, + @Nonnull List renamedList, SnapshotInfo fromSnapshot, List keysToUpdate) { super(omResponse); this.purgeKeyList = keyList; + this.renamedList = renamedList; this.fromSnapshot = fromSnapshot; this.keysToUpdateList = keysToUpdate; } @@ -122,6 +125,10 @@ private void processKeys(BatchOperation batchOp, metadataManager.getDeletedTable().deleteWithBatch(batchOp, key); } + // Delete rename entries. + for (String key : renamedList) { + metadataManager.getSnapshotInfoTable().deleteWithBatch(batchOp, key); + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index 3726faacfd70..a01eda17537f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -54,13 +54,13 @@ public class OMSnapshotMoveDeletedKeysResponse extends OMClientResponse { private List renamedKeysList; private List movedDirs; + private boolean purgeMovedKeys; + public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, - @Nonnull SnapshotInfo fromSnapshot, - SnapshotInfo nextSnapshot, - List nextDBKeysList, - List reclaimKeysList, - List renamedKeysList, - List movedDirs) { + @Nonnull SnapshotInfo fromSnapshot, SnapshotInfo nextSnapshot, + List nextDBKeysList, List reclaimKeysList, + List renamedKeysList, List movedDirs, + boolean purgeMovedKeys) { super(omResponse); this.fromSnapshot = fromSnapshot; this.nextSnapshot = nextSnapshot; @@ -68,6 +68,7 @@ public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, this.reclaimKeysList = reclaimKeysList; this.renamedKeysList = renamedKeysList; this.movedDirs = movedDirs; + this.purgeMovedKeys = purgeMovedKeys; } /** @@ -80,40 +81,29 @@ public OMSnapshotMoveDeletedKeysResponse(@Nonnull OMResponse omResponse) { } @Override - protected void addToDBBatch(OMMetadataManager omMetadataManager, - BatchOperation batchOperation) throws IOException { + protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { // Note: a trick to get // To do this properly, refactor OzoneManagerDoubleBuffer#addToBatch and // add OmSnapshotManager as a parameter. - OmSnapshotManager omSnapshotManager = - ((OmMetadataManagerImpl) omMetadataManager) - .getOzoneManager().getOmSnapshotManager(); + OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) omMetadataManager) + .getOzoneManager().getOmSnapshotManager(); - try (ReferenceCounted rcOmFromSnapshot = - omSnapshotManager.getSnapshot( - fromSnapshot.getVolumeName(), - fromSnapshot.getBucketName(), - fromSnapshot.getName())) { + try (ReferenceCounted rcOmFromSnapshot = omSnapshotManager.getSnapshot( + fromSnapshot.getVolumeName(), fromSnapshot.getBucketName(), fromSnapshot.getName())) { OmSnapshot fromOmSnapshot = rcOmFromSnapshot.get(); if (nextSnapshot != null) { - try (ReferenceCounted - rcOmNextSnapshot = omSnapshotManager.getSnapshot( - nextSnapshot.getVolumeName(), - nextSnapshot.getBucketName(), - nextSnapshot.getName())) { + try (ReferenceCounted rcOmNextSnapshot = omSnapshotManager.getSnapshot( + nextSnapshot.getVolumeName(), nextSnapshot.getBucketName(), nextSnapshot.getName())) { OmSnapshot nextOmSnapshot = rcOmNextSnapshot.get(); - RDBStore nextSnapshotStore = - (RDBStore) nextOmSnapshot.getMetadataManager().getStore(); + RDBStore nextSnapshotStore = (RDBStore) nextOmSnapshot.getMetadataManager().getStore(); // Init Batch Operation for snapshot db. - try (BatchOperation writeBatch = - nextSnapshotStore.initBatchOperation()) { + try (BatchOperation writeBatch = nextSnapshotStore.initBatchOperation()) { processKeys(writeBatch, nextOmSnapshot.getMetadataManager()); - processDirs(writeBatch, nextOmSnapshot.getMetadataManager(), - fromOmSnapshot); + processDirs(writeBatch, nextOmSnapshot.getMetadataManager(), fromOmSnapshot); nextSnapshotStore.commitBatchOperation(writeBatch); nextSnapshotStore.getDb().flushWal(true); nextSnapshotStore.getDb().flush(); @@ -126,67 +116,76 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, } // Update From Snapshot Deleted Table. - RDBStore fromSnapshotStore = - (RDBStore) fromOmSnapshot.getMetadataManager().getStore(); - try (BatchOperation fromSnapshotBatchOp = - fromSnapshotStore.initBatchOperation()) { - processReclaimKeys(fromSnapshotBatchOp, - fromOmSnapshot.getMetadataManager()); - deleteDirsFromSnapshot(fromSnapshotBatchOp, fromOmSnapshot); + RDBStore fromSnapshotStore = (RDBStore) fromOmSnapshot.getMetadataManager().getStore(); + try (BatchOperation fromSnapshotBatchOp = fromSnapshotStore.initBatchOperation()) { + deleteDirsFromSnapshot(fromSnapshotBatchOp, fromOmSnapshot.getMetadataManager()); + // if purgeMovedKeys is true, reclaimKeys would be always empty. + if (purgeMovedKeys) { + deleteKeysFromSnapshot(fromSnapshotBatchOp, fromOmSnapshot.getMetadataManager()); + } else { + processReclaimKeys(fromSnapshotBatchOp, fromOmSnapshot.getMetadataManager()); + } fromSnapshotStore.commitBatchOperation(fromSnapshotBatchOp); fromSnapshotStore.getDb().flushWal(true); fromSnapshotStore.getDb().flush(); } } + // Flush snapshot info to rocksDB. + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); + if (nextSnapshot != null) { + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, nextSnapshot.getTableKey(), nextSnapshot); + } } - private void deleteDirsFromSnapshot(BatchOperation batchOp, - OmSnapshot fromOmSnapshot) + private void deleteDirsFromSnapshot(BatchOperation batchOp, OMMetadataManager fromSnapshotMetadataManager) throws IOException { for (String movedDirsKey : movedDirs) { // Delete dirs from current snapshot that are moved to next snapshot. - fromOmSnapshot - .getMetadataManager() + fromSnapshotMetadataManager .getDeletedDirTable() .deleteWithBatch(batchOp, movedDirsKey); } } - private void processReclaimKeys(BatchOperation batchOp, - OMMetadataManager metadataManager) + private void deleteKeysFromSnapshot(BatchOperation batchOp, OMMetadataManager fromSnapshotMetadataManager) throws IOException { + for (SnapshotMoveKeyInfos movedKey : nextDBKeysList) { + // Delete keys from current snapshot that are moved to next snapshot. + fromSnapshotMetadataManager.getDeletedTable().deleteWithBatch(batchOp, movedKey.getKey()); + } + + // Delete renamed keys from current snapshot that are moved to next snapshot. + for (HddsProtos.KeyValue renamedKey: renamedKeysList) { + fromSnapshotMetadataManager.getSnapshotRenamedTable().deleteWithBatch(batchOp, renamedKey.getKey()); + } + } + + @Deprecated + private void processReclaimKeys(BatchOperation batchOp, OMMetadataManager metadataManager) throws IOException { for (SnapshotMoveKeyInfos dBKey : reclaimKeysList) { - RepeatedOmKeyInfo omKeyInfos = - createRepeatedOmKeyInfo(dBKey.getKeyInfosList()); + RepeatedOmKeyInfo omKeyInfos = createRepeatedOmKeyInfo(dBKey.getKeyInfosList()); // omKeyInfos can be null, because everything from RepeatedOmKeyInfo // is moved to next snapshot which means this key can be deleted in // the current snapshot processed by SDS. The reclaim key here indicates // the key can be removed from the deleted current snapshot if (omKeyInfos == null) { - metadataManager.getDeletedTable().deleteWithBatch(batchOp, - dBKey.getKey()); + metadataManager.getDeletedTable().deleteWithBatch(batchOp, dBKey.getKey()); continue; } - metadataManager.getDeletedTable().putWithBatch(batchOp, - dBKey.getKey(), omKeyInfos); + metadataManager.getDeletedTable().putWithBatch(batchOp, dBKey.getKey(), omKeyInfos); } } - private void processDirs(BatchOperation batchOp, - OMMetadataManager omMetadataManager, - OmSnapshot fromOmSnapshot) + private void processDirs(BatchOperation batchOp, OMMetadataManager omMetadataManager, OmSnapshot fromOmSnapshot) throws IOException { for (String movedDirsKey : movedDirs) { - OmKeyInfo keyInfo = fromOmSnapshot.getMetadataManager() - .getDeletedDirTable() - .get(movedDirsKey); + OmKeyInfo keyInfo = fromOmSnapshot.getMetadataManager().getDeletedDirTable().get(movedDirsKey); if (keyInfo == null) { continue; } // Move deleted dirs to next snapshot or active DB - omMetadataManager.getDeletedDirTable().putWithBatch( - batchOp, movedDirsKey, keyInfo); + omMetadataManager.getDeletedDirTable().putWithBatch(batchOp, movedDirsKey, keyInfo); } } @@ -195,18 +194,15 @@ private void processKeys(BatchOperation batchOp, // Move renamed keys to only the next snapshot or active DB. for (HddsProtos.KeyValue renamedKey: renamedKeysList) { - metadataManager.getSnapshotRenamedTable() - .putWithBatch(batchOp, renamedKey.getKey(), renamedKey.getValue()); + metadataManager.getSnapshotRenamedTable().putWithBatch(batchOp, renamedKey.getKey(), renamedKey.getValue()); } for (SnapshotMoveKeyInfos dBKey : nextDBKeysList) { - RepeatedOmKeyInfo omKeyInfos = - createRepeatedOmKeyInfo(dBKey, metadataManager); + RepeatedOmKeyInfo omKeyInfos = createRepeatedOmKeyInfo(dBKey, metadataManager); if (omKeyInfos == null) { continue; } - metadataManager.getDeletedTable().putWithBatch(batchOp, - dBKey.getKey(), omKeyInfos); + metadataManager.getDeletedTable().putWithBatch(batchOp, dBKey.getKey(), omKeyInfos); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeAndMoveResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeAndMoveResponse.java deleted file mode 100644 index 727bd4ef92c4..000000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeAndMoveResponse.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.hadoop.ozone.om.response.snapshot; - -import jakarta.annotation.Nonnull; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.fs.Stat; -import org.apache.hadoop.hdds.StringUtils; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.hdds.utils.db.BatchOperation; -import org.apache.hadoop.hdds.utils.db.RDBStore; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshot; -import org.apache.hadoop.ozone.om.OmSnapshotManager; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.hadoop.ozone.om.lock.OMLockDetails; -import org.apache.hadoop.ozone.om.response.CleanupTableInfo; -import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import java.util.Map; - -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; -import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.SNAPSHOT_LOCK; - -/** - * Response for OMSnapshotMoveDeletedKeysRequest. - */ -@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) -public class OMSnapshotPurgeAndMoveResponse extends OMClientResponse { - private static final Logger LOG = - LoggerFactory.getLogger(OMSnapshotPurgeAndMoveResponse.class); - - private SnapshotInfo purgedSnapshot; - private SnapshotInfo nextSnapshot; - Map updatedSnapInfos; - - Path purgedSnapshotPath; - private File statePath; - - enum State { - STARTING, RENAME_TABLE_WRITE, RENAME_TABLE_LOAD, DELETED_TABLE_WRITE, DELETED_TABLE_LOAD, DELETED_DIR_TABLE_WRITE, - DELETED_DIR_TABLE_LOAD - } - - - public OMSnapshotPurgeAndMoveResponse(@Nonnull OMResponse omResponse, - @Nonnull SnapshotInfo purgedSnapshot, - @Nonnull SnapshotInfo nextSnapshot, - Map updatedSnapInfos) { - super(omResponse); - this.purgedSnapshot = purgedSnapshot; - this.nextSnapshot = nextSnapshot; - this.updatedSnapInfos = updatedSnapInfos; - } - - private void writeState(State currentState) throws IOException { - try (FileWriter writer = new FileWriter(statePath)) { - writer.write(currentState.toString()); - } - } - - private State readState() throws IOException { - return State.valueOf(StringUtils.bytes2String(Files.readAllBytes(statePath.toPath()))); - } - - private State writeAndLoadTable(OmSnapshotManager sourceSnapshotManager, OMMetadataManager targetMetadataManager, - State currentState, State writeState, State loadState, - String tableName, String prefix) throws IOException { - - File sstFile = new File(purgedSnapshotPath + "_" + tableName + ".sst"); - if (currentState != writeState) { - if (sstFile.exists()) { - sstFile.delete(); - } - try (ReferenceCounted snapshotReference = sourceSnapshotManager.getSnapshot( - purgedSnapshot.getVolumeName(), purgedSnapshot.getBucketName(), purgedSnapshot.getName())) { - OmSnapshot omSnapshot = snapshotReference.get(); - (omSnapshot.getMetadataManager().getTable(tableName)).dumpToFileWithPrefix(sstFile, prefix); - } - currentState = writeState; - writeState(currentState); - } - if (!sstFile.exists()) { - throw new IOException("Sst file created: "+ sstFile.getAbsolutePath() +" doesn't exist"); - } - targetMetadataManager.getTable(tableName).loadFromFile(sstFile); - currentState = loadState; - writeState(currentState); - sstFile.delete(); - return currentState; - } - - /** - * For when the request is not successful. - * For a successful request, the other constructor should be used. - */ - public OMSnapshotPurgeAndMoveResponse(@Nonnull OMResponse omResponse) { - super(omResponse); - checkStatusNotOK(); - } - - @Override - protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - //Loading Deleted Table, RenameTable & DeletedDirectory table by creating a temporary sst file and ingesting it - // in the rocksDB directly to avoid memory pressure on rocksdb which would have been the case if a batch would - // have been happened. Here the entries would be directly written to a file and rocksdb would just add the file - // in its meta. - purgedSnapshotPath = OmSnapshotManager.getSnapshotPath(omMetadataManager, purgedSnapshot); - statePath = new File( purgedSnapshotPath + "_state"); - OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) omMetadataManager) - .getOzoneManager().getOmSnapshotManager(); - String dbBucketKey = omMetadataManager.getBucketKey(purgedSnapshot.getVolumeName(), - purgedSnapshot.getBucketName()); - long volumeId = omMetadataManager.getVolumeId(purgedSnapshot.getVolumeName()); - long bucketId = omMetadataManager.getBucketTable().get(dbBucketKey).getObjectID(); - String dbBucketKeyForDir = omMetadataManager - .getBucketKey(Long.toString(volumeId), Long.toString(bucketId)) + OM_KEY_PREFIX; - String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; - - State currentState = null; - if (!statePath.exists() && purgedSnapshotPath.toFile().exists()) { - currentState = State.STARTING; - } else if (statePath.exists()) { - currentState = readState(); - } - ReferenceCounted rcOmNextSnapshot = null; - OMMetadataManager nextTableMetadataManager = omMetadataManager; - if (nextSnapshot != null) { - rcOmNextSnapshot = omSnapshotManager.getSnapshot(nextSnapshot.getVolumeName(), nextSnapshot.getBucketName(), - nextSnapshot.getName()); - nextTableMetadataManager = rcOmNextSnapshot.get().getMetadataManager(); - } - try { - switch (currentState) { - case STARTING: - case RENAME_TABLE_WRITE: - currentState = writeAndLoadTable(omSnapshotManager, nextTableMetadataManager, currentState, - State.RENAME_TABLE_WRITE, State.RENAME_TABLE_LOAD, OmMetadataManagerImpl.SNAPSHOT_RENAMED_TABLE, - snapshotBucketKey); - case RENAME_TABLE_LOAD: - case DELETED_TABLE_WRITE: - currentState = writeAndLoadTable(omSnapshotManager, nextTableMetadataManager, currentState, - State.DELETED_TABLE_WRITE, State.DELETED_TABLE_LOAD, OmMetadataManagerImpl.DELETED_TABLE, - snapshotBucketKey); - case DELETED_TABLE_LOAD: - case DELETED_DIR_TABLE_WRITE: - currentState = writeAndLoadTable(omSnapshotManager, nextTableMetadataManager, currentState, - State.DELETED_DIR_TABLE_WRITE, State.DELETED_DIR_TABLE_LOAD, OmMetadataManagerImpl.DELETED_DIR_TABLE, - dbBucketKeyForDir); - } - deleteCheckpointDirectory(omMetadataManager, purgedSnapshot); - statePath.delete(); - } finally { - IOUtils.closeQuietly(rcOmNextSnapshot); - } - for (Map.Entry updatedSnapshotInfo : updatedSnapInfos.entrySet()) { - omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, updatedSnapshotInfo.getKey(), - updatedSnapshotInfo.getValue()); - } - omMetadataManager.getSnapshotInfoTable().deleteWithBatch(batchOperation, purgedSnapshot.getTableKey()); - } - - /** - * Deletes the checkpoint directory for a snapshot. - */ - private void deleteCheckpointDirectory(OMMetadataManager omMetadataManager, - SnapshotInfo snapshotInfo) { - // Acquiring write lock to avoid race condition with sst filtering service which creates a sst filtered file - // inside the snapshot directory. Any operation apart which doesn't create/delete files under this snapshot - // directory can run in parallel along with this operation. - OMLockDetails omLockDetails = omMetadataManager.getLock() - .acquireWriteLock(SNAPSHOT_LOCK, snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), - snapshotInfo.getName()); - boolean acquiredSnapshotLock = omLockDetails.isLockAcquired(); - if (acquiredSnapshotLock) { - Path snapshotDirPath = OmSnapshotManager.getSnapshotPath(omMetadataManager, snapshotInfo); - try { - FileUtils.deleteDirectory(snapshotDirPath.toFile()); - } catch (IOException ex) { - LOG.error("Failed to delete snapshot directory {} for snapshot {}", - snapshotDirPath, snapshotInfo.getTableKey(), ex); - } finally { - omMetadataManager.getLock().releaseWriteLock(SNAPSHOT_LOCK, snapshotInfo.getVolumeName(), - snapshotInfo.getBucketName(), snapshotInfo.getName()); - } - } - } - - -} - diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index fef7b4552963..619c711c2525 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -31,16 +31,23 @@ import org.apache.hadoop.ozone.common.DeleteBlockGroupResult; import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; +import org.apache.hadoop.ozone.om.lock.OMLockDetails; +import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeletedKeys; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -48,18 +55,26 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.util.CheckExceptionOperation; import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.util.Preconditions; +import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Queue; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -67,6 +82,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; /** @@ -99,9 +115,25 @@ public AbstractKeyDeletingService(String serviceName, long interval, this.runCount = new AtomicLong(0); } + //Removes deleted file from AOS and moves if it is not referenced in the previous snapshot. + // Returns True if it can be deleted and false if it cannot be deleted. + private boolean deleteKeyIfNotReferencedInPreviousSnapshot(long volumeId, OmBucketInfo bucketInfo, + OmKeyInfo deletedKeyInfo, SnapshotInfo prevSnapshotInfo, SnapshotInfo prevPrevSnapshotInfo, + Table renamedTable, Table prevRenamedTable, + Table previousKeyTable, Table previousPrevKeyTable, + Map exclusiveSizeMap, Map exclusiveReplicatedSizeMap) throws IOException { + if (isKeyReclaimable(previousKeyTable, renamedTable, deletedKeyInfo, bucketInfo, volumeId, null)) { + return true; + } + calculateExclusiveSize(prevSnapshotInfo, prevPrevSnapshotInfo, deletedKeyInfo, bucketInfo, volumeId, + renamedTable, previousKeyTable, prevRenamedTable, previousPrevKeyTable, exclusiveSizeMap, + exclusiveReplicatedSizeMap); + return false; + } + protected int processKeyDeletes(List keyBlocksList, KeyManager manager, - HashMap keysToModify, + Map keysToModify, String snapTableKey) throws IOException { long startTime = Time.monotonicNow(); @@ -124,8 +156,7 @@ protected int processKeyDeletes(List keyBlocksList, if (blockDeletionResults != null) { startTime = Time.monotonicNow(); if (isRatisEnabled()) { - delCount = submitPurgeKeysRequest(blockDeletionResults, - keysToModify, snapTableKey); + delCount = submitPurgeKeysRequest(blockDeletionResults, keysToModify, snapTableKey); } else { // TODO: Once HA and non-HA paths are merged, we should have // only one code path here. Purge keys should go through an @@ -177,8 +208,8 @@ private int deleteAllKeys(List results, * @param keysToModify Updated list of RepeatedOmKeyInfo */ private int submitPurgeKeysRequest(List results, - HashMap keysToModify, String snapTableKey) { - Map, List> purgeKeysMapPerBucket = + Map keysToModify, String snapTableKey) { + Map, Pair, List>> purgeKeysMapPerBucket = new HashMap<>(); // Put all keys to be purged in a list @@ -210,13 +241,14 @@ private int submitPurgeKeysRequest(List results, } // Add keys to PurgeKeysRequest bucket wise. - for (Map.Entry, List> entry : + for (Map.Entry, Pair, List>> entry : purgeKeysMapPerBucket.entrySet()) { Pair volumeBucketPair = entry.getKey(); DeletedKeys deletedKeysInBucket = DeletedKeys.newBuilder() .setVolumeName(volumeBucketPair.getLeft()) .setBucketName(volumeBucketPair.getRight()) - .addAllKeys(entry.getValue()) + .addAllKeys(entry.getValue().getKey()) + .addAllRenamedKeys(entry.getValue().getValue()) .build(); purgeKeysRequest.addDeletedKeys(deletedKeysInBucket); } @@ -262,6 +294,33 @@ private int submitPurgeKeysRequest(List results, return deletedCount; } + protected void submitRequest(OMRequest omRequest, ClientId clientId) { + try { + if (isRatisEnabled()) { + OzoneManagerRatisServer server = + getOzoneManager().getOmRatisServer(); + + RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder() + .setClientId(clientId) + .setServerId(server.getRaftPeerId()) + .setGroupId(server.getRaftGroupId()) + .setCallId(getRunCount().get()) + .setMessage(Message.valueOf( + OMRatisHelper.convertRequestToByteString(omRequest))) + .setType(RaftClientRequest.writeRequestType()) + .build(); + + server.submitRequest(omRequest, raftClientRequest); + } else { + getOzoneManager().getOmServerProtocol() + .submitRequest(null, omRequest); + } + } catch (ServiceException e) { + LOG.error("Snapshot deep cleaning request failed. " + + "Will retry at next run.", e); + } + } + protected RaftClientRequest createRaftClientRequestForPurge( OMRequest omRequest) { return RaftClientRequest.newBuilder() @@ -280,7 +339,7 @@ protected RaftClientRequest createRaftClientRequestForPurge( * Parse Volume and Bucket Name from ObjectKey and add it to given map of * keys to be purged per bucket. */ - private void addToMap(Map, List> map, String objectKey) { + private void addToMap(Map, Pair, List>> map, String objectKey) { // Parse volume and bucket name String[] split = objectKey.split(OM_KEY_PREFIX); Preconditions.assertTrue(split.length >= 3, "Volume and/or Bucket Name " + @@ -290,9 +349,10 @@ private void addToMap(Map, List> map, String object } Pair volumeBucketPair = Pair.of(split[1], split[2]); if (!map.containsKey(volumeBucketPair)) { - map.put(volumeBucketPair, new ArrayList<>()); + map.put(volumeBucketPair, Pair.of(new ArrayList<>(), new ArrayList<>())); } - map.get(volumeBucketPair).add(objectKey); + map.get(volumeBucketPair).getKey().add(objectKey); + } protected OzoneManagerProtocolProtos.OMResponse submitPurgePaths( @@ -373,10 +433,12 @@ private OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( return purgePathsRequest.build(); } - protected PurgePathRequest prepareDeleteDirRequest( + protected Optional prepareDeleteDirRequest( long remainNum, OmKeyInfo pendingDeletedDirInfo, String delDirName, List> subDirList, - KeyManager keyManager) throws IOException { + KeyManager keyManager, boolean deleteDir, + CheckExceptionOperation, Boolean, IOException> fileDeletionChecker) + throws IOException { // step-0: Get one pending deleted directory if (LOG.isDebugEnabled()) { LOG.debug("Pending deleted dir name: {}", @@ -387,10 +449,11 @@ protected PurgePathRequest prepareDeleteDirRequest( final long volumeId = Long.parseLong(keys[1]); final long bucketId = Long.parseLong(keys[2]); - // step-1: get all sub directories under the deletedDir + // step-1: get all sub directories under the deletedDir. Always expand all sub directories irrespective of + // reference of sub-directory in previous snapshot. List subDirs = keyManager .getPendingDeletionSubDirs(volumeId, bucketId, - pendingDeletedDirInfo, remainNum); + pendingDeletedDirInfo, (keyInfo) -> true, remainNum); remainNum = remainNum - subDirs.size(); OMMetadataManager omMetadataManager = keyManager.getMetadataManager(); @@ -404,9 +467,13 @@ protected PurgePathRequest prepareDeleteDirRequest( } // step-2: get all sub files under the deletedDir - List subFiles = keyManager + // Only remove sub files if the parent directory is going to be deleted or can be reclaimed. + List subFiles = new ArrayList<>(); + for (OmKeyInfo omKeyInfo : keyManager .getPendingDeletionSubFiles(volumeId, bucketId, - pendingDeletedDirInfo, remainNum); + pendingDeletedDirInfo, (keyInfo) -> deleteDir || fileDeletionChecker.apply(keyInfo), remainNum)) { + subFiles.add(omKeyInfo); + } remainNum = remainNum - subFiles.size(); if (LOG.isDebugEnabled()) { @@ -420,46 +487,59 @@ protected PurgePathRequest prepareDeleteDirRequest( // limit. If count reached limit then there can be some more child // paths to be visited and will keep the parent deleted directory // for one more pass. - String purgeDeletedDir = remainNum > 0 ? delDirName : null; - return wrapPurgeRequest(volumeId, bucketId, - purgeDeletedDir, subFiles, subDirs); + // If there are no subpaths to expand and the directory itself cannot be reclaimed then skip purge processing for + // this dir, since this would be a noop. + String purgeDeletedDir = deleteDir && remainNum > 0 ? delDirName : null; + if (purgeDeletedDir == null && subFiles.isEmpty() && subDirs.isEmpty()) { + return Optional.empty(); + } + return Optional.of(wrapPurgeRequest(volumeId, bucketId, + purgeDeletedDir, subFiles, subDirs)); } @SuppressWarnings("checkstyle:ParameterNumber") public Pair> optimizeDirDeletesAndSubmitRequest( long remainNum, long dirNum, long subDirNum, long subFileNum, List> allSubDirList, List purgePathRequestList, String snapTableKey, long startTime, int remainingBufLimit, - KeyManager keyManager) { + KeyManager keyManager, + CheckExceptionOperation, Boolean, IOException> subDirPurgeChecker, + CheckExceptionOperation, Boolean, IOException> fileDeletionChecker) { // Optimization to handle delete sub-dir and keys to remove quickly // This case will be useful to handle when depth of directory is high + // This is not supposed to be done for snapshots. int subdirDelNum = 0; int subDirRecursiveCnt = 0; int consumedSize = 0; while (remainNum > 0 && subDirRecursiveCnt < allSubDirList.size()) { try { - Pair stringOmKeyInfoPair - = allSubDirList.get(subDirRecursiveCnt); - PurgePathRequest request = prepareDeleteDirRequest( + Pair stringOmKeyInfoPair = allSubDirList.get(subDirRecursiveCnt); + Boolean result = subDirPurgeChecker.apply(Table.newKeyValue(stringOmKeyInfoPair.getKey(), + stringOmKeyInfoPair.getValue()) ); + Optional request = prepareDeleteDirRequest( remainNum, stringOmKeyInfoPair.getValue(), stringOmKeyInfoPair.getKey(), allSubDirList, - keyManager); + keyManager, result, fileDeletionChecker); + if (!request.isPresent()) { + continue; + } if (isBufferLimitCrossed(remainingBufLimit, consumedSize, - request.getSerializedSize())) { + request.get().getSerializedSize())) { // ignore further add request break; } - consumedSize += request.getSerializedSize(); - purgePathRequestList.add(request); - remainNum = remainNum - request.getDeletedSubFilesCount(); - remainNum = remainNum - request.getMarkDeletedSubDirsCount(); + PurgePathRequest requestVal = request.get(); + consumedSize += requestVal.getSerializedSize(); + purgePathRequestList.add(requestVal); + remainNum = remainNum - requestVal.getDeletedSubFilesCount(); + remainNum = remainNum - requestVal.getMarkDeletedSubDirsCount(); // Count up the purgeDeletedDir, subDirs and subFiles - if (request.getDeletedDir() != null - && !request.getDeletedDir().isEmpty()) { + requestVal.getDeletedDir(); + if (requestVal.hasDeletedDir() && !requestVal.getDeletedDir().isEmpty()) { subdirDelNum++; } - subDirNum += request.getMarkDeletedSubDirsCount(); - subFileNum += request.getDeletedSubFilesCount(); + subDirNum += requestVal.getMarkDeletedSubDirsCount(); + subFileNum += requestVal.getDeletedSubFilesCount(); subDirRecursiveCnt++; } catch (IOException e) { LOG.error("Error while running delete directories and files " + @@ -604,7 +684,7 @@ protected SnapshotInfo getPreviousActiveSnapshot(SnapshotInfo snapInfo, String tableKey = chainManager.getTableKey(prevPathSnapshot); SnapshotInfo prevSnapInfo = omSnapshotManager.getSnapshotInfo(tableKey); if (prevSnapInfo.getSnapshotStatus() == - SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { + SNAPSHOT_ACTIVE) { return prevSnapInfo; } currSnapInfo = prevSnapInfo; @@ -731,6 +811,32 @@ protected boolean isDirReclaimable( return prevDirectoryInfo.getObjectID() != deletedDirInfo.getObjectID(); } + protected boolean isRenameEntryReclaimable(Table.KeyValue renameEntry, + Table previousDirTable, + Table prevKeyInfoTable) throws IOException { + + if (previousDirTable == null && prevKeyInfoTable == null) { + return true; + } + String prevDbKey = renameEntry.getValue(); + + + if (previousDirTable != null) { + OmDirectoryInfo prevDirectoryInfo = previousDirTable.getIfExist(prevDbKey); + if (prevDirectoryInfo != null) { + return false; + } + } + + if (prevKeyInfoTable != null) { + OmKeyInfo omKeyInfo = prevKeyInfoTable.getIfExist(prevDbKey); + return omKeyInfo == null; + } + return true; + } + + + public boolean isRatisEnabled() { if (ozoneManager == null) { return false; @@ -790,4 +896,367 @@ public long getMovedFilesCount() { public BootstrapStateHandler.Lock getBootstrapStateLock() { return lock; } + + /** + * Class to take multiple locks on a resource. + */ + protected static class MultiLocks { + private final Queue objectLocks; + private final IOzoneManagerLock lock; + private final OzoneManagerLock.Resource resource; + private final boolean writeLock; + public MultiLocks(IOzoneManagerLock lock, OzoneManagerLock.Resource resource, boolean writeLock) { + this.writeLock = writeLock; + this.resource = resource; + this.lock = lock; + this.objectLocks = new LinkedList<>(); + } + + public OMLockDetails acquireLock(Collection objects) throws OMException { + if (!objectLocks.isEmpty()) { + throw new OMException("More locks cannot be acquired when locks have been already acquired. Locks acquired : " + + objectLocks, OMException.ResultCodes.INTERNAL_ERROR); + } + OMLockDetails omLockDetails = OMLockDetails.EMPTY_DETAILS_LOCK_ACQUIRED; + for (T object : objects) { + if (object != null) { + omLockDetails = this.writeLock ? lock.acquireWriteLock(resource, object.toString()) + : lock.acquireReadLock(resource, object.toString()); + objectLocks.add(object); + if (!omLockDetails.isLockAcquired()) { + break; + } + } + } + if (!omLockDetails.isLockAcquired()) { + releaseLock(); + } + return omLockDetails; + } + + public void releaseLock() { + while (!objectLocks.isEmpty()) { + T object = objectLocks.poll(); + OMLockDetails lockDetails = this.writeLock ? lock.releaseWriteLock(resource, object.toString()) + : lock.releaseReadLock(resource, object.toString()); + } + } + } + + /** + * This class is responsible for opening last N snapshot given snapshot or AOS metadata manager by acquiring a lock. + */ + private abstract class ReclaimableFilter implements CheckExceptionOperation, + Boolean, IOException>, Closeable { + + private final SnapshotInfo currentSnapshotInfo; + private final OmSnapshotManager omSnapshotManager; + private final SnapshotChainManager snapshotChainManager; + + private final List previousSnapshotInfos; + private final List> previousOmSnapshots; + private final MultiLocks snapshotIdLocks; + private Long volumeId; + private OmBucketInfo bucketInfo; + private final OMMetadataManager metadataManager; + private final int numberOfPreviousSnapshotsFromChain; + + /** + * Filter to return deleted keys/directories which are reclaimable based on their presence in previous snapshot in + * the snapshot chain. + * @param omSnapshotManager + * @param snapshotChainManager + * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot + * in the snapshot chain corresponding to bucket key needs to be processed. + * @param metadataManager : MetadataManager corresponding to snapshot or AOS. + * @param lock : Lock for Active OM. + */ + public ReclaimableFilter(OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, + SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, + IOzoneManagerLock lock, + int numberOfPreviousSnapshotsFromChain) { + this.omSnapshotManager = omSnapshotManager; + this.currentSnapshotInfo = currentSnapshotInfo; + this.snapshotChainManager = snapshotChainManager; + this.snapshotIdLocks = new MultiLocks<>(lock, OzoneManagerLock.Resource.SNAPSHOT_GC_LOCK, false); + this.metadataManager = metadataManager; + this.numberOfPreviousSnapshotsFromChain = numberOfPreviousSnapshotsFromChain; + this.previousOmSnapshots = new ArrayList<>(numberOfPreviousSnapshotsFromChain); + this.previousSnapshotInfos = new ArrayList<>(numberOfPreviousSnapshotsFromChain); + } + + private List getLastNSnapshotInChain(String volume, String bucket) throws IOException { + if (currentSnapshotInfo != null && (!currentSnapshotInfo.getVolumeName().equals(volume) || + !currentSnapshotInfo.getBucketName().equals(bucket))) { + throw new IOException("Volume & Bucket name for snapshot : " + currentSnapshotInfo + " not matching for " + + "key in volume: " + volume + " bucket: " + bucket); + } + SnapshotInfo expectedPreviousSnapshotInfo = currentSnapshotInfo == null + ? SnapshotUtils.getLatestSnapshotInfo(volume, bucket, snapshotChainManager, omSnapshotManager) + : SnapshotUtils.getPreviousSnapshot(currentSnapshotInfo, snapshotChainManager, omSnapshotManager); + List snapshotInfos = new ArrayList<>(); + snapshotInfos.add(expectedPreviousSnapshotInfo); + SnapshotInfo snapshotInfo = expectedPreviousSnapshotInfo; + while (snapshotInfos.size() < numberOfPreviousSnapshotsFromChain) { + snapshotInfo = snapshotInfo == null ? null + : SnapshotUtils.getPreviousSnapshot(expectedPreviousSnapshotInfo, snapshotChainManager, omSnapshotManager); + snapshotInfos.add(snapshotInfo); + // If changes made to the snapshot have not been flushed to disk, throw exception immediately, next run of + // garbage collection would process the snapshot. + if (snapshotInfo != null && + !OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapshotInfo)) { + throw new IOException("Changes made to the snapshot " + snapshotInfo + " have not been flushed to the disk "); + } + } + + // Reversing list to get the correct order in chain. To ensure locking order is as per the chain ordering. + Collections.reverse(snapshotInfos); + return snapshotInfos; + } + + // Initialize the last N snapshots in the chain by acquiring locks. Throw IOException if it fails. + private void initializePreviousSnapshotsFromChain(String volume, String bucket) throws IOException { + List expectedLastNSnapshotsInChain = getLastNSnapshotInChain(volume, bucket); + List expectedSnapshotIds = expectedLastNSnapshotsInChain.stream() + .map(snapshotInfo -> snapshotInfo == null ? null : snapshotInfo.getSnapshotId()) + .collect(Collectors.toList()); + List existingSnapshotIds = previousOmSnapshots.stream() + .map(omSnapshotReferenceCounted -> omSnapshotReferenceCounted == null ? null : + omSnapshotReferenceCounted.get().getSnapshotID()).collect(Collectors.toList()); + // If snapshotIds don't match then close all snapshots and reopen the previous N snapshots. + if (!expectedSnapshotIds.equals(existingSnapshotIds)) { + close(); + try { + // Acquire lock only on last N-1 snapshot & current snapshot(AOS if it is null). + List lockIds = new ArrayList<>(expectedSnapshotIds.subList(1, existingSnapshotIds.size())); + lockIds.add(currentSnapshotInfo == null ? null : currentSnapshotInfo.getSnapshotId()); + + if (snapshotIdLocks.acquireLock(lockIds).isLockAcquired()) { + expectedLastNSnapshotsInChain = getLastNSnapshotInChain(volume, bucket); + for (SnapshotInfo snapshotInfo : expectedLastNSnapshotsInChain) { + if (snapshotInfo != null) { + // For AOS fail operation if any of the previous snapshots are not active. currentSnapshotInfo for + // AOS will be null. + previousOmSnapshots.add(currentSnapshotInfo == null + ? omSnapshotManager.getActiveSnapshot(snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), + snapshotInfo.getName()) + : omSnapshotManager.getSnapshot(snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), + snapshotInfo.getName())); + previousSnapshotInfos.add(snapshotInfo); + } else { + previousOmSnapshots.add(null); + previousSnapshotInfos.add(null); + } + + // TODO: Getting volumeId and bucket from active OM. This would be wrong on volume & bucket renames + // support. + volumeId = getOzoneManager().getMetadataManager().getVolumeId(volume); + String dbBucketKey = getOzoneManager().getMetadataManager().getBucketKey(volume, bucket); + bucketInfo = getOzoneManager().getMetadataManager().getBucketTable().get(dbBucketKey); + } + } else { + throw new IOException("Lock acquisition failed for last N snapshots : " + + expectedLastNSnapshotsInChain + " " + currentSnapshotInfo); + } + } catch (IOException e) { + this.close(); + throw e; + } + } + } + + @Override + public Boolean apply(Table.KeyValue keyValue) throws IOException { + initializePreviousSnapshotsFromChain(getVolumeName(keyValue), getBucketName(keyValue)); + return isReclaimable(keyValue); + } + + protected abstract String getVolumeName(Table.KeyValue keyValue) throws IOException; + protected abstract String getBucketName(Table.KeyValue keyValue) throws IOException; + + protected abstract Boolean isReclaimable(Table.KeyValue omKeyInfo) throws IOException; + + @Override + public void close() throws IOException { + this.snapshotIdLocks.releaseLock(); + for (ReferenceCounted previousOmSnapshot : previousOmSnapshots) { + previousOmSnapshot.close(); + } + previousOmSnapshots.clear(); + previousSnapshotInfos.clear(); + } + + public ReferenceCounted getPreviousOmSnapshot(int index) { + return previousOmSnapshots.get(index); + } + + public OMMetadataManager getMetadataManager() { + return metadataManager; + } + + public Long getVolumeId() { + return volumeId; + } + + public OmBucketInfo getBucketInfo() { + return bucketInfo; + } + + public SnapshotInfo getPreviousSnapshotInfo(int index) { + return previousSnapshotInfos.get(index); + } + } + + protected class ReclaimableKeyFilter extends ReclaimableFilter { + private final Map exclusiveSizeMap; + private final Map exclusiveReplicatedSizeMap; + + /** + * Filter to return deleted keys which are reclaimable based on their presence in previous snapshot in + * the snapshot chain. + * @param omSnapshotManager + * @param snapshotChainManager + * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot + * in the snapshot chain corresponding to bucket key needs to be processed. + * @param metadataManager : MetadataManager corresponding to snapshot or AOS. + * @param lock : Lock for Active OM. + */ + public ReclaimableKeyFilter(OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, + SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, + IOzoneManagerLock lock) { + super(omSnapshotManager, snapshotChainManager, currentSnapshotInfo, metadataManager, lock, 2); + this.exclusiveSizeMap = new HashMap<>(); + this.exclusiveReplicatedSizeMap = new HashMap<>(); + } + + @Override + protected String getVolumeName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getVolumeName(); + } + + @Override + protected String getBucketName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getBucketName(); + } + + @Override + protected Boolean isReclaimable(Table.KeyValue deletedKeyInfo) throws IOException { + ReferenceCounted previousSnapshot = getPreviousOmSnapshot(1); + ReferenceCounted previousToPreviousSnapshot = getPreviousOmSnapshot(0); + + Table previousKeyTable = null; + Table previousPrevKeyTable = null; + + Table renamedTable = getMetadataManager().getSnapshotRenamedTable(); + Table prevRenamedTable = null; + + SnapshotInfo previousSnapshotInfo = getPreviousSnapshotInfo(1); + SnapshotInfo prevPrevSnapshotInfo = getPreviousSnapshotInfo(0); + + if (previousSnapshot != null) { + previousKeyTable = previousSnapshot.get().getMetadataManager().getKeyTable(getBucketInfo().getBucketLayout()); + prevRenamedTable = previousSnapshot.get().getMetadataManager().getSnapshotRenamedTable(); + } + if (previousToPreviousSnapshot != null) { + previousPrevKeyTable = previousToPreviousSnapshot.get().getMetadataManager() + .getKeyTable(getBucketInfo().getBucketLayout()); + } + if (isKeyReclaimable(previousKeyTable, renamedTable, deletedKeyInfo.getValue(), getBucketInfo(), getVolumeId(), + null)) { + return true; + } + calculateExclusiveSize(previousSnapshotInfo, prevPrevSnapshotInfo, deletedKeyInfo.getValue(), getBucketInfo(), + getVolumeId(), renamedTable, previousKeyTable, prevRenamedTable, previousPrevKeyTable, exclusiveSizeMap, + exclusiveReplicatedSizeMap); + return false; + } + + + public Map getExclusiveSizeMap() { + return exclusiveSizeMap; + } + + public Map getExclusiveReplicatedSizeMap() { + return exclusiveReplicatedSizeMap; + } + } + + protected class ReclaimableDirFilter extends ReclaimableFilter { + + /** + * Filter to return deleted directories which are reclaimable based on their presence in previous snapshot in + * the snapshot chain. + * @param omSnapshotManager + * @param snapshotChainManager + * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot + * in the snapshot chain corresponding to bucket key needs to be processed. + * @param metadataManager : MetadataManager corresponding to snapshot or AOS. + * @param lock : Lock for Active OM. + */ + public ReclaimableDirFilter(OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, + SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, + IOzoneManagerLock lock) { + super(omSnapshotManager, snapshotChainManager, currentSnapshotInfo, metadataManager, lock, 1); + } + + @Override + protected String getVolumeName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getVolumeName(); + } + + @Override + protected String getBucketName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getBucketName(); + } + + @Override + protected Boolean isReclaimable(Table.KeyValue deletedDirInfo) throws IOException { + ReferenceCounted previousSnapshot = getPreviousOmSnapshot(0); + Table prevDirTable = previousSnapshot == null ? null : + previousSnapshot.get().getMetadataManager().getDirectoryTable(); + return isDirReclaimable(deletedDirInfo, prevDirTable, getMetadataManager().getSnapshotRenamedTable()); + } + } + + protected class ReclaimableRenameEntryFilter extends ReclaimableFilter { + + /** + * Filter to return rename table entries which are reclaimable based on the key presence in previous snapshot's + * keyTable/DirectoryTable in the snapshot chain. + * @param omSnapshotManager + * @param snapshotChainManager + * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot + * in the snapshot chain corresponding to bucket key needs to be processed. + * @param metadataManager : MetadataManager corresponding to snapshot or AOS. + * @param lock : Lock for Active OM. + */ + public ReclaimableRenameEntryFilter(OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, + SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, + IOzoneManagerLock lock) { + super(omSnapshotManager, snapshotChainManager, currentSnapshotInfo, metadataManager, lock, 1); + } + + @Override + protected Boolean isReclaimable(Table.KeyValue renameEntry) throws IOException { + ReferenceCounted previousSnapshot = getPreviousOmSnapshot(0); + Table previousKeyTable = null; + Table prevDirTable = null; + if (previousSnapshot != null) { + previousKeyTable = previousSnapshot.get().getMetadataManager().getKeyTable(getBucketInfo().getBucketLayout()); + prevDirTable = previousSnapshot.get().getMetadataManager().getDirectoryTable(); + } + return isRenameEntryReclaimable(renameEntry, prevDirTable, previousKeyTable); + } + + @Override + protected String getVolumeName(Table.KeyValue keyValue) throws IOException { + return getMetadataManager().splitRenameKey(keyValue.getKey())[0]; + } + + @Override + protected String getBucketName(Table.KeyValue keyValue) throws IOException { + return getMetadataManager().splitRenameKey(keyValue.getKey())[1]; + } + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index fd4c2d3686c9..39a81ddbcc6c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -20,22 +20,29 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; +import org.apache.hadoop.ozone.util.CheckExceptionOperation; import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; import org.slf4j.Logger; @@ -43,9 +50,18 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT; @@ -83,7 +99,6 @@ public class DirectoryDeletingService extends AbstractKeyDeletingService { private final long pathLimitPerTask; private final int ratisByteLimit; private final AtomicBoolean suspended; - private boolean runningOnAOS; public DirectoryDeletingService(long interval, TimeUnit unit, long serviceTimeout, OzoneManager ozoneManager, @@ -100,7 +115,6 @@ public DirectoryDeletingService(long interval, TimeUnit unit, // always go to 90% of max limit for request as other header will be added this.ratisByteLimit = (int) (limit * 0.9); this.suspended = new AtomicBoolean(false); - this.runningOnAOS = false; } private boolean shouldRun() { @@ -127,10 +141,6 @@ public void resume() { suspended.set(false); } - public boolean isRunningOnAOS() { - return runningOnAOS; - } - @Override public BackgroundTaskQueue getTasks() { @@ -146,131 +156,202 @@ public int getPriority() { return 0; } + private OzoneManagerProtocolProtos.SetSnapshotPropertyRequest getSetSnapshotRequestUpdatingExclusiveSize( + Map exclusiveSizeMap, Map exclusiveReplicatedSizeMap, String prevSnapshotKeyTable) { + OzoneManagerProtocolProtos.SnapshotSize snapshotSize = OzoneManagerProtocolProtos.SnapshotSize.newBuilder() + .setExclusiveSize( + exclusiveSizeMap.getOrDefault(prevSnapshotKeyTable, 0L)) + .setExclusiveReplicatedSize( + exclusiveReplicatedSizeMap.getOrDefault( + prevSnapshotKeyTable, 0L)) + .build(); + exclusiveSizeMap.remove(prevSnapshotKeyTable); + exclusiveReplicatedSizeMap.remove(prevSnapshotKeyTable); + + return OzoneManagerProtocolProtos.SetSnapshotPropertyRequest.newBuilder() + .setSnapshotKey(prevSnapshotKeyTable) + .setSnapshotDirSize(snapshotSize) + .build(); + } + + private OzoneManagerProtocolProtos.SetSnapshotPropertyRequest + getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir(String snapshotKeyTable) { + return OzoneManagerProtocolProtos.SetSnapshotPropertyRequest.newBuilder() + .setSnapshotKey(snapshotKeyTable) + .setDeepCleanedDeletedDir(true) + .build(); + } + + private void submitSetSnapshotRequest( + List setSnapshotPropertyRequests) { + OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) + .addAllSetSnapshotPropertyRequests(setSnapshotPropertyRequests) + .setClientId(clientId.toString()) + .build(); + submitRequest(omRequest, clientId); + } + + + /** + * + * @param currentSnapshotInfo if null, deleted directories in AOS should be processed. + * @param keyManager KeyManager of the underlying store. + */ + private long processDeletedDirsForStore(SnapshotInfo currentSnapshotInfo, KeyManager keyManager, + long remainNum) throws IOException { + + long dirNum = 0L; + long subDirNum = 0L; + long subFileNum = 0L; + int consumedSize = 0; + long initialRemainNum = remainNum; + List purgePathRequestList = new ArrayList<>(); + List> allSubDirList + = new ArrayList<>((int) remainNum); + String volume = currentSnapshotInfo == null ? null : currentSnapshotInfo.getVolumeName(); + String bucket = currentSnapshotInfo == null ? null : currentSnapshotInfo.getBucketName(); + Table.KeyValue pendingDeletedDirInfo; + + try (TableIterator> + deletedDirsIterator = keyManager.getPendingDeletionDirs(volume, bucket)) { + OmSnapshotManager omSnapshotManager = getOzoneManager().getOmSnapshotManager(); + SnapshotChainManager snapshotChainManager = ((OmMetadataManagerImpl)getOzoneManager().getMetadataManager()) + .getSnapshotChainManager(); + IOzoneManagerLock lock = getOzoneManager().getMetadataManager().getLock(); + ReclaimableDirFilter reclaimableDirFilter = new ReclaimableDirFilter(omSnapshotManager, snapshotChainManager, + currentSnapshotInfo, keyManager.getMetadataManager(), lock); + ReclaimableKeyFilter reclaimableSubFileFilter = new ReclaimableKeyFilter(omSnapshotManager, snapshotChainManager, + currentSnapshotInfo, keyManager.getMetadataManager(), lock); + long startTime = Time.monotonicNow(); + while (remainNum > 0 && deletedDirsIterator.hasNext()) { + pendingDeletedDirInfo = deletedDirsIterator.next(); + // Always perform listing on AOS. + Optional request = prepareDeleteDirRequest( + remainNum, pendingDeletedDirInfo.getValue(), + pendingDeletedDirInfo.getKey(), allSubDirList, + getOzoneManager().getKeyManager(), reclaimableDirFilter.apply(pendingDeletedDirInfo), + reclaimableSubFileFilter); + + if (request.isPresent() && isBufferLimitCrossed(ratisByteLimit, consumedSize, + request.get().getSerializedSize())) { + if (purgePathRequestList.size() != 0) { + // if message buffer reaches max limit, avoid sending further + remainNum = 0; + break; + } + // if directory itself is having a lot of keys / files, + // reduce capacity to minimum level + remainNum = MIN_ERR_LIMIT_PER_TASK; + request = prepareDeleteDirRequest( + remainNum, pendingDeletedDirInfo.getValue(), + pendingDeletedDirInfo.getKey(), allSubDirList, + getOzoneManager().getKeyManager(), reclaimableDirFilter.apply(pendingDeletedDirInfo), + reclaimableSubFileFilter); + } + if (!request.isPresent()) { + continue; + } + PurgePathRequest purgePathRequest = request.get(); + consumedSize += purgePathRequest.getSerializedSize(); + purgePathRequestList.add(purgePathRequest); + remainNum = remainNum - purgePathRequest.getDeletedSubFilesCount(); + remainNum = remainNum - purgePathRequest.getMarkDeletedSubDirsCount(); + // Count up the purgeDeletedDir, subDirs and subFiles + if (purgePathRequest.getDeletedDir() != null + && !purgePathRequest.getDeletedDir().isEmpty()) { + dirNum++; + } + subDirNum += purgePathRequest.getMarkDeletedSubDirsCount(); + subFileNum += purgePathRequest.getDeletedSubFilesCount(); + } + + Pair> retVal = optimizeDirDeletesAndSubmitRequest( + remainNum, dirNum, subDirNum, subFileNum, + allSubDirList, purgePathRequestList, null, startTime, + ratisByteLimit - consumedSize, + getOzoneManager().getKeyManager(), reclaimableDirFilter, reclaimableSubFileFilter); + remainNum = retVal.getKey(); + List setSnapshotPropertyRequests = new ArrayList<>(); + if (remainNum == initialRemainNum && + retVal.getValue().map(OzoneManagerProtocolProtos.OMResponse::getSuccess).orElse(true)) { + Map exclusiveReplicatedSizeMap = reclaimableSubFileFilter.getExclusiveReplicatedSizeMap(); + Map exclusiveSizeMap = reclaimableSubFileFilter.getExclusiveSizeMap(); + for (String snapshot : Stream.of(exclusiveSizeMap.keySet(), + exclusiveReplicatedSizeMap.keySet()).flatMap(Collection::stream).distinct().collect(Collectors.toList())) { + setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(exclusiveSizeMap, + exclusiveReplicatedSizeMap, snapshot)); + setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir(snapshot)); + } + + submitSetSnapshotRequest(setSnapshotPropertyRequests); + } + + } catch (IOException e) { + throw e; + } + return remainNum; + } + @Override public BackgroundTaskResult call() { if (shouldRun()) { - runningOnAOS = true; if (LOG.isDebugEnabled()) { LOG.debug("Running DirectoryDeletingService"); } getRunCount().incrementAndGet(); - long dirNum = 0L; - long subDirNum = 0L; - long subFileNum = 0L; long remainNum = pathLimitPerTask; - int consumedSize = 0; - List purgePathRequestList = new ArrayList<>(); - List> allSubDirList - = new ArrayList<>((int) remainNum); - - Table.KeyValue pendingDeletedDirInfo; - try (TableIterator> - deleteTableIterator = getOzoneManager().getMetadataManager(). - getDeletedDirTable().iterator()) { - - long startTime = Time.monotonicNow(); - while (remainNum > 0 && deleteTableIterator.hasNext()) { - pendingDeletedDirInfo = deleteTableIterator.next(); - // Do not reclaim if the directory is still being referenced by - // the previous snapshot. - if (previousSnapshotHasDir(pendingDeletedDirInfo)) { - continue; - } - - PurgePathRequest request = prepareDeleteDirRequest( - remainNum, pendingDeletedDirInfo.getValue(), - pendingDeletedDirInfo.getKey(), allSubDirList, - getOzoneManager().getKeyManager()); - if (isBufferLimitCrossed(ratisByteLimit, consumedSize, - request.getSerializedSize())) { - if (purgePathRequestList.size() != 0) { - // if message buffer reaches max limit, avoid sending further - remainNum = 0; - break; - } - // if directory itself is having a lot of keys / files, - // reduce capacity to minimum level - remainNum = MIN_ERR_LIMIT_PER_TASK; - request = prepareDeleteDirRequest( - remainNum, pendingDeletedDirInfo.getValue(), - pendingDeletedDirInfo.getKey(), allSubDirList, - getOzoneManager().getKeyManager()); - } - consumedSize += request.getSerializedSize(); - purgePathRequestList.add(request); - remainNum = remainNum - request.getDeletedSubFilesCount(); - remainNum = remainNum - request.getMarkDeletedSubDirsCount(); - // Count up the purgeDeletedDir, subDirs and subFiles - if (request.getDeletedDir() != null - && !request.getDeletedDir().isEmpty()) { - dirNum++; - } - subDirNum += request.getMarkDeletedSubDirsCount(); - subFileNum += request.getDeletedSubFilesCount(); - } - - optimizeDirDeletesAndSubmitRequest( - remainNum, dirNum, subDirNum, subFileNum, - allSubDirList, purgePathRequestList, null, startTime, - ratisByteLimit - consumedSize, - getOzoneManager().getKeyManager()); - + try { + remainNum = processDeletedDirsForStore(null, getOzoneManager().getKeyManager(), + remainNum); } catch (IOException e) { LOG.error("Error while running delete directories and files " + - "background task. Will retry at next run.", e); + "background task. Will retry at next run. on active object store", e); } - runningOnAOS = false; - } - // place holder by returning empty results of this call back. - return BackgroundTaskResult.EmptyTaskResult.newResult(); - } + if (remainNum > 0) { + SnapshotChainManager chainManager = + ((OmMetadataManagerImpl)getOzoneManager().getMetadataManager()).getSnapshotChainManager(); + OmSnapshotManager omSnapshotManager = getOzoneManager().getOmSnapshotManager(); + Iterator iterator = null; + try { + iterator = chainManager.iterator(true); - private boolean previousSnapshotHasDir( - KeyValue pendingDeletedDirInfo) throws IOException { - String key = pendingDeletedDirInfo.getKey(); - OmKeyInfo deletedDirInfo = pendingDeletedDirInfo.getValue(); - OmSnapshotManager omSnapshotManager = - getOzoneManager().getOmSnapshotManager(); - OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) - getOzoneManager().getMetadataManager(); - SnapshotInfo snapshotInfo = metadataManager.getLatestSnapshotInfo(deletedDirInfo.getVolumeName(), - deletedDirInfo.getBucketName(), omSnapshotManager); - if (snapshotInfo.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE || - !OmSnapshotManager.areSnapshotChangesFlushedToDB(metadataManager, snapshotInfo)) { - return true; - } - try (ReferenceCounted rcLatestSnapshot = - metadataManager.getLatestSnapshot( - deletedDirInfo.getVolumeName(), - deletedDirInfo.getBucketName(), - omSnapshotManager, false)) { - - if (rcLatestSnapshot != null) { - String dbRenameKey = metadataManager - .getRenameKey(deletedDirInfo.getVolumeName(), - deletedDirInfo.getBucketName(), deletedDirInfo.getObjectID()); - Table prevDirTable = - rcLatestSnapshot.get().getMetadataManager().getDirectoryTable(); - Table prevDeletedDirTable = - rcLatestSnapshot.get().getMetadataManager().getDeletedDirTable(); - OmKeyInfo prevDeletedDirInfo = prevDeletedDirTable.get(key); - if (prevDeletedDirInfo != null) { - return true; + } catch (IOException e) { + LOG.error("Error while initializing snapshot chain iterator."); + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + + while (iterator.hasNext() && remainNum > 0) { + UUID snapshotId = iterator.next(); + try { + SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(chainManager, snapshotId, omSnapshotManager); + // Wait for snapshot changes to be flushed to disk. + if (!OmSnapshotManager.areSnapshotChangesFlushedToDB( + getOzoneManager().getMetadataManager(), snapInfo)) { + LOG.info("Skipping snapshot processing since changes to snapshot {} have not been flushed to disk", + snapInfo); + continue; + } + // Check if snapshot has been directory deep cleaned. Return if deep cleaning already done. + if (snapInfo.getDeepCleanedDeletedDir()) { + LOG.debug("Snapshot {} has already been directory deep cleaned", snapInfo); + continue; + } + try (ReferenceCounted omSnapshot = omSnapshotManager.getSnapshot(snapInfo.getVolumeName(), + snapInfo.getBucketName(), snapInfo.getName())) { + remainNum = processDeletedDirsForStore(snapInfo, omSnapshot.get().getKeyManager(), remainNum); + } + + } catch (IOException e) { + LOG.error("Error while running delete directories and files " + + "background task for snapshot: {}. Will retry at next run. on active object store", snapshotId, e); + } } - String prevDirTableDBKey = metadataManager.getSnapshotRenamedTable() - .get(dbRenameKey); - // In OMKeyDeleteResponseWithFSO OzonePathKey is converted to - // OzoneDeletePathKey. Changing it back to check the previous DirTable - String prevDbKey = prevDirTableDBKey == null ? - metadataManager.getOzoneDeletePathDirKey(key) : prevDirTableDBKey; - OmDirectoryInfo prevDirInfo = prevDirTable.get(prevDbKey); - return prevDirInfo != null && - prevDirInfo.getObjectID() == deletedDirInfo.getObjectID(); } } - - return false; + // place holder by returning empty results of this call back. + return BackgroundTaskResult.EmptyTaskResult.newResult(); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 13233dc0a543..3749da2f6e29 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -30,6 +30,7 @@ import com.google.protobuf.ServiceException; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; @@ -88,7 +89,6 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private final KeyManager manager; - private boolean runningOnAOS; private int keyLimitPerTask; private final AtomicLong deletedKeyCount; private final AtomicBoolean suspended; @@ -113,7 +113,6 @@ public KeyDeletingService(OzoneManager ozoneManager, this.exclusiveSizeMap = new HashMap<>(); this.exclusiveReplicatedSizeMap = new HashMap<>(); this.snapshotSeekMap = new HashMap<>(); - this.runningOnAOS = false; } /** @@ -165,10 +164,6 @@ public void setKeyLimitPerTask(int keyLimitPerTask) { this.keyLimitPerTask = keyLimitPerTask; } - public boolean isRunningOnAOS() { - return runningOnAOS; - } - /** * A key deleting task scans OM DB and looking for a certain number of * pending-deletion keys, sends these keys along with their associated blocks @@ -178,6 +173,20 @@ public boolean isRunningOnAOS() { */ private class KeyDeletingTask implements BackgroundTask { + private void processDeletions(SnapshotInfo snapshotInfo) { + // if snapshotInfo is null, then the function should run for AOS. + final String volume; + final String bucket; + if (snapshotInfo == null) { + volume = null; + bucket = null; + + } + String volume = snapshotInfo == null ? null : snapshotInfo.getVolumeName(); + String bucket = snapshotInfo == null ? null : snapshotInfo.getBucketName(); + + } + @Override public int getPriority() { return 0; @@ -188,7 +197,6 @@ public BackgroundTaskResult call() { // Check if this is the Leader OM. If not leader, no need to execute this // task. if (shouldRun()) { - runningOnAOS = true; final long run = getRunCount().incrementAndGet(); LOG.debug("Running KeyDeletingService {}", run); @@ -246,6 +254,7 @@ private void processSnapshotDeepClean(int delCount) while (delCount < keyLimitPerTask && iterator.hasNext()) { List keysToPurge = new ArrayList<>(); HashMap keysToModify = new HashMap<>(); + Map renamedKeyMap = new HashMap<>(); SnapshotInfo currSnapInfo = iterator.next().getValue(); // Deep clean only on snapshots whose deleted directory table has been deep cleaned. This is to avoid @@ -359,18 +368,9 @@ private void processSnapshotDeepClean(int delCount) RepeatedOmKeyInfo newRepeatedOmKeyInfo = new RepeatedOmKeyInfo(); for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) { - if (previousSnapshot != null) { - // Calculates the exclusive size for the previous - // snapshot. See Java Doc for more info. - calculateExclusiveSize(previousSnapshot, - previousToPrevSnapshot, keyInfo, bucketInfo, volumeId, - snapRenamedTable, previousKeyTable, prevRenamedTable, - previousToPrevKeyTable, exclusiveSizeMap, - exclusiveReplicatedSizeMap); - } - + HddsProtos.KeyValue.Builder renamedKey = HddsProtos.KeyValue.newBuilder(); if (isKeyReclaimable(previousKeyTable, snapRenamedTable, - keyInfo, bucketInfo, volumeId, null)) { + keyInfo, bucketInfo, volumeId, renamedKey)) { List blocksForKeyDelete = currOmSnapshot .getMetadataManager() .getBlocksForKeyDelete(deletedKey); @@ -378,8 +378,14 @@ private void processSnapshotDeepClean(int delCount) blockGroupList.addAll(blocksForKeyDelete); } delCount++; + renamedKeyMap.put(deletedKey, renamedKey.getKey()); } else { newRepeatedOmKeyInfo.addOmKeyInfo(keyInfo); + calculateExclusiveSize(previousSnapshot, + previousToPrevSnapshot, keyInfo, bucketInfo, volumeId, + snapRenamedTable, previousKeyTable, prevRenamedTable, + previousToPrevKeyTable, exclusiveSizeMap, + exclusiveReplicatedSizeMap); } } @@ -417,7 +423,7 @@ private void processSnapshotDeepClean(int delCount) if (!keysToPurge.isEmpty()) { processKeyDeletes(keysToPurge, currOmSnapshot.getKeyManager(), - keysToModify, currSnapInfo.getTableKey()); + keysToModify, renamedKeyMap,currSnapInfo.getTableKey()); } } finally { IOUtils.closeQuietly(rcPrevOmSnapshot, rcPrevToPrevOmSnapshot); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java index 89f78d8bb247..d8bef39ecca1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java @@ -18,9 +18,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ServiceException; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; @@ -28,10 +28,7 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; @@ -43,12 +40,12 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; +import org.apache.hadoop.ozone.util.CheckExceptionOperation; import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.Message; @@ -59,18 +56,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Stack; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT; -import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getDirectoryInfo; import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getOmKeyInfo; -import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.setKeyNameAndFileName; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getOzonePathKeyForFso; /** @@ -145,82 +139,28 @@ public BackgroundTaskQueue getTasks() { private class SnapshotAOSDirCleanupTask implements BackgroundTask { //Expands deleted directory from active AOS if it is not referenced in the previous and breaks the dfs iteration. - private boolean expandDirectoryAndPurgeIfDirNotReferenced( - Table.KeyValue deletedDir, - Table previousDirTable, - Table renamedTable, - AtomicLong remainNum, AtomicInteger consumedSize, AtomicLong dirNum, - AtomicLong subDirNum, AtomicLong subFileNum, - List purgePathRequestList, - List> allSubDirList, - KeyManager keyManager) throws IOException { - if (isDirReclaimable(deletedDir, previousDirTable, renamedTable)) { - OzoneManagerProtocolProtos.PurgePathRequest request = prepareDeleteDirRequest( - remainNum.get(), deletedDir.getValue(), deletedDir.getKey(), - allSubDirList, keyManager); - if (isBufferLimitCrossed(ratisByteLimit, consumedSize.get(), - request.getSerializedSize())) { - if (purgePathRequestList.size() != 0) { - // if message buffer reaches max limit, avoid sending further - remainNum.set(0); - return false; - } - // if directory itself is having a lot of keys / files, - // reduce capacity to minimum level - remainNum.set(MIN_ERR_LIMIT_PER_TASK); - request = prepareDeleteDirRequest( - remainNum.get(), deletedDir.getValue(), deletedDir.getKey(), - allSubDirList, keyManager); - } - consumedSize.addAndGet(request.getSerializedSize()); - purgePathRequestList.add(request); - remainNum.addAndGet(-1 * request.getDeletedSubFilesCount()); - remainNum.addAndGet(-1 * request.getMarkDeletedSubDirsCount()); - // Count up the purgeDeletedDir, subDirs and subFiles - boolean dirDeleted = false; - if (request.getDeletedDir() != null - && !request.getDeletedDir().isEmpty()) { - dirNum.incrementAndGet(); - dirDeleted = true; - } - subDirNum.addAndGet(request.getMarkDeletedSubDirsCount()); - subFileNum.addAndGet(request.getDeletedSubFilesCount()); - // returning true if there are only files and all files along with the directory is removed. Otherwise this - // directory needs another iteration to cleanup. - return request.getMarkDeletedSubDirsCount() == 0 && dirDeleted; - } - return true; + private Pair expandDirectoryAndPurgeIfDirNotReferenced( + Table.KeyValue deletedDir, Table previousDirTable, + Table renamedTable) throws IOException { + HddsProtos.KeyValue.Builder renamedEntry = HddsProtos.KeyValue.newBuilder(); + boolean isDirReclaimable = isDirReclaimable(deletedDir, previousDirTable, renamedTable, renamedEntry); + return Pair.of(isDirReclaimable, renamedEntry.hasKey() ? renamedEntry.getKey() : null); } //Removes deleted file from AOS and moves if it is not referenced in the previous snapshot. - // Returns + // Returns True if it can be deleted and false if it cannot be deleted. private boolean deleteKeyIfNotReferencedInPreviousSnapshot( long volumeId, OmBucketInfo bucketInfo, - Table.KeyValue deletedKeyInfoEntry, - SnapshotInfo prevSnapshotInfo, SnapshotInfo prevPrevSnapshotInfo, + OmKeyInfo deletedKeyInfo, SnapshotInfo prevSnapshotInfo, SnapshotInfo prevPrevSnapshotInfo, Table renamedTable, Table prevRenamedTable, - Table previousKeyTable, Table previousPrevKeyTable, - List deletedKeyInfos, - AtomicLong remainNum, AtomicInteger consumedSize) throws IOException { - OmKeyInfo deletedKeyInfo = deletedKeyInfoEntry.getValue(); + Table previousKeyTable, Table previousPrevKeyTable) throws IOException { if (isKeyReclaimable(previousKeyTable, renamedTable, deletedKeyInfo, bucketInfo, volumeId, null)) { - OzoneManagerProtocolProtos.KeyInfo keyInfo = deletedKeyInfo.getProtobuf(true, - ClientVersion.CURRENT_VERSION); - if (isBufferLimitCrossed(ratisByteLimit, consumedSize.get(), - keyInfo.getSerializedSize()) && consumedSize.get() > 0) { - // if message buffer reaches max limit, avoid sending further - remainNum.set(0); - return false; - } - deletedKeyInfos.add(keyInfo); - remainNum.decrementAndGet(); - consumedSize.addAndGet(keyInfo.getSerializedSize()); - } else { - calculateExclusiveSize(prevSnapshotInfo, prevPrevSnapshotInfo, deletedKeyInfo, bucketInfo, volumeId, - renamedTable, previousKeyTable, prevRenamedTable, previousPrevKeyTable, exclusiveSizeMap, - exclusiveReplicatedSizeMap); + return true; } - return true; + calculateExclusiveSize(prevSnapshotInfo, prevPrevSnapshotInfo, deletedKeyInfo, bucketInfo, volumeId, + renamedTable, previousKeyTable, prevRenamedTable, previousPrevKeyTable, exclusiveSizeMap, + exclusiveReplicatedSizeMap); + return false; } @Override @@ -260,7 +200,7 @@ public BackgroundTaskResult call() { } ReferenceCounted rcPrevOmSnapshot = null; - ReferenceCounted rcPrevToPrevOmSnapshot = null; + ReferenceCounted rcPrevToPrevOmSnapshot; try { long volumeId = metadataManager .getVolumeId(currSnapInfo.getVolumeName()); @@ -291,19 +231,16 @@ public BackgroundTaskResult call() { final Table previousToPrevKeyTable; if (previousSnapshot != null) { - rcPrevOmSnapshot = omSnapshotManager.getActiveSnapshot( + rcPrevOmSnapshot = omSnapshotManager.getSnapshot( previousSnapshot.getVolumeName(), previousSnapshot.getBucketName(), previousSnapshot.getName()); OmSnapshot omPreviousSnapshot = rcPrevOmSnapshot.get(); - previousKeyTable = omPreviousSnapshot.getMetadataManager() - .getKeyTable(bucketInfo.getBucketLayout()); - prevRenamedTable = omPreviousSnapshot - .getMetadataManager().getSnapshotRenamedTable(); + previousKeyTable = omPreviousSnapshot.getMetadataManager().getKeyTable(bucketInfo.getBucketLayout()); + prevRenamedTable = omPreviousSnapshot.getMetadataManager().getSnapshotRenamedTable(); previousDirTable = omPreviousSnapshot.getMetadataManager().getDirectoryTable(); - previousToPrevSnapshot = getPreviousSnapshot( - previousSnapshot, snapChainManager, omSnapshotManager); + previousToPrevSnapshot = getPreviousSnapshot(previousSnapshot, snapChainManager, omSnapshotManager); } else { previousKeyTable = null; previousDirTable = null; @@ -313,7 +250,7 @@ public BackgroundTaskResult call() { if (previousToPrevSnapshot != null) { - rcPrevToPrevOmSnapshot = omSnapshotManager.getActiveSnapshot( + rcPrevToPrevOmSnapshot = omSnapshotManager.getSnapshot( previousToPrevSnapshot.getVolumeName(), previousToPrevSnapshot.getBucketName(), previousToPrevSnapshot.getName()); @@ -345,41 +282,67 @@ public BackgroundTaskResult call() { long startTime = Time.monotonicNow(); List purgePathRequestList = new ArrayList<>(); List> allSubDirList = new ArrayList<>(); - List deletedSubKeyInfos = new ArrayList<>(); - Operation, Boolean> operationOnDirectory = - deletedDirectoryEntry -> expandDirectoryAndPurgeIfDirNotReferenced(deletedDirectoryEntry, previousDirTable, - snapRenameTable, remainNum, consumedSize, dirNum, subDirNum, subFileNum, - purgePathRequestList, allSubDirList, getOzoneManager().getKeyManager());; - Operation, Boolean> operationOnFile = + CheckExceptionOperation checkExceptionOperationOnFile = deletedKeyInfo -> deleteKeyIfNotReferencedInPreviousSnapshot(volumeId, bucketInfo, deletedKeyInfo, previousSnapshot, previousToPrevSnapshot, snapRenameTable, prevRenamedTable, - previousKeyTable, previousToPrevKeyTable, deletedSubKeyInfos, remainNum, consumedSize);; - Supplier breakConditionSupplier = () -> remainNum.get() > 0; - boolean retVal = true; + previousKeyTable, previousToPrevKeyTable); + while (deletedDirIterator.hasNext()) { Table.KeyValue deletedDirInfo = deletedDirIterator.next(); - if (breakConditionSupplier.get()) { - retVal = false; - break; + + HddsProtos.KeyValue.Builder renamedEntry = HddsProtos.KeyValue.newBuilder(); + + // Check if the directory is reclaimable. If it is not we cannot delete the directory. + boolean isDirReclaimable = isDirReclaimable(deletedDirInfo, previousDirTable, snapRenameTable, + renamedEntry); + Optional request = prepareDeleteDirRequest( + remainNum.get(), deletedDirInfo.getValue(), deletedDirInfo.getKey(), + allSubDirList, getOzoneManager().getKeyManager(), renamedEntry.hasKey() ? + renamedEntry.getKey() : null, isDirReclaimable, checkExceptionOperationOnFile); + if (!request.isPresent()) { + continue; + } + if (isBufferLimitCrossed(ratisByteLimit, consumedSize.get(), + request.get().getSerializedSize())) { + if (purgePathRequestList.size() != 0) { + // if message buffer reaches max limit, avoid sending further + remainNum.set(0); + } + // if directory itself is having a lot of keys / files, + // reduce capacity to minimum level + remainNum.set(MIN_ERR_LIMIT_PER_TASK); + request = prepareDeleteDirRequest( + remainNum.get(), deletedDirInfo.getValue(), deletedDirInfo.getKey(), + allSubDirList, getOzoneManager().getKeyManager(), renamedEntry.getKey(), isDirReclaimable, + checkExceptionOperationOnFile); + } + if (request.isPresent()) { + OzoneManagerProtocolProtos.PurgePathRequest requestVal = request.get(); + consumedSize.addAndGet(requestVal.getSerializedSize()); + purgePathRequestList.add(requestVal); + remainNum.addAndGet(-1 * requestVal.getDeletedSubFilesCount()); + remainNum.addAndGet(-1 * requestVal.getMarkDeletedSubDirsCount()); + // Count up the purgeDeletedDir, subDirs and subFiles + if (requestVal.hasDeletedDir() && !requestVal.getDeletedDir().isEmpty()) { + dirNum.incrementAndGet(); + } + subDirNum.addAndGet(requestVal.getMarkDeletedSubDirsCount()); + subFileNum.addAndGet(requestVal.getDeletedSubFilesCount()); } - // For each deleted directory we do an in-memory DFS and - // do a deep clean based on the previous snapshot. - retVal = retVal && iterateDirectoryTree(deletedDirInfo, volumeId, bucketInfo, - operationOnFile, operationOnDirectory, - breakConditionSupplier, metadataManager); - } - if (deletedSubKeyInfos.size() > 0) { - purgePathRequestList.add(wrapPurgeRequest(volumeId, bucketInfo.getObjectID(), deletedSubKeyInfos)); } - retVal = retVal && optimizeDirDeletesAndSubmitRequest(0, dirNum.get(), subDirNum.get(), - subFileNum.get(), - allSubDirList, purgePathRequestList, currSnapInfo.getTableKey(), startTime, 0, - getOzoneManager().getKeyManager()).getRight().map(OzoneManagerProtocolProtos.OMResponse::getSuccess) - .orElse(false); + + boolean retVal = optimizeDirDeletesAndSubmitRequest(remainNum.get(), dirNum.get(), subDirNum.get(), + subFileNum.get(), allSubDirList, purgePathRequestList, + currSnapInfo.getTableKey(), startTime, 0, getOzoneManager().getKeyManager(), + (deletedDir) -> expandDirectoryAndPurgeIfDirNotReferenced(deletedDir, previousDirTable, + snapRenameTable), checkExceptionOperationOnFile).getRight() + .map(OzoneManagerProtocolProtos.OMResponse::getSuccess) + .orElse(true); + List setSnapshotPropertyRequests = new ArrayList<>(); - if (retVal && subDirNum.get() == 0) { + if (retVal && remainNum.get() == keyLimitPerSnapshot) { if (previousSnapshot != null) { setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(previousSnapshot.getTableKey())); } @@ -401,108 +364,6 @@ public BackgroundTaskResult call() { } } - /** - * Performs a DFS iteration on a given deleted directory and performs operation on the sub deleted directories and - * subfiles in the tree. - * @param deletedDirInfo - * @param volumeId - * @param bucketInfo - * @param operationOnFile - * @param operationOnDirectory - * @param breakConditionSupplier - * @param metadataManager - * @return True if complete directory was iterated and all operations returned a true otherwise false. - * middle of iteration. - * @throws IOException - */ - private boolean iterateDirectoryTree( - Table.KeyValue deletedDirInfo, - long volumeId, OmBucketInfo bucketInfo, Operation, Boolean> operationOnFile, - Operation, Boolean> operationOnDirectory, - Supplier breakConditionSupplier, - OMMetadataManager metadataManager) throws IOException { - Table dirTable = metadataManager.getDirectoryTable(); - Table fileTable = metadataManager.getFileTable(); - Stack stackNodes = new Stack<>(); - OmDirectoryInfo omDeletedDirectoryInfo = getDirectoryInfo(deletedDirInfo.getValue()); - String dirPathDbKey = metadataManager.getOzonePathKey(volumeId, bucketInfo.getObjectID(), - omDeletedDirectoryInfo); - StackNode topLevelDir = new StackNode(); - topLevelDir.setDirKey(dirPathDbKey); - topLevelDir.setDirValue(deletedDirInfo.getValue()); - stackNodes.push(topLevelDir); - String bucketPrefixKeyFSO = getOzonePathKeyForFso(metadataManager, bucketInfo.getVolumeName(), - bucketInfo.getBucketName()); - boolean retVal = true; - try (TableIterator> - directoryIterator = dirTable.iterator(bucketPrefixKeyFSO); - TableIterator> fileIterator = - fileTable.iterator(bucketPrefixKeyFSO)) { - while (!stackNodes.isEmpty()) { - StackNode stackTop = stackNodes.peek(); - // We are doing a pre order traversal here meaning, first process the current directory and all the files - // and then do a DFS for directory. This is so that we can exit early avoiding unnecessary traversal. - if (StringUtils.isEmpty(stackTop.getSubDirSeek())) { - boolean directoryOpReturnValue = operationOnDirectory.apply(Table.newKeyValue(stackTop.getDirKey(), - stackTop.getDirValue())); - if (!directoryOpReturnValue) { - retVal = false; - stackNodes.pop(); - } - if (breakConditionSupplier.get()) { - return retVal; - } - String subFileSeekValue = metadataManager.getOzonePathKey(volumeId, - bucketInfo.getObjectID(), - stackTop.getDirValue().getObjectID(), ""); - fileIterator.seek(subFileSeekValue); - while (fileIterator.hasNext()) { - Table.KeyValue fileEntry = fileIterator.next(); - OmKeyInfo omKeyInfo = fileEntry.getValue(); - if (!OMFileRequest.isImmediateChild(omKeyInfo.getParentObjectID(), - stackTop.getDirValue().getObjectID())) { - break; - } - setKeyNameAndFileName(stackTop.getDirValue(), omKeyInfo); - if(!operationOnFile.apply(fileEntry)) { - retVal = false; - stackNodes.pop(); - break; - } - if (breakConditionSupplier.get()) { - return retVal; - } - } - // Format : /volId/bucketId/parentId/ - String seekDirInDB = metadataManager - .getOzonePathKey(volumeId, bucketInfo.getObjectID(), - stackTop.getDirValue().getObjectID(), ""); - stackTop.setSubDirSeek(seekDirInDB); - } else { - // Adding \0 to seek the next greater element. - directoryIterator.seek(stackTop.getSubDirSeek() + "\0"); - if (directoryIterator.hasNext()) { - Table.KeyValue deletedSubDirInfo = directoryIterator.next(); - String deletedSubDirKey = deletedSubDirInfo.getKey(); - String prefixCheck = metadataManager.getOzoneDeletePathDirKey(stackTop.getSubDirSeek()); - // Exit if it is out of the sub dir prefix scope. - if (!deletedSubDirKey.startsWith(prefixCheck)) { - stackNodes.pop(); - } else { - stackTop.setSubDirSeek(deletedSubDirKey); - StackNode nextSubDir = new StackNode(); - nextSubDir.setDirKey(deletedSubDirInfo.getKey()); - nextSubDir.setDirValue(getOmKeyInfo(stackTop.getDirValue(), deletedSubDirInfo.getValue())); - stackNodes.push(nextSubDir); - } - } else { - stackNodes.pop(); - } - } - } - } - return retVal; - } private SetSnapshotPropertyRequest getSetSnapshotRequestUpdatingExclusiveSize(String prevSnapshotKeyTable) { SnapshotSize snapshotSize = SnapshotSize.newBuilder() @@ -606,9 +467,4 @@ public String toString() { '}'; } } - - private interface Operation { - R apply(T t) throws IOException; - - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index 4cbef9812713..50b23d265f86 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; @@ -39,6 +40,8 @@ import java.util.NoSuchElementException; import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.UUID; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; @@ -89,6 +92,12 @@ public static SnapshotInfo getSnapshotInfo(final OzoneManager ozoneManager, return snapshotInfo; } + public static SnapshotInfo getSnapshotInfo(SnapshotChainManager chainManager, UUID snapshotId, + OmSnapshotManager omSnapshotManager) throws IOException { + String tableKey = chainManager.getTableKey(snapshotId); + return omSnapshotManager.getSnapshotInfo(tableKey); + } + public static void dropColumnFamilyHandle( final ManagedRocksDB rocksDB, final ColumnFamilyHandle columnFamilyHandle) { @@ -141,43 +150,14 @@ public static void checkSnapshotActive(SnapshotInfo snapInfo, } } - public static SnapshotInfo getSnapshotInfo(SnapshotChainManager chainManager, UUID snapshotId, - OmSnapshotManager omSnapshotManager) throws IOException { - String tableKey = chainManager.getTableKey(snapshotId); - return omSnapshotManager.getSnapshotInfo(tableKey); - } - - /** - * Get the next non deleted snapshot in the snapshot chain. - */ - public static SnapshotInfo getNextActiveSnapshot(SnapshotInfo snapInfo, - SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager) throws IOException { - do { - snapInfo = getNextSnapshot(snapInfo, chainManager, omSnapshotManager); - } while (snapInfo != null && !snapInfo.getSnapshotStatus().equals(SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)); - return snapInfo; - } - /** - * Set Sanspho + * Set transactionInfo in snapshotInfo. */ public static void setTransactionInfoInSnapshot(SnapshotInfo snapshot, TermIndex termIndex) throws IOException { TransactionInfo transactionInfo = TransactionInfo.valueOf(termIndex); snapshot.setLastTransactionInfo(TransactionInfo.getCodec().toPersistedFormat(transactionInfo)); } - /** - * Get the previous non deleted snapshot in the snapshot chain. - */ - public static SnapshotInfo getPreviousActiveSnapshot(SnapshotInfo snapInfo, - SnapshotChainManager chainManager, - OmSnapshotManager omSnapshotManager) throws IOException { - do { - snapInfo = getPreviousSnapshot(snapInfo, chainManager, omSnapshotManager); - } while (snapInfo != null && !snapInfo.getSnapshotStatus().equals(SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)); - return snapInfo; - } - /** * Get the next in the snapshot chain. */ @@ -212,6 +192,16 @@ public static SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo, SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager) throws IOException { + UUID previousSnapshotId = getPreviousSnapshotId(snapInfo, chainManager); + return previousSnapshotId == null ? null : getSnapshotInfo(chainManager, previousSnapshotId, omSnapshotManager); + } + + /** + * Get the previous in the snapshot chain. + */ + public static UUID getPreviousSnapshotId(SnapshotInfo snapInfo, + SnapshotChainManager chainManager) + throws IOException { // If the snapshot is deleted in the previous run, then the in-memory // SnapshotChainManager might throw NoSuchElementException as the snapshot // is removed in-memory but OMDoubleBuffer has not flushed yet. @@ -221,9 +211,8 @@ public static SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo, try { if (chainManager.hasPreviousPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId())) { - UUID previousPathSnapshot = chainManager.previousPathSnapshot(snapInfo.getSnapshotPath(), + return chainManager.previousPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); - return getSnapshotInfo(chainManager, previousPathSnapshot, omSnapshotManager); } } catch (NoSuchElementException ex) { LOG.error("The snapshot {} is not longer in snapshot chain, It " + @@ -297,4 +286,62 @@ public static String getOzonePathKeyForFso(OMMetadataManager metadataManager, final long bucketId = metadataManager.getBucketId(volumeName, bucketName); return OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId + OM_KEY_PREFIX; } + + public static SnapshotInfo getLatestSnapshotInfo(String volumeName, String bucketName, + SnapshotChainManager snapshotChainManager, + OmSnapshotManager snapshotManager) throws IOException { + Optional latestPathSnapshot = Optional.ofNullable( + getLatestSnapshotId(volumeName, bucketName, snapshotChainManager)); + return latestPathSnapshot.isPresent() ? + getSnapshotInfo(snapshotChainManager, latestPathSnapshot.get(), snapshotManager) : null; + } + + /** + * Get the latest OmSnapshot for a snapshot path. If snapshot is not active should return null if isActive argument + * is true. + */ + public static ReferenceCounted getLatestSnapshot(String volumeName, String bucketName, + SnapshotChainManager snapshotChainManager, + OmSnapshotManager snapshotManager, boolean isActive) + throws IOException { + Optional snapshotInfo = Optional.ofNullable( + SnapshotUtils.getLatestSnapshotInfo(volumeName, bucketName, snapshotChainManager, snapshotManager)); + if (isActive && snapshotInfo.map(si -> si.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) + .orElse(false)) { + return null; + } + Optional> rcOmSnapshot = snapshotInfo.isPresent() ? + Optional.ofNullable(snapshotManager.getSnapshot(volumeName, bucketName, snapshotInfo.get().getName())) : + Optional.empty(); + + return rcOmSnapshot.orElse(null); + } + + public static UUID getLatestSnapshotId(String volumeName, String bucketName, + SnapshotChainManager snapshotChainManager) throws IOException { + String snapshotPath = volumeName + OM_KEY_PREFIX + bucketName; + return snapshotChainManager.getLatestPathSnapshotId(snapshotPath); + } + + // Validates previous snapshotId given a snapshotInfo or volumeName & bucketName. Incase snapshotInfo is null, this + // would be considered as AOS and previous snapshot becomes the latest snapshot for the bucket in the snapshot chain. + public static boolean validatePreviousSnapshotId(SnapshotInfo snapshotInfo, String volumeName, String bucketName, + SnapshotChainManager snapshotChainManager, + UUID expectedPreviousSnapshotId) { + try { + if (snapshotInfo != null && (!snapshotInfo.getVolumeName().equals(volumeName) || + !snapshotInfo.getBucketName().equals(bucketName))) { + LOG.error("Volume & Bucket mismatch expected volume: {}, expected bucket: {} for snapshot: {}", + volumeName, bucketName, snapshotInfo); + return false; + } + UUID previousSnapshotId = snapshotInfo == null ? SnapshotUtils.getLatestSnapshotId(volumeName, bucketName, + snapshotChainManager) : SnapshotUtils.getPreviousSnapshotId(snapshotInfo, snapshotChainManager); + return Objects.equals(expectedPreviousSnapshotId, previousSnapshotId); + } catch (IOException e) { + LOG.error("Error while validating previous snapshot for volume: {}, bucket: {}, snapshot: {}", volumeName, + bucketName, snapshotInfo == null ? null : snapshotInfo.getName(), e); + } + return false; + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java index c807c04688d6..ab3d810473e2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java @@ -285,6 +285,16 @@ public static void addKeyToTable(boolean openKeyTable, boolean addToCache, omMetadataManager); } + /** + * Add key entry to SnapshotRenamedTable + */ + public static String addRenamedEntryToTable(long trxnLogIndex, String volumeName, String bucketName, String key, + OMMetadataManager omMetadataManager) throws Exception { + String renameKey = omMetadataManager.getRenameKey(volumeName, bucketName, trxnLogIndex); + omMetadataManager.getSnapshotRenamedTable().put(renameKey, key); + return renameKey; + } + /** * Add key entry to KeyTable. if openKeyTable flag is true, add's entries * to openKeyTable, else add's it to keyTable. diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java index a912f549b3ce..17a59efe502a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.UUID; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; @@ -60,7 +61,7 @@ public class TestOMKeyPurgeRequestAndResponse extends TestOMKeyRequest { * Creates volume, bucket and key entries and adds to OM DB and then * deletes these keys to move them to deletedKeys table. */ - private List createAndDeleteKeys(Integer trxnIndex, String bucket) + private Pair, List> createAndDeleteKeysAndRenamedEntry(Integer trxnIndex, String bucket) throws Exception { if (bucket == null) { bucket = bucketName; @@ -70,11 +71,14 @@ private List createAndDeleteKeys(Integer trxnIndex, String bucket) omMetadataManager); List ozoneKeyNames = new ArrayList<>(numKeys); + List renamedEntries = new ArrayList<>(numKeys); for (int i = 1; i <= numKeys; i++) { String key = keyName + "-" + i; OMRequestTestUtils.addKeyToTable(false, false, volumeName, bucket, key, clientID, replicationConfig, trxnIndex++, omMetadataManager); + renamedEntries.add(OMRequestTestUtils.addRenamedEntryToTable(trxnIndex, volumeName, bucket, key, + omMetadataManager)); ozoneKeyNames.add(omMetadataManager.getOzoneKey( volumeName, bucket, key)); } @@ -86,19 +90,20 @@ private List createAndDeleteKeys(Integer trxnIndex, String bucket) deletedKeyNames.add(deletedKeyName); } - return deletedKeyNames; + return Pair.of(deletedKeyNames, renamedEntries); } /** * Create OMRequest which encapsulates DeleteKeyRequest. * @return OMRequest */ - private OMRequest createPurgeKeysRequest(List deletedKeys, + private OMRequest createPurgeKeysRequest(List deletedKeys, List renamedEntries, String snapshotDbKey) { DeletedKeys deletedKeysInBucket = DeletedKeys.newBuilder() .setVolumeName(volumeName) .setBucketName(bucketName) .addAllKeys(deletedKeys) + .addAllRenamedKeys(renamedEntries) .build(); PurgeKeysRequest.Builder purgeKeysRequest = PurgeKeysRequest.newBuilder() .addDeletedKeys(deletedKeysInBucket); @@ -159,16 +164,20 @@ private OMRequest preExecute(OMRequest originalOmRequest) throws IOException { @Test public void testValidateAndUpdateCache() throws Exception { // Create and Delete keys. The keys should be moved to DeletedKeys table - List deletedKeyNames = createAndDeleteKeys(1, null); + Pair, List> deleteKeysAndRenamedEntry = createAndDeleteKeysAndRenamedEntry(1, null); // The keys should be present in the DeletedKeys table before purging - for (String deletedKey : deletedKeyNames) { + for (String deletedKey : deleteKeysAndRenamedEntry.getKey()) { assertTrue(omMetadataManager.getDeletedTable().isExist( deletedKey)); } + for (String renamedKey : deleteKeysAndRenamedEntry.getValue()) { + assertTrue(omMetadataManager.getSnapshotRenamedTable().isExist(renamedKey)); + } // Create PurgeKeysRequest to purge the deleted keys - OMRequest omRequest = createPurgeKeysRequest(deletedKeyNames, null); + OMRequest omRequest = createPurgeKeysRequest(deleteKeysAndRenamedEntry.getKey(), + deleteKeysAndRenamedEntry.getValue(), null); OMRequest preExecutedRequest = preExecute(omRequest); OMKeyPurgeRequest omKeyPurgeRequest = @@ -186,7 +195,8 @@ public void testValidateAndUpdateCache() throws Exception { omMetadataManager.getStore().initBatchOperation()) { OMKeyPurgeResponse omKeyPurgeResponse = new OMKeyPurgeResponse( - omResponse, deletedKeyNames, null, null); + omResponse, deleteKeysAndRenamedEntry.getKey(), deleteKeysAndRenamedEntry.getValue(), null, + null); omKeyPurgeResponse.addToDBBatch(omMetadataManager, batchOperation); // Do manual commit and see whether addToBatch is successful or not. @@ -194,21 +204,28 @@ public void testValidateAndUpdateCache() throws Exception { } // The keys should not exist in the DeletedKeys table - for (String deletedKey : deletedKeyNames) { + for (String deletedKey : deleteKeysAndRenamedEntry.getKey()) { assertFalse(omMetadataManager.getDeletedTable().isExist(deletedKey)); } + // Renamed entry should not exist + for (String renamedKey : deleteKeysAndRenamedEntry.getValue()) { + assertFalse(omMetadataManager.getSnapshotRenamedTable().isExist(renamedKey)); + } } @Test public void testKeyPurgeInSnapshot() throws Exception { // Create and Delete keys. The keys should be moved to DeletedKeys table - List deletedKeyNames = createAndDeleteKeys(1, null); + Pair, List> deleteKeysAndRenamedEntry = createAndDeleteKeysAndRenamedEntry(1, null); SnapshotInfo snapInfo = createSnapshot("snap1"); // The keys should be not present in the active Db's deletedTable - for (String deletedKey : deletedKeyNames) { + for (String deletedKey : deleteKeysAndRenamedEntry.getKey()) { assertFalse(omMetadataManager.getDeletedTable().isExist(deletedKey)); } + for (String renamedKey : deleteKeysAndRenamedEntry.getValue()) { + assertFalse(omMetadataManager.getSnapshotRenamedTable().isExist(renamedKey)); + } SnapshotInfo fromSnapshotInfo = new SnapshotInfo.Builder() .setVolumeName(volumeName) @@ -224,14 +241,19 @@ public void testKeyPurgeInSnapshot() throws Exception { OmSnapshot omSnapshot = rcOmSnapshot.get(); // The keys should be present in the snapshot's deletedTable - for (String deletedKey : deletedKeyNames) { + for (String deletedKey : deleteKeysAndRenamedEntry.getKey()) { assertTrue(omSnapshot.getMetadataManager() .getDeletedTable().isExist(deletedKey)); } + // The keys should be present in the snapshot's deletedTable + for (String renamedKey : deleteKeysAndRenamedEntry.getValue()) { + assertTrue(omSnapshot.getMetadataManager() + .getSnapshotRenamedTable().isExist(renamedKey)); + } // Create PurgeKeysRequest to purge the deleted keys - OMRequest omRequest = createPurgeKeysRequest(deletedKeyNames, - snapInfo.getTableKey()); + OMRequest omRequest = createPurgeKeysRequest(deleteKeysAndRenamedEntry.getKey(), + deleteKeysAndRenamedEntry.getValue(), snapInfo.getTableKey()); OMRequest preExecutedRequest = preExecute(omRequest); OMKeyPurgeRequest omKeyPurgeRequest = @@ -249,7 +271,8 @@ public void testKeyPurgeInSnapshot() throws Exception { omMetadataManager.getStore().initBatchOperation()) { OMKeyPurgeResponse omKeyPurgeResponse = new OMKeyPurgeResponse( - omResponse, deletedKeyNames, fromSnapshotInfo, null); + omResponse, deleteKeysAndRenamedEntry.getKey(), deleteKeysAndRenamedEntry.getValue(), + fromSnapshotInfo, null); omKeyPurgeResponse.addToDBBatch(omMetadataManager, batchOperation); // Do manual commit and see whether addToBatch is successful or not. @@ -257,11 +280,16 @@ public void testKeyPurgeInSnapshot() throws Exception { } // The keys should not exist in the DeletedKeys table - for (String deletedKey : deletedKeyNames) { + for (String deletedKey : deleteKeysAndRenamedEntry.getKey()) { assertFalse(omSnapshot.getMetadataManager() .getDeletedTable().isExist(deletedKey)); } + for (String renamedEntry : deleteKeysAndRenamedEntry.getValue()) { + assertFalse(omSnapshot.getMetadataManager() + .getSnapshotRenamedTable().isExist(renamedEntry)); + } + omSnapshot = null; rcOmSnapshot.close(); } From 536b63a8a9fa8fd52f74354f1057374aba9d9575 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 5 Sep 2024 13:19:44 -0700 Subject: [PATCH 03/22] Push interim merge in progress Change-Id: I3cd843cd6e4075b8bac2ff78bcb35dcc6f55f9bd --- .../TestSnapshotDirectoryCleaningService.java | 270 ---------- .../apache/hadoop/ozone/om/KeyManager.java | 23 +- .../hadoop/ozone/om/KeyManagerImpl.java | 69 +-- .../snapshot/OMSnapshotCreateRequest.java | 11 +- .../OMSnapshotMoveDeletedKeysRequest.java | 6 +- .../snapshot/OMSnapshotPurgeRequest.java | 3 +- .../service/AbstractKeyDeletingService.java | 62 +-- .../om/service/DirectoryDeletingService.java | 18 +- .../ozone/om/service/KeyDeletingService.java | 445 ++++++----------- .../om/service/SnapshotDeletingService.java | 4 +- .../SnapshotDirectoryCleaningService.java | 470 ------------------ .../ozone/om/snapshot/SnapshotUtils.java | 46 +- 12 files changed, 269 insertions(+), 1158 deletions(-) delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java deleted file mode 100644 index 03df331087b8..000000000000 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.om.snapshot; - -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.hadoop.ozone.om.service.SnapshotDirectoryCleaningService; -import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.tag.Flaky; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicLong; - -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * Test Snapshot Directory Service. - */ -@Timeout(300) -public class TestSnapshotDirectoryCleaningService { - - private static final Logger LOG = - LoggerFactory.getLogger(TestSnapshotDirectoryCleaningService.class); - - private static MiniOzoneCluster cluster; - private static FileSystem fs; - private static String volumeName; - private static String bucketName; - private static OzoneClient client; - - @BeforeAll - public static void init() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OMConfigKeys.OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL, 2500); - conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 2500, - TimeUnit.MILLISECONDS); - conf.setBoolean(OZONE_ACL_ENABLED, true); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) - .build(); - cluster.waitForClusterToBeReady(); - client = cluster.newClient(); - - // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, - BucketLayout.FILE_SYSTEM_OPTIMIZED); - volumeName = bucket.getVolumeName(); - bucketName = bucket.getName(); - - String rootPath = String.format("%s://%s.%s/", - OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName); - - // Set the fs.defaultFS and start the filesystem - conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); - // Set the number of keys to be processed during batch operate. - conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); - - fs = FileSystem.get(conf); - } - - @AfterAll - public static void teardown() { - IOUtils.closeQuietly(client); - if (cluster != null) { - cluster.shutdown(); - } - IOUtils.closeQuietly(fs); - } - - @AfterEach - public void cleanup() { - assertDoesNotThrow(() -> { - Path root = new Path("/"); - FileStatus[] fileStatuses = fs.listStatus(root); - for (FileStatus fileStatus : fileStatuses) { - fs.delete(fileStatus.getPath(), true); - } - }); - } - - @SuppressWarnings("checkstyle:LineLength") - @Test - @Flaky("HDDS-11129") - public void testExclusiveSizeWithDirectoryDeepClean() throws Exception { - - Table deletedDirTable = - cluster.getOzoneManager().getMetadataManager().getDeletedDirTable(); - Table keyTable = - cluster.getOzoneManager().getMetadataManager() - .getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED); - Table dirTable = - cluster.getOzoneManager().getMetadataManager().getDirectoryTable(); - Table deletedKeyTable = - cluster.getOzoneManager().getMetadataManager().getDeletedTable(); - Table snapshotInfoTable = - cluster.getOzoneManager().getMetadataManager().getSnapshotInfoTable(); - SnapshotDirectoryCleaningService snapshotDirectoryCleaningService = - cluster.getOzoneManager().getKeyManager().getSnapshotDirectoryService(); - - /* DirTable - /v/b/snapDir - /v/b/snapDir/appRoot0-2/ - /v/b/snapDir/appRoot0-2/parentDir0-2/ - FileTable - /v/b/snapDir/testKey0 - testKey4 = 5 keys - /v/b/snapDir/appRoot0-2/parentDir0-2/childFile = 9 keys - /v/b/snapDir/appRoot0/parentDir0-2/childFile0-4 = 15 keys - */ - - Path root = new Path("/snapDir"); - // Create parent dir from root. - fs.mkdirs(root); - - // Add 5 files inside root dir - // Creates /v/b/snapDir/testKey0 - testKey4 - for (int i = 0; i < 5; i++) { - Path path = new Path(root, "testKey" + i); - try (FSDataOutputStream stream = fs.create(path)) { - stream.write(1); - } - } - - // Creates /v/b/snapDir/appRoot0-2/parentDir0-2/childFile - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - Path appRoot = new Path(root, "appRoot" + j); - Path parent = new Path(appRoot, "parentDir" + i); - Path child = new Path(parent, "childFile"); - try (FSDataOutputStream stream = fs.create(child)) { - stream.write(1); - } - } - } - - assertTableRowCount(keyTable, 14); - assertTableRowCount(dirTable, 13); - // Create snapshot - client.getObjectStore().createSnapshot(volumeName, bucketName, "snap1"); - - // Creates /v/b/snapDir/appRoot0/parentDir0-2/childFile0-4 - for (int i = 0; i < 3; i++) { - Path appRoot = new Path(root, "appRoot0"); - Path parent = new Path(appRoot, "parentDir" + i); - for (int j = 0; j < 5; j++) { - Path child = new Path(parent, "childFile" + j); - try (FSDataOutputStream stream = fs.create(child)) { - stream.write(1); - } - } - } - - for (int i = 5; i < 10; i++) { - Path path = new Path(root, "testKey" + i); - try (FSDataOutputStream stream = fs.create(path)) { - stream.write(1); - } - } - - assertTableRowCount(deletedDirTable, 0); - assertTableRowCount(keyTable, 34); - assertTableRowCount(dirTable, 13); - Path appRoot0 = new Path(root, "appRoot0"); - // Only parentDir0-2/childFile under appRoot0 is exclusive for snap1 - fs.delete(appRoot0, true); - assertTableRowCount(deletedDirTable, 1); - client.getObjectStore().createSnapshot(volumeName, bucketName, "snap2"); - - // Delete testKey0-9 - for (int i = 0; i < 10; i++) { - Path testKey = new Path(root, "testKey" + i); - fs.delete(testKey, false); - } - - fs.delete(root, true); - assertTableRowCount(deletedKeyTable, 10); - client.getObjectStore().createSnapshot(volumeName, bucketName, "snap3"); - long prevRunCount = snapshotDirectoryCleaningService.getRunCount().get(); - GenericTestUtils.waitFor(() -> snapshotDirectoryCleaningService.getRunCount().get() - > prevRunCount + 1, 100, 10000); - - Thread.sleep(2000); - Map expectedSize = new HashMap() {{ - // /v/b/snapDir/appRoot0/parentDir0-2/childFile contribute - // exclusive size, /v/b/snapDir/appRoot0/parentDir0-2/childFile0-4 - // are deep cleaned and hence don't contribute to size. - put("snap1", 3L); - // Only testKey5-9 contribute to the exclusive size - put("snap2", 5L); - put("snap3", 0L); - }}; - Thread.sleep(500); - try (TableIterator> - iterator = snapshotInfoTable.iterator()) { - while (iterator.hasNext()) { - Table.KeyValue snapshotEntry = iterator.next(); - String snapshotName = snapshotEntry.getValue().getName(); - assertEquals(expectedSize.get(snapshotName), snapshotEntry.getValue(). - getExclusiveSize()); - // Since for the test we are using RATIS/THREE - assertEquals(expectedSize.get(snapshotName) * 3, - snapshotEntry.getValue().getExclusiveReplicatedSize()); - - } - } - } - - private void assertTableRowCount(Table table, int count) - throws TimeoutException, InterruptedException { - GenericTestUtils.waitFor(() -> assertTableRowCount(count, table), 1000, - 120000); // 2 minutes - } - - private boolean assertTableRowCount(int expectedCount, - Table table) { - AtomicLong count = new AtomicLong(0L); - assertDoesNotThrow(() -> { - count.set(cluster.getOzoneManager().getMetadataManager().countRowsInTable(table)); - LOG.info("{} actual row count={}, expectedCount={}", table.getName(), - count.get(), expectedCount); - }); - return count.get() == expectedCount; - } -} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 75b6b6fb3f79..12ac39082f11 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -29,17 +29,14 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; import org.apache.hadoop.ozone.om.fs.OzoneManagerFS; import org.apache.hadoop.hdds.utils.BackgroundService; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; -import org.apache.hadoop.ozone.om.service.SnapshotDirectoryCleaningService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket; import org.apache.hadoop.ozone.util.CheckExceptionOperation; import java.io.IOException; import java.time.Duration; -import java.util.Iterator; import java.util.List; /** @@ -141,6 +138,20 @@ PendingKeysDeletion getPendingDeletionKeys( CheckExceptionOperation, Boolean, IOException> filter, int count) throws IOException; + /** + * Returns a list renamed entries from the snapshotRenamedTable. + * + * @param count max number of keys to return. + * @param filter filter to apply on the entries. + * @return a Pair of list of {@link org.apache.hadoop.hdds.utils.db.Table.KeyValue} representing the keys in the + * underlying metadataManager. + * @throws IOException + */ + List> getRenamesKeyEntries( + String volume, String bucket, String startKey, + CheckExceptionOperation, Boolean, IOException> filter, int count) + throws IOException; + /** * Returns the names of up to {@code count} open keys whose age is * greater than or equal to {@code expireThreshold}. @@ -305,10 +316,4 @@ List getPendingDeletionSubFiles(long volumeId, long bucketId, OmKeyIn */ SnapshotDeletingService getSnapshotDeletingService(); - /** - * Returns the instance of Snapshot Directory service. - * @return Background service. - */ - SnapshotDirectoryCleaningService getSnapshotDirectoryService(); - } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 3b0d43b6b1f3..366d790313ba 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -69,8 +69,6 @@ import org.apache.hadoop.net.TableMapping; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.common.BlockGroup; -import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.util.CheckExceptionOperation; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -101,7 +99,6 @@ import org.apache.hadoop.ozone.om.service.MultipartUploadCleanupService; import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; -import org.apache.hadoop.ozone.om.service.SnapshotDirectoryCleaningService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager; @@ -137,7 +134,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.ETAG; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT; @@ -149,10 +145,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_TIMEOUT_DEFAULT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL_DEFAULT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DIRECTORY_SERVICE_TIMEOUT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DIRECTORY_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OzoneManagerUtils.getBucketLayout; @@ -163,7 +155,6 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.SCM_GET_PIPELINE_EXCEPTION; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; -import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.KEY; @@ -203,7 +194,6 @@ public class KeyManagerImpl implements KeyManager { private BackgroundService openKeyCleanupService; private BackgroundService multipartUploadCleanupService; - private SnapshotDirectoryCleaningService snapshotDirectoryCleaningService; private DNSToSwitchMapping dnsToSwitchMapping; public KeyManagerImpl(OzoneManager om, ScmClient scmClient, @@ -321,22 +311,6 @@ public void start(OzoneConfiguration configuration) { } } - if (snapshotDirectoryCleaningService == null && - ozoneManager.isFilesystemSnapshotEnabled()) { - long dirDeleteInterval = configuration.getTimeDuration( - OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL, - OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL_DEFAULT, - TimeUnit.MILLISECONDS); - long serviceTimeout = configuration.getTimeDuration( - OZONE_SNAPSHOT_DIRECTORY_SERVICE_TIMEOUT, - OZONE_SNAPSHOT_DIRECTORY_SERVICE_TIMEOUT_DEFAULT, - TimeUnit.MILLISECONDS); - snapshotDirectoryCleaningService = new SnapshotDirectoryCleaningService( - dirDeleteInterval, TimeUnit.MILLISECONDS, serviceTimeout, - ozoneManager, scmClient.getBlockClient()); - snapshotDirectoryCleaningService.start(); - } - if (multipartUploadCleanupService == null) { long serviceInterval = configuration.getTimeDuration( OZONE_OM_MPU_CLEANUP_SERVICE_INTERVAL, @@ -393,10 +367,6 @@ public void stop() throws IOException { multipartUploadCleanupService.shutdown(); multipartUploadCleanupService = null; } - if (snapshotDirectoryCleaningService != null) { - snapshotDirectoryCleaningService.shutdown(); - snapshotDirectoryCleaningService = null; - } } private OmBucketInfo getBucketInfo(String volumeName, String bucketName) @@ -727,6 +697,41 @@ public PendingKeysDeletion getPendingDeletionKeys( return new PendingKeysDeletion(keyBlocksList, keysToModify, nextPageStartKey); } + @Override + public List> getRenamesKeyEntries( + String volume, String bucket, String startKey, + CheckExceptionOperation, Boolean, IOException> filter, + int count) throws IOException { + // Bucket prefix would be empty if volume is empty i.e. either null or "". + Optional bucketPrefix = Optional.ofNullable(volume).map(vol -> vol.isEmpty() ? null : vol) + .map(vol -> metadataManager.getBucketKeyPrefix(vol, bucket)); + List> renamedEntries = new ArrayList<>(); + try (TableIterator> + renamedKeyIter = metadataManager.getSnapshotRenamedTable().iterator(bucketPrefix.orElse(""))) { + + /* Seeking to the start key if it not null. The next key picked up would be ensured to start with the bucket + prefix, {@link org.apache.hadoop.hdds.utils.db.Table#iterator(bucketPrefix)} would ensure this. + */ + if (startKey != null) { + renamedKeyIter.seek(startKey); + } + int currentCount = 0; + while (renamedKeyIter.hasNext() && currentCount < count) { + RepeatedOmKeyInfo notReclaimableKeyInfo = new RepeatedOmKeyInfo(); + Table.KeyValue kv = renamedKeyIter.next(); + if (kv != null) { + + // Multiple keys with the same path can be queued in one DB entry + if (filter.apply(kv)) { + renamedEntries.add(Table.newKeyValue(kv.getKey(), kv.getValue())); + currentCount++; + } + } + } + } + return renamedEntries; + } + @Override public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, int count, BucketLayout bucketLayout, Duration leaseThreshold) throws IOException { @@ -777,10 +782,6 @@ public SnapshotDeletingService getSnapshotDeletingService() { return snapshotDeletingService; } - public SnapshotDirectoryCleaningService getSnapshotDirectoryService() { - return snapshotDirectoryCleaningService; - } - public boolean isSstFilteringSvcEnabled() { long serviceInterval = ozoneManager.getConfiguration() .getTimeDuration(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 002e25b7637d..a140fdada763 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -187,7 +187,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn // pre-replicated key size counter in OmBucketInfo. snapshotInfo.setReferencedSize(estimateBucketDataSize(omBucketInfo)); SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); - addSnapshotInfoToSnapshotChainAndCache(ozoneManager.getOmSnapshotManager(), omMetadataManager, + addSnapshotInfoToSnapshotChainAndCache(ozoneManager, omMetadataManager, termIndex.getIndex()); omResponse.setCreateSnapshotResponse( @@ -250,7 +250,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * it was removed at T-5. */ private void addSnapshotInfoToSnapshotChainAndCache( - OmSnapshotManager omSnapshotManager, + OzoneManager ozoneManager, OmMetadataManagerImpl omMetadataManager, long transactionLogIndex ) throws IOException { @@ -270,11 +270,10 @@ private void addSnapshotInfoToSnapshotChainAndCache( snapshotInfo.setPathPreviousSnapshotId(latestPathSnapshot); snapshotInfo.setGlobalPreviousSnapshotId(latestGlobalSnapshot); Optional previousSnapshot = Optional.ofNullable(SnapshotUtils.getLatestSnapshotInfo( - snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), - snapshotChainManager, omSnapshotManager)); + snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), ozoneManager, snapshotChainManager)); Optional previousPrevSnapshot = previousSnapshot.isPresent() ? - Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(previousSnapshot.get(), snapshotChainManager, - omSnapshotManager)) : Optional.empty(); + Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(ozoneManager, + snapshotChainManager, previousSnapshot.get())) : Optional.empty(); // Reset the deep clean flag for the next active snapshot if and only if the last 2 snapshots in the // chain are active, otherwise set it to prevent deep cleaning from running till the deleted snapshots don't diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index 4479e07d144a..0c64e8cc3b00 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -78,10 +78,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OMClientResponse omClientResponse = null; OzoneManagerProtocolProtos.OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); try { - fromSnapshot = moveDeletedKeysRequest.hasFromSnapshotID() ? SnapshotUtils.getSnapshotInfo(snapshotChainManager, - fromProtobuf(moveDeletedKeysRequest.getFromSnapshotID()), omSnapshotManager) : + fromSnapshot = moveDeletedKeysRequest.hasFromSnapshotID() ? SnapshotUtils.getSnapshotInfo(ozoneManager, + snapshotChainManager, fromProtobuf(moveDeletedKeysRequest.getFromSnapshotID())) : SnapshotInfo.getFromProtobuf(moveDeletedKeysRequest.getFromSnapshot()); - nextSnapshot = SnapshotUtils.getNextSnapshot(fromSnapshot, snapshotChainManager, omSnapshotManager); + nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager,fromSnapshot); // Only skip the move operation for newer requests. Newer requests would always have the purgeMovedKeys in the // request. Skip move operation if the next snapshot is not active. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index f362283667b5..b298a5b8a97c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -79,6 +79,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn ozoneManager.getMetadataManager(); SnapshotChainManager snapshotChainManager = omMetadataManager.getSnapshotChainManager(); + OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); OMClientResponse omClientResponse = null; @@ -108,7 +109,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } SnapshotInfo nextSnapshot = - SnapshotUtils.getNextSnapshot(fromSnapshot, snapshotChainManager, omSnapshotManager); + SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); Optional previousSnapshot = Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(fromSnapshot, snapshotChainManager, omSnapshotManager)); Optional previousPrevSnapshot = previousSnapshot.isPresent() ? diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index d2d7ca8dbc1b..05e8a0935b33 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -131,13 +131,13 @@ private boolean deleteKeyIfNotReferencedInPreviousSnapshot(long volumeId, OmBuck return false; } - protected int processKeyDeletes(List keyBlocksList, + protected Pair processKeyDeletes(List keyBlocksList, KeyManager manager, Map keysToModify, String snapTableKey) throws IOException { long startTime = Time.monotonicNow(); - int delCount = 0; + Pair purgeResult = Pair.of(0, false); if (LOG.isDebugEnabled()) { LOG.debug("Send {} key(s) to SCM: {}", keyBlocksList.size(), keyBlocksList); @@ -156,17 +156,17 @@ protected int processKeyDeletes(List keyBlocksList, if (blockDeletionResults != null) { startTime = Time.monotonicNow(); if (isRatisEnabled()) { - delCount = submitPurgeKeysRequest(blockDeletionResults, keysToModify, snapTableKey); + purgeResult = submitPurgeKeysRequest(blockDeletionResults, keysToModify, snapTableKey); } else { // TODO: Once HA and non-HA paths are merged, we should have // only one code path here. Purge keys should go through an // OMRequest model. - delCount = deleteAllKeys(blockDeletionResults, manager); + purgeResult = deleteAllKeys(blockDeletionResults, manager); } LOG.info("Blocks for {} (out of {}) keys are deleted from DB in {} ms", - delCount, blockDeletionResults.size(), Time.monotonicNow() - startTime); + purgeResult, blockDeletionResults.size(), Time.monotonicNow() - startTime); } - return delCount; + return purgeResult; } /** @@ -175,12 +175,12 @@ protected int processKeyDeletes(List keyBlocksList, * @param results DeleteBlockGroups returned by SCM. * @throws IOException on Error */ - private int deleteAllKeys(List results, + private Pair deleteAllKeys(List results, KeyManager manager) throws IOException { Table deletedTable = manager.getMetadataManager().getDeletedTable(); DBStore store = manager.getMetadataManager().getStore(); - + boolean purgeSuccess = true; // Put all keys to delete in a single transaction and call for delete. int deletedCount = 0; try (BatchOperation writeBatch = store.initBatchOperation()) { @@ -193,12 +193,14 @@ private int deleteAllKeys(List results, LOG.debug("Key {} deleted from OM DB", result.getObjectKey()); } deletedCount++; + } else { + purgeSuccess = false; } } // Write a single transaction for delete. store.commitBatchOperation(writeBatch); } - return deletedCount; + return Pair.of(deletedCount, purgeSuccess); } /** @@ -207,13 +209,15 @@ private int deleteAllKeys(List results, * @param results DeleteBlockGroups returned by SCM. * @param keysToModify Updated list of RepeatedOmKeyInfo */ - private int submitPurgeKeysRequest(List results, - Map keysToModify, String snapTableKey) { + private Pair submitPurgeKeysRequest(List results, + Map keysToModify, List renameEntriesToBeDeleted, + String snapTableKey) { Map, Pair, List>> purgeKeysMapPerBucket = new HashMap<>(); // Put all keys to be purged in a list int deletedCount = 0; + boolean purgeSuccess = true; for (DeleteBlockGroupResult result : results) { if (result.isSuccess()) { // Add key to PurgeKeys list. @@ -232,6 +236,8 @@ private int submitPurgeKeysRequest(List results, } } deletedCount++; + } else { + purgeSuccess = false; } } @@ -284,14 +290,17 @@ private int submitPurgeKeysRequest(List results, try { RaftClientRequest raftClientRequest = createRaftClientRequestForPurge(omRequest); - ozoneManager.getOmRatisServer().submitRequest(omRequest, + OzoneManagerProtocolProtos.OMResponse omResponse = ozoneManager.getOmRatisServer().submitRequest(omRequest, raftClientRequest); + if (omResponse != null) { + purgeSuccess = purgeSuccess && omResponse.getSuccess(); + } } catch (ServiceException e) { LOG.error("PurgeKey request failed. Will retry at next run."); - return 0; + return Pair.of(0, false); } - return deletedCount; + return Pair.of(deletedCount, purgeSuccess); } protected void submitRequest(OMRequest omRequest, ClientId clientId) { @@ -672,25 +681,6 @@ protected boolean isBufferLimitCrossed( return cLimit + increment >= maxLimit; } - protected SnapshotInfo getPreviousActiveSnapshot(SnapshotInfo snapInfo, SnapshotChainManager chainManager) - throws IOException { - SnapshotInfo currSnapInfo = snapInfo; - while (chainManager.hasPreviousPathSnapshot( - currSnapInfo.getSnapshotPath(), currSnapInfo.getSnapshotId())) { - - UUID prevPathSnapshot = chainManager.previousPathSnapshot( - currSnapInfo.getSnapshotPath(), currSnapInfo.getSnapshotId()); - String tableKey = chainManager.getTableKey(prevPathSnapshot); - SnapshotInfo prevSnapInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, tableKey); - if (prevSnapInfo.getSnapshotStatus() == - SNAPSHOT_ACTIVE) { - return prevSnapInfo; - } - currSnapInfo = prevSnapInfo; - } - return null; - } - protected SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo, SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager) throws IOException { @@ -991,14 +981,14 @@ private List getLastNSnapshotInChain(String volume, String bucket) "key in volume: " + volume + " bucket: " + bucket); } SnapshotInfo expectedPreviousSnapshotInfo = currentSnapshotInfo == null - ? SnapshotUtils.getLatestSnapshotInfo(volume, bucket, snapshotChainManager, omSnapshotManager) - : SnapshotUtils.getPreviousSnapshot(currentSnapshotInfo, snapshotChainManager, omSnapshotManager); + ? SnapshotUtils.getLatestSnapshotInfo(volume, bucket, ozoneManager, snapshotChainManager) + : SnapshotUtils.getPreviousSnapshot(ozoneManager, snapshotChainManager, currentSnapshotInfo); List snapshotInfos = new ArrayList<>(); snapshotInfos.add(expectedPreviousSnapshotInfo); SnapshotInfo snapshotInfo = expectedPreviousSnapshotInfo; while (snapshotInfos.size() < numberOfPreviousSnapshotsFromChain) { snapshotInfo = snapshotInfo == null ? null - : SnapshotUtils.getPreviousSnapshot(expectedPreviousSnapshotInfo, snapshotChainManager, omSnapshotManager); + : SnapshotUtils.getPreviousSnapshot(ozoneManager, snapshotChainManager, expectedPreviousSnapshotInfo); snapshotInfos.add(snapshotInfo); // If changes made to the snapshot have not been flushed to disk, throw exception immediately, next run of // garbage collection would process the snapshot. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index 39a81ddbcc6c..4fb5fca248c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -211,6 +211,7 @@ private long processDeletedDirsForStore(SnapshotInfo currentSnapshotInfo, KeyMan = new ArrayList<>((int) remainNum); String volume = currentSnapshotInfo == null ? null : currentSnapshotInfo.getVolumeName(); String bucket = currentSnapshotInfo == null ? null : currentSnapshotInfo.getBucketName(); + String snapshotTableKey = currentSnapshotInfo == null ? null : currentSnapshotInfo.getTableKey(); Table.KeyValue pendingDeletedDirInfo; try (TableIterator> @@ -221,8 +222,8 @@ private long processDeletedDirsForStore(SnapshotInfo currentSnapshotInfo, KeyMan IOzoneManagerLock lock = getOzoneManager().getMetadataManager().getLock(); ReclaimableDirFilter reclaimableDirFilter = new ReclaimableDirFilter(omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); - ReclaimableKeyFilter reclaimableSubFileFilter = new ReclaimableKeyFilter(omSnapshotManager, snapshotChainManager, - currentSnapshotInfo, keyManager.getMetadataManager(), lock); + ReclaimableKeyFilter reclaimableSubFileFilter = new ReclaimableKeyFilter(omSnapshotManager, + snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); long startTime = Time.monotonicNow(); while (remainNum > 0 && deletedDirsIterator.hasNext()) { pendingDeletedDirInfo = deletedDirsIterator.next(); @@ -268,20 +269,25 @@ private long processDeletedDirsForStore(SnapshotInfo currentSnapshotInfo, KeyMan Pair> retVal = optimizeDirDeletesAndSubmitRequest( remainNum, dirNum, subDirNum, subFileNum, - allSubDirList, purgePathRequestList, null, startTime, + allSubDirList, purgePathRequestList, snapshotTableKey, startTime, ratisByteLimit - consumedSize, getOzoneManager().getKeyManager(), reclaimableDirFilter, reclaimableSubFileFilter); remainNum = retVal.getKey(); - List setSnapshotPropertyRequests = new ArrayList<>(); + if (remainNum == initialRemainNum && retVal.getValue().map(OzoneManagerProtocolProtos.OMResponse::getSuccess).orElse(true)) { + List setSnapshotPropertyRequests = new ArrayList<>(); Map exclusiveReplicatedSizeMap = reclaimableSubFileFilter.getExclusiveReplicatedSizeMap(); Map exclusiveSizeMap = reclaimableSubFileFilter.getExclusiveSizeMap(); for (String snapshot : Stream.of(exclusiveSizeMap.keySet(), exclusiveReplicatedSizeMap.keySet()).flatMap(Collection::stream).distinct().collect(Collectors.toList())) { setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(exclusiveSizeMap, exclusiveReplicatedSizeMap, snapshot)); - setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir(snapshot)); + } + //Updating directory deep clean flag of snapshot. + if (currentSnapshotInfo != null) { + setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir( + currentSnapshotInfo.getTableKey())); } submitSetSnapshotRequest(setSnapshotPropertyRequests); @@ -325,7 +331,7 @@ public BackgroundTaskResult call() { while (iterator.hasNext() && remainNum > 0) { UUID snapshotId = iterator.next(); try { - SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(chainManager, snapshotId, omSnapshotManager); + SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(getOzoneManager(), chainManager, snapshotId); // Wait for snapshot changes to be flushed to disk. if (!OmSnapshotManager.areSnapshotChangesFlushedToDB( getOzoneManager().getMetadataManager(), snapInfo)) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 3749da2f6e29..3702ee7fbfdc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -18,16 +18,22 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.Stream; import com.google.common.base.Preconditions; import com.google.protobuf.ServiceException; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -44,8 +50,11 @@ import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; @@ -95,6 +104,7 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private final Map exclusiveSizeMap; private final Map exclusiveReplicatedSizeMap; private final Map snapshotSeekMap; + private static ClientId clientId = ClientId.randomId(); public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, @@ -173,18 +183,111 @@ public void setKeyLimitPerTask(int keyLimitPerTask) { */ private class KeyDeletingTask implements BackgroundTask { - private void processDeletions(SnapshotInfo snapshotInfo) { - // if snapshotInfo is null, then the function should run for AOS. - final String volume; - final String bucket; - if (snapshotInfo == null) { - volume = null; - bucket = null; + private OzoneManagerProtocolProtos.SetSnapshotPropertyRequest getSetSnapshotRequestUpdatingExclusiveSize( + Map exclusiveSizeMap, Map exclusiveReplicatedSizeMap, String prevSnapshotKeyTable) { + OzoneManagerProtocolProtos.SnapshotSize snapshotSize = OzoneManagerProtocolProtos.SnapshotSize.newBuilder() + .setExclusiveSize( + exclusiveSizeMap.getOrDefault(prevSnapshotKeyTable, 0L)) + .setExclusiveReplicatedSize( + exclusiveReplicatedSizeMap.getOrDefault( + prevSnapshotKeyTable, 0L)) + .build(); + exclusiveSizeMap.remove(prevSnapshotKeyTable); + exclusiveReplicatedSizeMap.remove(prevSnapshotKeyTable); - } - String volume = snapshotInfo == null ? null : snapshotInfo.getVolumeName(); - String bucket = snapshotInfo == null ? null : snapshotInfo.getBucketName(); + return OzoneManagerProtocolProtos.SetSnapshotPropertyRequest.newBuilder() + .setSnapshotKey(prevSnapshotKeyTable) + .setSnapshotSize(snapshotSize) + .build(); + } + + private OzoneManagerProtocolProtos.SetSnapshotPropertyRequest + getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir(String snapshotKeyTable) { + return OzoneManagerProtocolProtos.SetSnapshotPropertyRequest.newBuilder() + .setSnapshotKey(snapshotKeyTable) + .setDeepCleanedDeletedKey(true) + .build(); + } + private void submitSetSnapshotRequest( + List setSnapshotPropertyRequests) { + OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) + .addAllSetSnapshotPropertyRequests(setSnapshotPropertyRequests) + .setClientId(clientId.toString()) + .build(); + submitRequest(omRequest, clientId); + } + + + /** + * + * @param currentSnapshotInfo if null, deleted directories in AOS should be processed. + * @param keyManager KeyManager of the underlying store. + */ + private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyManager keyManager, + int remainNum) throws IOException { + String volume = currentSnapshotInfo == null ? null : currentSnapshotInfo.getVolumeName(); + String bucket = currentSnapshotInfo == null ? null : currentSnapshotInfo.getBucketName(); + String snapshotTableKey = currentSnapshotInfo == null ? null : currentSnapshotInfo.getTableKey(); + String startKey = ""; + int initialRemainNum = remainNum; + boolean successStatus = true; + try { + // TODO: [SNAPSHOT] HDDS-7968. Reclaim eligible key blocks in + // snapshot's deletedTable when active DB's deletedTable + // doesn't have enough entries left. + // OM would have to keep track of which snapshot the key is coming + // from if the above would be done inside getPendingDeletionKeys(). + OmSnapshotManager omSnapshotManager = getOzoneManager().getOmSnapshotManager(); + SnapshotChainManager snapshotChainManager = ((OmMetadataManagerImpl)getOzoneManager().getMetadataManager()) + .getSnapshotChainManager(); + IOzoneManagerLock lock = getOzoneManager().getMetadataManager().getLock(); + + ReclaimableKeyFilter reclaimableKeyFilter = new ReclaimableKeyFilter(omSnapshotManager, snapshotChainManager, + currentSnapshotInfo, keyManager.getMetadataManager(), lock); + ReclaimableRenameEntryFilter renameEntryFilter = new ReclaimableRenameEntryFilter( + omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); + List> renamedTableEntries = keyManager.getRenamesKeyEntries(volume, bucket, + startKey, renameEntryFilter, remainNum); + submitPurgeKeysRequest() + + PendingKeysDeletion pendingKeysDeletion = keyManager.getPendingDeletionKeys(volume, bucket, startKey, + reclaimableKeyFilter, remainNum); + List keyBlocksList = pendingKeysDeletion.getKeyBlocksList(); + if (keyBlocksList != null && !keyBlocksList.isEmpty()) { + Pair purgeResult = processKeyDeletes(keyBlocksList, getOzoneManager().getKeyManager(), + pendingKeysDeletion.getKeysToModify(), snapshotTableKey); + remainNum -= purgeResult.getKey(); + successStatus = purgeResult.getValue(); + } + + // Checking remainNum is greater than zero and not equal to the initial value if there were some keys to + // reclaim. This is to check if + if (remainNum > 0 && successStatus) { + List setSnapshotPropertyRequests = new ArrayList<>(); + Map exclusiveReplicatedSizeMap = reclaimableKeyFilter.getExclusiveReplicatedSizeMap(); + Map exclusiveSizeMap = reclaimableKeyFilter.getExclusiveSizeMap(); + for (String snapshot : Stream.of(exclusiveSizeMap.keySet(), + exclusiveReplicatedSizeMap.keySet()).flatMap(Collection::stream).distinct().collect(Collectors.toList())) { + setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(exclusiveSizeMap, + exclusiveReplicatedSizeMap, snapshot)); + } + + //Updating directory deep clean flag of snapshot. + if (currentSnapshotInfo != null) { + setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir( + snapshotTableKey)); + } + + + submitSetSnapshotRequest(setSnapshotPropertyRequests); + } + + } catch (IOException e) { + throw e; + } + return remainNum; } @Override @@ -200,298 +303,64 @@ public BackgroundTaskResult call() { final long run = getRunCount().incrementAndGet(); LOG.debug("Running KeyDeletingService {}", run); - int delCount = 0; + int remainNum = keyLimitPerTask; try { - // TODO: [SNAPSHOT] HDDS-7968. Reclaim eligible key blocks in - // snapshot's deletedTable when active DB's deletedTable - // doesn't have enough entries left. - // OM would have to keep track of which snapshot the key is coming - // from if the above would be done inside getPendingDeletionKeys(). - PendingKeysDeletion pendingKeysDeletion = manager - .getPendingDeletionKeys(getKeyLimitPerTask()); - List keyBlocksList = pendingKeysDeletion - .getKeyBlocksList(); - if (keyBlocksList != null && !keyBlocksList.isEmpty()) { - delCount = processKeyDeletes(keyBlocksList, - getOzoneManager().getKeyManager(), - pendingKeysDeletion.getKeysToModify(), null); - deletedKeyCount.addAndGet(delCount); - } - runningOnAOS = false; + remainNum = processDeletedKeysForStore(null, getOzoneManager().getKeyManager(), + remainNum); } catch (IOException e) { - LOG.error("Error while running delete keys background task. Will " + - "retry at next run.", e); + LOG.error("Error while running delete directories and files " + + "background task. Will retry at next run. on active object store", e); } - try { - if (delCount < keyLimitPerTask) { - processSnapshotDeepClean(delCount); + if (remainNum > 0) { + SnapshotChainManager chainManager = + ((OmMetadataManagerImpl)getOzoneManager().getMetadataManager()).getSnapshotChainManager(); + OmSnapshotManager omSnapshotManager = getOzoneManager().getOmSnapshotManager(); + Iterator iterator = null; + try { + iterator = chainManager.iterator(true); + + } catch (IOException e) { + LOG.error("Error while initializing snapshot chain iterator."); + return BackgroundTaskResult.EmptyTaskResult.newResult(); } - } catch (Exception e) { - LOG.error("Error while running deep clean on snapshots. Will " + - "retry at next run.", e); - } - } - // By design, no one cares about the results of this call back. - return EmptyTaskResult.newResult(); - } - @SuppressWarnings("checkstyle:MethodLength") - private void processSnapshotDeepClean(int delCount) - throws IOException { - OmSnapshotManager omSnapshotManager = - getOzoneManager().getOmSnapshotManager(); - OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) - getOzoneManager().getMetadataManager(); - SnapshotChainManager snapChainManager = metadataManager - .getSnapshotChainManager(); - Table snapshotInfoTable = - getOzoneManager().getMetadataManager().getSnapshotInfoTable(); - Map> deepCleanedSnapshots = new HashMap<>(); - try (TableIterator> iterator = snapshotInfoTable.iterator()) { - - while (delCount < keyLimitPerTask && iterator.hasNext()) { - List keysToPurge = new ArrayList<>(); - HashMap keysToModify = new HashMap<>(); - Map renamedKeyMap = new HashMap<>(); - SnapshotInfo currSnapInfo = iterator.next().getValue(); - - // Deep clean only on snapshots whose deleted directory table has been deep cleaned. This is to avoid - // unnecessary additional iteration on the deleted table after SnapshotDirectoryDeletingService adds - // entries in deleted table. We should wait for all changes on the snapshot to be flushed to the db. - if (!currSnapInfo.getDeepCleanedDeletedDir() || currSnapInfo.getDeepClean() - || !OmSnapshotManager.areSnapshotChangesFlushedToDB(metadataManager, currSnapInfo.getTableKey())) { - continue; - } - - try (ReferenceCounted - rcCurrOmSnapshot = omSnapshotManager.getSnapshot( - currSnapInfo.getVolumeName(), - currSnapInfo.getBucketName(), - currSnapInfo.getName())) { - OmSnapshot currOmSnapshot = rcCurrOmSnapshot.get(); - - Table snapDeletedTable = - currOmSnapshot.getMetadataManager().getDeletedTable(); - Table snapRenamedTable = - currOmSnapshot.getMetadataManager().getSnapshotRenamedTable(); - - long volumeId = metadataManager.getVolumeId( - currSnapInfo.getVolumeName()); - // Get bucketInfo for the snapshot bucket to get bucket layout. - String dbBucketKey = metadataManager.getBucketKey( - currSnapInfo.getVolumeName(), currSnapInfo.getBucketName()); - OmBucketInfo bucketInfo = metadataManager.getBucketTable() - .get(dbBucketKey); - - if (bucketInfo == null) { - throw new IllegalStateException("Bucket " + "/" + currSnapInfo - .getVolumeName() + "/" + currSnapInfo.getBucketName() + - " is not found. BucketInfo should not be null for" + - " snapshotted bucket. The OM is in unexpected state."); - } - - String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; - SnapshotInfo previousSnapshot = getPreviousSnapshot( - currSnapInfo, snapChainManager, omSnapshotManager); - SnapshotInfo previousToPrevSnapshot = null; - - if (previousSnapshot != null) { - previousToPrevSnapshot = getPreviousSnapshot( - previousSnapshot, snapChainManager, omSnapshotManager); - } - - Table previousKeyTable = null; - Table prevRenamedTable = null; - ReferenceCounted rcPrevOmSnapshot = null; - - // Split RepeatedOmKeyInfo and update current snapshot - // deletedKeyTable and next snapshot deletedKeyTable. - if (previousSnapshot != null) { - rcPrevOmSnapshot = omSnapshotManager.getSnapshot( - previousSnapshot.getVolumeName(), - previousSnapshot.getBucketName(), - previousSnapshot.getName()); - OmSnapshot omPreviousSnapshot = rcPrevOmSnapshot.get(); - - previousKeyTable = omPreviousSnapshot.getMetadataManager() - .getKeyTable(bucketInfo.getBucketLayout()); - prevRenamedTable = omPreviousSnapshot - .getMetadataManager().getSnapshotRenamedTable(); - } - - Table previousToPrevKeyTable = null; - ReferenceCounted rcPrevToPrevOmSnapshot = null; - if (previousToPrevSnapshot != null) { - rcPrevToPrevOmSnapshot = omSnapshotManager.getSnapshot( - previousToPrevSnapshot.getVolumeName(), - previousToPrevSnapshot.getBucketName(), - previousToPrevSnapshot.getName()); - OmSnapshot omPreviousToPrevSnapshot = rcPrevToPrevOmSnapshot.get(); - - previousToPrevKeyTable = omPreviousToPrevSnapshot - .getMetadataManager() - .getKeyTable(bucketInfo.getBucketLayout()); - } - - try (TableIterator> deletedIterator = snapDeletedTable - .iterator()) { - - String lastKeyInCurrentRun = null; - String deletedTableSeek = snapshotSeekMap.getOrDefault( - currSnapInfo.getTableKey(), snapshotBucketKey); - deletedIterator.seek(deletedTableSeek); - // To avoid processing the last key from the previous - // run again. - if (!deletedTableSeek.equals(snapshotBucketKey) && - deletedIterator.hasNext()) { - deletedIterator.next(); + while (iterator.hasNext() && remainNum > 0) { + UUID snapshotId = iterator.next(); + try { + SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(getOzoneManager(), chainManager, snapshotId); + // Wait for snapshot changes to be flushed to disk. + if (!OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapInfo)) { + LOG.info("Skipping snapshot processing since changes to snapshot {} have not been flushed to disk", + snapInfo); + continue; } - - while (deletedIterator.hasNext() && delCount < keyLimitPerTask) { - Table.KeyValue - deletedKeyValue = deletedIterator.next(); - String deletedKey = deletedKeyValue.getKey(); - lastKeyInCurrentRun = deletedKey; - - // Exit if it is out of the bucket scope. - if (!deletedKey.startsWith(snapshotBucketKey)) { - break; - } - - RepeatedOmKeyInfo repeatedOmKeyInfo = - deletedKeyValue.getValue(); - - List blockGroupList = new ArrayList<>(); - RepeatedOmKeyInfo newRepeatedOmKeyInfo = - new RepeatedOmKeyInfo(); - for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) { - HddsProtos.KeyValue.Builder renamedKey = HddsProtos.KeyValue.newBuilder(); - if (isKeyReclaimable(previousKeyTable, snapRenamedTable, - keyInfo, bucketInfo, volumeId, renamedKey)) { - List blocksForKeyDelete = currOmSnapshot - .getMetadataManager() - .getBlocksForKeyDelete(deletedKey); - if (blocksForKeyDelete != null) { - blockGroupList.addAll(blocksForKeyDelete); - } - delCount++; - renamedKeyMap.put(deletedKey, renamedKey.getKey()); - } else { - newRepeatedOmKeyInfo.addOmKeyInfo(keyInfo); - calculateExclusiveSize(previousSnapshot, - previousToPrevSnapshot, keyInfo, bucketInfo, volumeId, - snapRenamedTable, previousKeyTable, prevRenamedTable, - previousToPrevKeyTable, exclusiveSizeMap, - exclusiveReplicatedSizeMap); - } - } - - if (newRepeatedOmKeyInfo.getOmKeyInfoList().size() > 0 && - newRepeatedOmKeyInfo.getOmKeyInfoList().size() != - repeatedOmKeyInfo.getOmKeyInfoList().size()) { - keysToModify.put(deletedKey, newRepeatedOmKeyInfo); - } - - if (newRepeatedOmKeyInfo.getOmKeyInfoList().size() != - repeatedOmKeyInfo.getOmKeyInfoList().size()) { - keysToPurge.addAll(blockGroupList); - } + // Check if snapshot has been directory deep cleaned. Return if directory deep cleaning is not + // done. + if (!snapInfo.getDeepCleanedDeletedDir()) { + LOG.debug("Snapshot {} hasn't done deleted directory deep cleaning yet. Skipping the snapshot in this" + + " iteration.", snapInfo); + continue; } - - if (delCount < keyLimitPerTask) { - // Deep clean is completed, we can update the SnapInfo. - deepCleanedSnapshots.put(currSnapInfo.getTableKey(), Optional.empty()); - // exclusiveSizeList contains check is used to prevent - // case where there is no entry in deletedTable, this - // will throw NPE when we submit request. - if (previousSnapshot != null && exclusiveSizeMap - .containsKey(previousSnapshot.getTableKey())) { - deepCleanedSnapshots.put(currSnapInfo.getTableKey(), Optional.of(previousSnapshot.getTableKey())); - } - snapshotSeekMap.remove(currSnapInfo.getTableKey()); - } else { - // There are keys that still needs processing - // we can continue from it in the next iteration - if (lastKeyInCurrentRun != null) { - snapshotSeekMap.put(currSnapInfo.getTableKey(), - lastKeyInCurrentRun); - } + // Checking if snapshot has been key deep cleaned already. + if (snapInfo.getDeepClean()) { + LOG.debug("Snapshot {} has already done deleted key deep cleaning.", snapInfo); + continue; } - - if (!keysToPurge.isEmpty()) { - processKeyDeletes(keysToPurge, currOmSnapshot.getKeyManager(), - keysToModify, renamedKeyMap,currSnapInfo.getTableKey()); + try (ReferenceCounted omSnapshot = omSnapshotManager.getSnapshot(snapInfo.getVolumeName(), + snapInfo.getBucketName(), snapInfo.getName())) { + remainNum = processDeletedKeysForStore(snapInfo, omSnapshot.get().getKeyManager(), remainNum); } - } finally { - IOUtils.closeQuietly(rcPrevOmSnapshot, rcPrevToPrevOmSnapshot); + + } catch (IOException e) { + LOG.error("Error while running delete directories and files " + + "background task for snapshot: {}. Will retry at next run. on active object store", snapshotId, e); } } - } } - updateDeepCleanedSnapshots(deepCleanedSnapshots); - } - - private SnapshotSize getSnapshotSize(String snapshotDBKey) { - return SnapshotSize.newBuilder() - .setExclusiveSize(exclusiveSizeMap.getOrDefault(snapshotDBKey, 0L)) - .setExclusiveReplicatedSize( - exclusiveReplicatedSizeMap.getOrDefault(snapshotDBKey, 0L)) - .setAddSize(true) - .build(); - } - - private void updateDeepCleanedSnapshots(Map> deepCleanedSnapshots) { - for (Map.Entry> deepCleanedSnapshot : deepCleanedSnapshots.entrySet()) { - ClientId clientId = ClientId.randomId(); - String snapshotDBKey = deepCleanedSnapshot.getKey(); - Optional previousSnapshotDBKey = deepCleanedSnapshot.getValue(); - SetSnapshotPropertyRequest.Builder setSnapshotPropertyRequest = SetSnapshotPropertyRequest.newBuilder() - .setSnapshotKey(snapshotDBKey) - .setDeepCleanedDeletedKey(true); - OMRequest.Builder omRequest = OMRequest.newBuilder() - .setCmdType(Type.SetSnapshotProperty) - .addSetSnapshotPropertyRequests(setSnapshotPropertyRequest) - .setClientId(clientId.toString()); - if (previousSnapshotDBKey.isPresent()) { - SetSnapshotPropertyRequest setPreviousSnapshotPropertyRequest = - SetSnapshotPropertyRequest.newBuilder() - .setSnapshotKey(previousSnapshotDBKey.get()) - .setSnapshotSize(getSnapshotSize(previousSnapshotDBKey.get())) - .build(); - omRequest.addSetSnapshotPropertyRequests(setPreviousSnapshotPropertyRequest); - } - submitRequest(omRequest.build(), clientId); - } - } - - public void submitRequest(OMRequest omRequest, ClientId clientId) { - try { - if (isRatisEnabled()) { - OzoneManagerRatisServer server = getOzoneManager().getOmRatisServer(); - - RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder() - .setClientId(clientId) - .setServerId(server.getRaftPeerId()) - .setGroupId(server.getRaftGroupId()) - .setCallId(getRunCount().get()) - .setMessage(Message.valueOf( - OMRatisHelper.convertRequestToByteString(omRequest))) - .setType(RaftClientRequest.writeRequestType()) - .build(); - - server.submitRequest(omRequest, raftClientRequest); - } else { - getOzoneManager().getOmServerProtocol() - .submitRequest(null, omRequest); - } - } catch (ServiceException e) { - LOG.error("Snapshot deep cleaning request failed. " + - "Will retry at next run.", e); - } + // By design, no one cares about the results of this call back. + return EmptyTaskResult.newResult(); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index a673d846e15d..d553c157c0ea 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -130,13 +130,13 @@ public BackgroundTaskResult call() throws InterruptedException { Iterator iterator = snapshotChainManager.iterator(true); long snapshotLimit = snapshotDeletionPerTask; while (iterator.hasNext() && snapshotLimit > 0) { - SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(chainManager, iterator.next(), omSnapshotManager); + SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, chainManager, iterator.next()); // Only Iterate in deleted snapshot & only if all the changes have been flushed into disk. if (shouldIgnoreSnapshot(snapInfo)) { continue; } - SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(snapInfo, chainManager, omSnapshotManager); + SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, chainManager, snapInfo); // Continue if the next snapshot is not active. This is to avoid unnecessary copies from one snapshot to // another. if (nextSnapshot != null && nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java deleted file mode 100644 index d8bef39ecca1..000000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.ozone.om.service; - -import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ServiceException; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; -import org.apache.hadoop.hdds.utils.BackgroundTask; -import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; -import org.apache.hadoop.hdds.utils.BackgroundTaskResult; -import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshot; -import org.apache.hadoop.ozone.om.OmSnapshotManager; -import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.SnapshotChainManager; -import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; -import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; -import org.apache.hadoop.ozone.util.CheckExceptionOperation; -import org.apache.hadoop.util.Time; -import org.apache.ratis.protocol.ClientId; -import org.apache.ratis.protocol.Message; -import org.apache.ratis.protocol.RaftClientRequest; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT; -import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getOmKeyInfo; -import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getOzonePathKeyForFso; - -/** - * Snapshot BG Service for deleted directory deep clean and exclusive size - * calculation for deleted directories. - */ -public class SnapshotDirectoryCleaningService - extends AbstractKeyDeletingService { - // Use only a single thread for DirDeletion. Multiple threads would read - // or write to same tables and can send deletion requests for same key - // multiple times. - private static final int SNAPSHOT_DIR_CORE_POOL_SIZE = 1; - private static final int MIN_ERR_LIMIT_PER_TASK = 1000; - private final AtomicBoolean suspended; - private final Map exclusiveSizeMap; - private final Map exclusiveReplicatedSizeMap; - private final long keyLimitPerSnapshot; - private final int ratisByteLimit; - - public SnapshotDirectoryCleaningService(long interval, TimeUnit unit, - long serviceTimeout, - OzoneManager ozoneManager, - ScmBlockLocationProtocol scmClient) { - super(SnapshotDirectoryCleaningService.class.getSimpleName(), - interval, unit, SNAPSHOT_DIR_CORE_POOL_SIZE, serviceTimeout, - ozoneManager, scmClient); - this.suspended = new AtomicBoolean(false); - this.exclusiveSizeMap = new HashMap<>(); - this.exclusiveReplicatedSizeMap = new HashMap<>(); - this.keyLimitPerSnapshot = ozoneManager.getConfiguration().getLong( - OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK, - OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); - int limit = (int) ozoneManager.getConfiguration().getStorageSize( - OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, - OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT, - StorageUnit.BYTES); - // always go to 90% of max limit for request as other header will be added - this.ratisByteLimit = (int) (limit * 0.9); - } - - private boolean shouldRun() { - if (getOzoneManager() == null) { - // OzoneManager can be null for testing - return true; - } - return getOzoneManager().isLeaderReady() && !suspended.get(); - } - - /** - * Suspend the service. - */ - @VisibleForTesting - public void suspend() { - suspended.set(true); - } - - /** - * Resume the service if suspended. - */ - @VisibleForTesting - public void resume() { - suspended.set(false); - } - - @Override - public BackgroundTaskQueue getTasks() { - BackgroundTaskQueue queue = new BackgroundTaskQueue(); - queue.add(new SnapshotDirectoryCleaningService.SnapshotAOSDirCleanupTask()); - return queue; - } - - private class SnapshotAOSDirCleanupTask implements BackgroundTask { - - //Expands deleted directory from active AOS if it is not referenced in the previous and breaks the dfs iteration. - private Pair expandDirectoryAndPurgeIfDirNotReferenced( - Table.KeyValue deletedDir, Table previousDirTable, - Table renamedTable) throws IOException { - HddsProtos.KeyValue.Builder renamedEntry = HddsProtos.KeyValue.newBuilder(); - boolean isDirReclaimable = isDirReclaimable(deletedDir, previousDirTable, renamedTable, renamedEntry); - return Pair.of(isDirReclaimable, renamedEntry.hasKey() ? renamedEntry.getKey() : null); - } - - //Removes deleted file from AOS and moves if it is not referenced in the previous snapshot. - // Returns True if it can be deleted and false if it cannot be deleted. - private boolean deleteKeyIfNotReferencedInPreviousSnapshot( - long volumeId, OmBucketInfo bucketInfo, - OmKeyInfo deletedKeyInfo, SnapshotInfo prevSnapshotInfo, SnapshotInfo prevPrevSnapshotInfo, - Table renamedTable, Table prevRenamedTable, - Table previousKeyTable, Table previousPrevKeyTable) throws IOException { - if (isKeyReclaimable(previousKeyTable, renamedTable, deletedKeyInfo, bucketInfo, volumeId, null)) { - return true; - } - calculateExclusiveSize(prevSnapshotInfo, prevPrevSnapshotInfo, deletedKeyInfo, bucketInfo, volumeId, - renamedTable, previousKeyTable, prevRenamedTable, previousPrevKeyTable, exclusiveSizeMap, - exclusiveReplicatedSizeMap); - return false; - } - - @Override - public BackgroundTaskResult call() { - if (!shouldRun()) { - return BackgroundTaskResult.EmptyTaskResult.newResult(); - } - LOG.debug("Running SnapshotDirectoryCleaningService"); - - getRunCount().incrementAndGet(); - OmSnapshotManager omSnapshotManager = - getOzoneManager().getOmSnapshotManager(); - Table snapshotInfoTable = - getOzoneManager().getMetadataManager().getSnapshotInfoTable(); - OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) - getOzoneManager().getMetadataManager(); - SnapshotChainManager snapChainManager = metadataManager - .getSnapshotChainManager(); - - - try (TableIterator> iterator = snapshotInfoTable.iterator()) { - while (iterator.hasNext()) { - AtomicLong dirNum = new AtomicLong(0); - AtomicLong subDirNum = new AtomicLong(0); - AtomicLong subFileNum = new AtomicLong(0); - AtomicLong remainNum = new AtomicLong(keyLimitPerSnapshot); - AtomicInteger consumedSize = new AtomicInteger(0); - // Getting the Snapshot info from cache. - SnapshotInfo currSnapInfo = iterator.next().getValue(); - // Checking if all the transactions corresponding to the snapshot has been flushed by comparing the active - // om's transaction. Since here we are just iterating through snapshots only on disk. There is a - // possibility of missing deletedDirectory entries added from the previous run which has not been flushed. - if (currSnapInfo == null || currSnapInfo.getDeepCleanedDeletedDir() || - !OmSnapshotManager.areSnapshotChangesFlushedToDB(metadataManager, currSnapInfo.getTableKey())) { - continue; - } - - ReferenceCounted rcPrevOmSnapshot = null; - ReferenceCounted rcPrevToPrevOmSnapshot; - try { - long volumeId = metadataManager - .getVolumeId(currSnapInfo.getVolumeName()); - // Get bucketInfo for the snapshot bucket to get bucket layout. - String dbBucketKey = metadataManager - .getBucketKey(currSnapInfo.getVolumeName(), - currSnapInfo.getBucketName()); - OmBucketInfo bucketInfo = metadataManager - .getBucketTable().get(dbBucketKey); - - if (bucketInfo == null) { - throw new IllegalStateException("Bucket " + "/" + - currSnapInfo.getVolumeName() + "/" + currSnapInfo - .getBucketName() + - " is not found. BucketInfo should not be " + - "null for snapshotted bucket. The OM is in " + - "unexpected state."); - } - - SnapshotInfo previousSnapshot = getPreviousSnapshot( - currSnapInfo, snapChainManager, omSnapshotManager); - final SnapshotInfo previousToPrevSnapshot; - - final Table previousKeyTable; - final Table previousDirTable; - final Table prevRenamedTable; - - final Table previousToPrevKeyTable; - - if (previousSnapshot != null) { - rcPrevOmSnapshot = omSnapshotManager.getSnapshot( - previousSnapshot.getVolumeName(), - previousSnapshot.getBucketName(), - previousSnapshot.getName()); - OmSnapshot omPreviousSnapshot = rcPrevOmSnapshot.get(); - - previousKeyTable = omPreviousSnapshot.getMetadataManager().getKeyTable(bucketInfo.getBucketLayout()); - prevRenamedTable = omPreviousSnapshot.getMetadataManager().getSnapshotRenamedTable(); - previousDirTable = omPreviousSnapshot.getMetadataManager().getDirectoryTable(); - previousToPrevSnapshot = getPreviousSnapshot(previousSnapshot, snapChainManager, omSnapshotManager); - } else { - previousKeyTable = null; - previousDirTable = null; - prevRenamedTable = null; - previousToPrevSnapshot = null; - } - - - if (previousToPrevSnapshot != null) { - rcPrevToPrevOmSnapshot = omSnapshotManager.getSnapshot( - previousToPrevSnapshot.getVolumeName(), - previousToPrevSnapshot.getBucketName(), - previousToPrevSnapshot.getName()); - OmSnapshot omPreviousToPrevSnapshot = rcPrevToPrevOmSnapshot.get(); - - previousToPrevKeyTable = omPreviousToPrevSnapshot - .getMetadataManager() - .getKeyTable(bucketInfo.getBucketLayout()); - } else { - previousToPrevKeyTable = null; - } - - String dbBucketKeyForDir = getOzonePathKeyForFso(metadataManager, - currSnapInfo.getVolumeName(), currSnapInfo.getBucketName()); - try (ReferenceCounted rcCurrOmSnapshot = omSnapshotManager.getSnapshot( - currSnapInfo.getVolumeName(), - currSnapInfo.getBucketName(), - currSnapInfo.getName())) { - - OmSnapshot currOmSnapshot = rcCurrOmSnapshot.get(); - Table snapDeletedDirTable = - currOmSnapshot.getMetadataManager().getDeletedDirTable(); - Table snapRenameTable = - currOmSnapshot.getMetadataManager().getSnapshotRenamedTable(); - - try (TableIterator> deletedDirIterator = snapDeletedDirTable - .iterator(dbBucketKeyForDir)) { - long startTime = Time.monotonicNow(); - List purgePathRequestList = new ArrayList<>(); - List> allSubDirList = new ArrayList<>(); - - CheckExceptionOperation checkExceptionOperationOnFile = - deletedKeyInfo -> deleteKeyIfNotReferencedInPreviousSnapshot(volumeId, bucketInfo, deletedKeyInfo, - previousSnapshot, previousToPrevSnapshot, snapRenameTable, prevRenamedTable, - previousKeyTable, previousToPrevKeyTable); - - while (deletedDirIterator.hasNext()) { - Table.KeyValue deletedDirInfo = - deletedDirIterator.next(); - - HddsProtos.KeyValue.Builder renamedEntry = HddsProtos.KeyValue.newBuilder(); - - // Check if the directory is reclaimable. If it is not we cannot delete the directory. - boolean isDirReclaimable = isDirReclaimable(deletedDirInfo, previousDirTable, snapRenameTable, - renamedEntry); - Optional request = prepareDeleteDirRequest( - remainNum.get(), deletedDirInfo.getValue(), deletedDirInfo.getKey(), - allSubDirList, getOzoneManager().getKeyManager(), renamedEntry.hasKey() ? - renamedEntry.getKey() : null, isDirReclaimable, checkExceptionOperationOnFile); - if (!request.isPresent()) { - continue; - } - if (isBufferLimitCrossed(ratisByteLimit, consumedSize.get(), - request.get().getSerializedSize())) { - if (purgePathRequestList.size() != 0) { - // if message buffer reaches max limit, avoid sending further - remainNum.set(0); - } - // if directory itself is having a lot of keys / files, - // reduce capacity to minimum level - remainNum.set(MIN_ERR_LIMIT_PER_TASK); - request = prepareDeleteDirRequest( - remainNum.get(), deletedDirInfo.getValue(), deletedDirInfo.getKey(), - allSubDirList, getOzoneManager().getKeyManager(), renamedEntry.getKey(), isDirReclaimable, - checkExceptionOperationOnFile); - } - if (request.isPresent()) { - OzoneManagerProtocolProtos.PurgePathRequest requestVal = request.get(); - consumedSize.addAndGet(requestVal.getSerializedSize()); - purgePathRequestList.add(requestVal); - remainNum.addAndGet(-1 * requestVal.getDeletedSubFilesCount()); - remainNum.addAndGet(-1 * requestVal.getMarkDeletedSubDirsCount()); - // Count up the purgeDeletedDir, subDirs and subFiles - if (requestVal.hasDeletedDir() && !requestVal.getDeletedDir().isEmpty()) { - dirNum.incrementAndGet(); - } - subDirNum.addAndGet(requestVal.getMarkDeletedSubDirsCount()); - subFileNum.addAndGet(requestVal.getDeletedSubFilesCount()); - } - } - - boolean retVal = optimizeDirDeletesAndSubmitRequest(remainNum.get(), dirNum.get(), subDirNum.get(), - subFileNum.get(), allSubDirList, purgePathRequestList, - currSnapInfo.getTableKey(), startTime, 0, getOzoneManager().getKeyManager(), - (deletedDir) -> expandDirectoryAndPurgeIfDirNotReferenced(deletedDir, previousDirTable, - snapRenameTable), checkExceptionOperationOnFile).getRight() - .map(OzoneManagerProtocolProtos.OMResponse::getSuccess) - .orElse(true); - - List setSnapshotPropertyRequests = new ArrayList<>(); - if (retVal && remainNum.get() == keyLimitPerSnapshot) { - if (previousSnapshot != null) { - setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(previousSnapshot.getTableKey())); - } - setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir( - currSnapInfo.getTableKey())); - submitSetSnapshotRequest(setSnapshotPropertyRequests); - } - } - } - } finally { - IOUtils.closeQuietly(rcPrevOmSnapshot); - } - } - } catch (IOException ex) { - LOG.error("Error while running directory deep clean on snapshots." + - " Will retry at next run.", ex); - } - return BackgroundTaskResult.EmptyTaskResult.newResult(); - } - } - - - private SetSnapshotPropertyRequest getSetSnapshotRequestUpdatingExclusiveSize(String prevSnapshotKeyTable) { - SnapshotSize snapshotSize = SnapshotSize.newBuilder() - .setExclusiveSize( - exclusiveSizeMap.getOrDefault(prevSnapshotKeyTable, 0L)) - .setExclusiveReplicatedSize( - exclusiveReplicatedSizeMap.getOrDefault( - prevSnapshotKeyTable, 0L)) - .build(); - exclusiveSizeMap.remove(prevSnapshotKeyTable); - exclusiveReplicatedSizeMap.remove(prevSnapshotKeyTable); - - return SetSnapshotPropertyRequest.newBuilder() - .setSnapshotKey(prevSnapshotKeyTable) - .setSnapshotSize(snapshotSize) - .build(); - } - - private SetSnapshotPropertyRequest getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir(String snapshotKeyTable) { - return SetSnapshotPropertyRequest.newBuilder() - .setSnapshotKey(snapshotKeyTable) - .setDeepCleanedDeletedDir(true) - .build(); - } - - private void submitSetSnapshotRequest(List setSnapshotPropertyRequests) { - ClientId clientId = ClientId.randomId(); - OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) - .addAllSetSnapshotPropertyRequests(setSnapshotPropertyRequests) - .setClientId(clientId.toString()) - .build(); - submitRequest(omRequest, clientId); - } - - public void submitRequest(OMRequest omRequest, ClientId clientId) { - try { - if (isRatisEnabled()) { - OzoneManagerRatisServer server = - getOzoneManager().getOmRatisServer(); - - RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder() - .setClientId(clientId) - .setServerId(server.getRaftPeerId()) - .setGroupId(server.getRaftGroupId()) - .setCallId(getRunCount().get()) - .setMessage(Message.valueOf( - OMRatisHelper.convertRequestToByteString(omRequest))) - .setType(RaftClientRequest.writeRequestType()) - .build(); - - server.submitRequest(omRequest, raftClientRequest); - } else { - getOzoneManager().getOmServerProtocol() - .submitRequest(null, omRequest); - } - } catch (ServiceException e) { - LOG.error("Snapshot deep cleaning request failed. " + - "Will retry at next run.", e); - } - } - - /** - * Stack node data for directory deep clean for snapshot. - */ - private static class StackNode { - private String dirKey; - private OmKeyInfo dirValue; - private String subDirSeek; - - public String getDirKey() { - return dirKey; - } - - public void setDirKey(String dirKey) { - this.dirKey = dirKey; - } - - public OmKeyInfo getDirValue() { - return dirValue; - } - - public void setDirValue(OmKeyInfo dirValue) { - this.dirValue = dirValue; - } - - public String getSubDirSeek() { - return subDirSeek; - } - - public void setSubDirSeek(String subDirSeek) { - this.subDirSeek = subDirSeek; - } - - @Override - public String toString() { - return "StackNode{" + - "dirKey='" + dirKey + '\'' + - ", dirObjectId=" + dirValue.getObjectID() + - ", subDirSeek='" + subDirSeek + '\'' + - '}'; - } - } -} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index 50b23d265f86..f24d979ef71d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -92,10 +92,11 @@ public static SnapshotInfo getSnapshotInfo(final OzoneManager ozoneManager, return snapshotInfo; } - public static SnapshotInfo getSnapshotInfo(SnapshotChainManager chainManager, UUID snapshotId, - OmSnapshotManager omSnapshotManager) throws IOException { + public static SnapshotInfo getSnapshotInfo(OzoneManager ozoneManager, + SnapshotChainManager chainManager, + UUID snapshotId) throws IOException { String tableKey = chainManager.getTableKey(snapshotId); - return omSnapshotManager.getSnapshotInfo(tableKey); + return SnapshotUtils.getSnapshotInfo(ozoneManager, tableKey); } public static void dropColumnFamilyHandle( @@ -161,9 +162,9 @@ public static void setTransactionInfoInSnapshot(SnapshotInfo snapshot, TermIndex /** * Get the next in the snapshot chain. */ - public static SnapshotInfo getNextSnapshot(SnapshotInfo snapInfo, + public static SnapshotInfo getNextSnapshot(OzoneManager ozoneManager, SnapshotChainManager chainManager, - OmSnapshotManager omSnapshotManager) + SnapshotInfo snapInfo) throws IOException { // If the snapshot is deleted in the previous run, then the in-memory // SnapshotChainManager might throw NoSuchElementException as the snapshot @@ -175,7 +176,7 @@ public static SnapshotInfo getNextSnapshot(SnapshotInfo snapInfo, if (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId())) { UUID nextPathSnapshot = chainManager.nextPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); - return getSnapshotInfo(chainManager, nextPathSnapshot, omSnapshotManager); + return getSnapshotInfo(ozoneManager, chainManager, nextPathSnapshot); } } catch (NoSuchElementException ex) { LOG.error("The snapshot {} is not longer in snapshot chain, It " + @@ -188,12 +189,12 @@ public static SnapshotInfo getNextSnapshot(SnapshotInfo snapInfo, /** * Get the previous in the snapshot chain. */ - public static SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo, + public static SnapshotInfo getPreviousSnapshot(OzoneManager ozoneManager, SnapshotChainManager chainManager, - OmSnapshotManager omSnapshotManager) + SnapshotInfo snapInfo) throws IOException { UUID previousSnapshotId = getPreviousSnapshotId(snapInfo, chainManager); - return previousSnapshotId == null ? null : getSnapshotInfo(chainManager, previousSnapshotId, omSnapshotManager); + return previousSnapshotId == null ? null : getSnapshotInfo(ozoneManager, chainManager, previousSnapshotId); } /** @@ -288,33 +289,12 @@ public static String getOzonePathKeyForFso(OMMetadataManager metadataManager, } public static SnapshotInfo getLatestSnapshotInfo(String volumeName, String bucketName, - SnapshotChainManager snapshotChainManager, - OmSnapshotManager snapshotManager) throws IOException { + OzoneManager ozoneManager, + SnapshotChainManager snapshotChainManager) throws IOException { Optional latestPathSnapshot = Optional.ofNullable( getLatestSnapshotId(volumeName, bucketName, snapshotChainManager)); return latestPathSnapshot.isPresent() ? - getSnapshotInfo(snapshotChainManager, latestPathSnapshot.get(), snapshotManager) : null; - } - - /** - * Get the latest OmSnapshot for a snapshot path. If snapshot is not active should return null if isActive argument - * is true. - */ - public static ReferenceCounted getLatestSnapshot(String volumeName, String bucketName, - SnapshotChainManager snapshotChainManager, - OmSnapshotManager snapshotManager, boolean isActive) - throws IOException { - Optional snapshotInfo = Optional.ofNullable( - SnapshotUtils.getLatestSnapshotInfo(volumeName, bucketName, snapshotChainManager, snapshotManager)); - if (isActive && snapshotInfo.map(si -> si.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) - .orElse(false)) { - return null; - } - Optional> rcOmSnapshot = snapshotInfo.isPresent() ? - Optional.ofNullable(snapshotManager.getSnapshot(volumeName, bucketName, snapshotInfo.get().getName())) : - Optional.empty(); - - return rcOmSnapshot.orElse(null); + getSnapshotInfo(ozoneManager, snapshotChainManager, latestPathSnapshot.get()) : null; } public static UUID getLatestSnapshotId(String volumeName, String bucketName, From e030d90159085ff01f5c9900e2515dd5f329dd50 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 6 Sep 2024 18:56:33 -0700 Subject: [PATCH 04/22] Push interim merge in progress Change-Id: I019493535aaebdc779f3431dbc88d8cf0f1c0c1d --- .../src/main/proto/OmClientProtocol.proto | 15 +- .../apache/hadoop/ozone/om/KeyManager.java | 15 ++ .../hadoop/ozone/om/KeyManagerImpl.java | 55 ++++++ .../key/OMDirectoriesPurgeRequestWithFSO.java | 40 ++-- .../om/request/key/OMKeyPurgeRequest.java | 37 ++-- .../snapshot/OMSnapshotCreateRequest.java | 2 +- .../snapshot/OMSnapshotPurgeRequest.java | 33 ++-- .../OMDirectoriesPurgeResponseWithFSO.java | 4 + .../om/response/key/OMKeyPurgeResponse.java | 2 +- .../service/AbstractKeyDeletingService.java | 173 +++++++++++------- .../om/service/DirectoryDeletingService.java | 137 +++++++------- .../ozone/om/service/KeyDeletingService.java | 92 ++++++---- .../om/service/SnapshotDeletingService.java | 157 ++++++---------- .../om/snapshot/SnapshotDiffManager.java | 6 +- .../ozone/om/snapshot/SnapshotUtils.java | 38 ++-- .../om/snapshot/TestSnapshotDiffManager.java | 3 +- 16 files changed, 453 insertions(+), 356 deletions(-) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 59ba3b61b08f..b85dff849010 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -296,7 +296,6 @@ message OMRequest { optional GetQuotaRepairStatusRequest GetQuotaRepairStatusRequest = 133; optional StartQuotaRepairRequest StartQuotaRepairRequest = 134; repeated SetSnapshotPropertyRequest SetSnapshotPropertyRequests = 135; - optional SnapshotPurgeAndMoveRequest SnapshotPurgeAndMoveRequest = 136; } message OMResponse { @@ -1374,8 +1373,6 @@ message DeletedKeys { required string volumeName = 1; required string bucketName = 2; repeated string keys = 3; - optional hadoop.hdds.UUID expectedPathPreviousSnapshotUUID = 4; - repeated string renamedKeys = 5; } message PurgeKeysRequest { @@ -1383,6 +1380,9 @@ message PurgeKeysRequest { // if set, will purge keys in a snapshot DB instead of active DB optional string snapshotTableKey = 2; repeated SnapshotMoveKeyInfos keysToUpdate = 3; + optional hadoop.hdds.UUID expectedPreviousSnapshotID = 4; + repeated string renamedKeys = 5; + } message PurgeKeysResponse { @@ -1405,6 +1405,7 @@ message PurgePathsResponse { message PurgeDirectoriesRequest { repeated PurgePathRequest deletedPath = 1; optional string snapshotTableKey = 2; + optional hadoop.hdds.UUID expectedPreviousSnapshotID = 3; } message PurgeDirectoriesResponse { @@ -1417,9 +1418,6 @@ message PurgePathRequest { optional string deletedDir = 3; repeated KeyInfo deletedSubFiles = 4; repeated KeyInfo markDeletedSubDirs = 5; - optional hadoop.hdds.UUID expectedPathPreviousSnapshotUUID = 6; - optional string volumeName = 7; - optional string bucketName = 8; } message DeleteOpenKeysRequest { @@ -2000,11 +1998,6 @@ message SnapshotPurgeRequest { repeated string updatedSnapshotDBKey = 2 [deprecated = true]; } -message SnapshotPurgeAndMoveRequest { - required hadoop.hdds.UUID snapshotID = 1; - optional hadoop.hdds.UUID expectedNextSnapshotID = 2; -} - message SetSnapshotPropertyRequest { optional SnapshotProperty snapshotProperty = 1 [deprecated = true]; optional string snapshotKey = 2; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 12ac39082f11..986fd9beef97 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; import org.apache.hadoop.ozone.om.fs.OzoneManagerFS; import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; @@ -152,6 +153,20 @@ List> getRenamesKeyEntries( CheckExceptionOperation, Boolean, IOException> filter, int count) throws IOException; + /** + * Returns a list rename entries from the snapshotRenamedTable. + * + * @param count max number of keys to return. + * @param filter filter to apply on the entries. + * @return a Pair of list of {@link org.apache.hadoop.hdds.utils.db.Table.KeyValue} representing the keys in the + * underlying metadataManager. + * @throws IOException + */ + List>> getDeletedKeyEntries( + String volume, String bucket, String startKey, + CheckExceptionOperation, Boolean, IOException> filter, int count) + throws IOException; + /** * Returns the names of up to {@code count} open keys whose age is * greater than or equal to {@code expireThreshold}. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 366d790313ba..2395722a9d9e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -732,6 +732,61 @@ public List> getRenamesKeyEntries( return renamedEntries; } + @Override + public List>> getDeletedKeyEntries( + String volume, String bucket, String startKey, + CheckExceptionOperation, Boolean, IOException> filter, + int count) throws IOException { + // Bucket prefix would be empty if volume is empty i.e. either null or "". + Optional bucketPrefix = Optional.ofNullable(volume).map(vol -> vol.isEmpty() ? null : vol) + .map(vol -> metadataManager.getBucketKeyPrefix(vol, bucket)); + List>> deletedKeyEntries = new ArrayList<>(count); + try (TableIterator> + delKeyIter = metadataManager.getDeletedTable().iterator(bucketPrefix.orElse(""))) { + + /* Seeking to the start key if it not null. The next key picked up would be ensured to start with the bucket + prefix, {@link org.apache.hadoop.hdds.utils.db.Table#iterator(bucketPrefix)} would ensure this. + */ + if (startKey != null) { + delKeyIter.seek(startKey); + } + int currentCount = 0; + while (delKeyIter.hasNext() && currentCount < count) { + Table.KeyValue kv = delKeyIter.next(); + if (kv != null && filter.apply(kv)) { + deletedKeyEntries.add(Table.newKeyValue(kv.getKey(), kv.getValue().cloneOmKeyInfoList())); + + for (OmKeyInfo info : infoList.cloneOmKeyInfoList()) { + + // Skip the key if the filter doesn't allow the file to be deleted. + if (filter.apply(Table.newKeyValue(kv.getKey(), info))) { + List blockIDS = info.getKeyLocationVersions().stream() + .flatMap(versionLocations -> versionLocations.getLocationList().stream() + .map(b -> new BlockID(b.getContainerID(), b.getLocalID()))).collect(Collectors.toList()); + BlockGroup keyBlocks = BlockGroup.newBuilder().setKeyName(kv.getKey()) + .addAllBlockIDs(blockIDS).build(); + blockGroupList.add(keyBlocks); + currentCount++; + } else { + notReclaimableKeyInfo.addOmKeyInfo(info); + } + } + + List notReclaimableKeyInfoList = notReclaimableKeyInfo.getOmKeyInfoList(); + + // If all the versions are not reclaimable, then modify key by just purging the key that can be purged. + if (notReclaimableKeyInfoList.size() > 0 && + notReclaimableKeyInfoList.size() != infoList.getOmKeyInfoList().size()) { + keysToModify.put(kv.getKey(), notReclaimableKeyInfo); + } + keyBlocksList.addAll(blockGroupList); + } + } + nextPageStartKey = delKeyIter.hasNext() ? delKeyIter.next().getKey() : null; + } + return new PendingKeysDeletion(keyBlocksList, keysToModify, nextPageStartKey); + } + @Override public ExpiredOpenKeys getExpiredOpenKeys(Duration expireThreshold, int count, BucketLayout bucketLayout, Duration leaseThreshold) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index cba44386c56a..b6fd4b367343 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -82,31 +82,22 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn Map, OmBucketInfo> volBucketInfoMap = new HashMap<>(); OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); Map openKeyInfoMap = new HashMap<>(); - + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); OMMetrics omMetrics = ozoneManager.getMetrics(); try { fromSnapshotInfo = fromSnapshot != null ? SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot) : null; - - Map, Boolean> previousSnapshotValidatedMap = Maps.newHashMap(); - + // Validating previous snapshot since while purging deletes, a snapshot create request could make this purge + // directory request invalid on AOS since the deletedDirectory would be in the newly created snapshot. Adding + // subdirectories could lead to not being able to reclaim sub-files and subdirectories since the + // file/directory would be present in the newly created snapshot. + // Validating previous snapshot can ensure the chain hasn't changed. + UUID expectedPreviousSnapshotId = purgeDirsRequest.hasExpectedPreviousSnapshotID() + ? fromProtobuf(purgeDirsRequest.getExpectedPreviousSnapshotID()) : null; + validatePreviousSnapshotId(fromSnapshotInfo, omMetadataManager.getSnapshotChainManager(), + expectedPreviousSnapshotId); for (OzoneManagerProtocolProtos.PurgePathRequest path : purgeRequests) { - // Validating previous snapshot since while purging deletes, a snapshot create request could make this purge - // directory request invalid on AOS since the deletedDirectory would be in the newly created snapshot. Adding - // subdirectories could lead to not being able to reclaim sub-files and subdirectories since the - // file/directory would be present in the newly created snapshot. - // Validating previous snapshot can ensure the chain hasn't changed. - boolean isPreviousSnapshotValid = previousSnapshotValidatedMap.computeIfAbsent(Pair.of(path.getVolumeName(), - path.getBucketName()), - (volumeBucketPair) -> validatePreviousSnapshotId(fromSnapshotInfo, volumeBucketPair.getLeft(), - volumeBucketPair.getRight(), omMetadataManager.getSnapshotChainManager(), - path.hasExpectedPathPreviousSnapshotUUID() - ? fromProtobuf(path.getExpectedPathPreviousSnapshotUUID()) : null)); - - if (!isPreviousSnapshotValid) { - continue; - } - validPurgePathRequests.add(path); for (OzoneManagerProtocolProtos.KeyInfo key : path.getMarkDeletedSubDirsList()) { @@ -184,10 +175,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn CacheValue.get(termIndex.getIndex(), fromSnapshotInfo)); } } catch (IOException ex) { - // Case of IOException for fromProtobuf will not happen - // as this is created and send within OM - // only case of upgrade where compatibility is broken can have - throw new IllegalStateException(ex); + // Handling IOException thrown when snapshot is not found or previous snapshot validation fails. + LOG.error("Error occured while performing OMDirectoriesPurge. ", ex); + return new OMDirectoriesPurgeResponseWithFSO(createErrorOMResponse(omResponse, ex)); } finally { lockSet.stream().forEach(e -> omMetadataManager.getLock() .releaseWriteLock(BUCKET_LOCK, e.getKey(), @@ -197,8 +187,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn entry.setValue(entry.getValue().copyObject()); } } - OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( - getOmRequest()); OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO( omResponse.build(), validPurgePathRequests, ozoneManager.isRatisEnabled(), getBucketLayout(), volBucketInfoMap, fromSnapshotInfo, openKeyInfoMap); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java index cd7862e5362f..61b7c66f1f7e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; +import java.util.UUID; import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.validatePreviousSnapshotId; @@ -66,8 +67,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn PurgeKeysRequest purgeKeysRequest = getOmRequest().getPurgeKeysRequest(); List bucketDeletedKeysList = purgeKeysRequest.getDeletedKeysList(); List keysToUpdateList = purgeKeysRequest.getKeysToUpdateList(); - String fromSnapshot = purgeKeysRequest.hasSnapshotTableKey() ? - purgeKeysRequest.getSnapshotTableKey() : null; + String fromSnapshot = purgeKeysRequest.hasSnapshotTableKey() ? purgeKeysRequest.getSnapshotTableKey() : null; OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( @@ -75,30 +75,31 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn final SnapshotInfo fromSnapshotInfo; try { - fromSnapshotInfo = fromSnapshot != null ? null : SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot); + fromSnapshotInfo = fromSnapshot == null ? null : SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot); } catch (IOException ex) { return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, ex)); } List keysToBePurgedList = new ArrayList<>(); - List renamedKeysToBePurged = new ArrayList<>(); - Map, Boolean> previousSnapshotValidatedMap = Maps.newHashMap(); + // Validating previous snapshot since while purging rename keys, a snapshot create request could make this purge + // rename entry invalid on AOS since the key could be now present on the newly created snapshot since a rename + // request could have come through after creation of a snapshot. The purged deletedKey would not be even + // present. + UUID expectedPreviousSnapshotId = purgeKeysRequest.hasExpectedPreviousSnapshotID() + ? fromProtobuf(purgeKeysRequest.getExpectedPreviousSnapshotID()) : null; + try { + validatePreviousSnapshotId(fromSnapshotInfo, omMetadataManager.getSnapshotChainManager(), + expectedPreviousSnapshotId); + } catch (IOException e) { + LOG.error("Previous snapshot validation failed.", e); + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, e)); + } + + + List renamedKeysToBePurged = new ArrayList<>(purgeKeysRequest.getRenamedKeysList()); for (DeletedKeys bucketWithDeleteKeys : bucketDeletedKeysList) { - // Validating previous snapshot since while purging rename keys, a snapshot create request could make this purge - // rename entry invalid on AOS since the key could be now present on the newly created snapshot since a rename - // request could have come through after creation of a snapshot. The purged deletedKey would not be even - // present - boolean isPreviousSnapshotValid = previousSnapshotValidatedMap.computeIfAbsent( - Pair.of(bucketWithDeleteKeys.getVolumeName(), bucketWithDeleteKeys.getBucketName()), - (volumeBucketPair) -> validatePreviousSnapshotId(fromSnapshotInfo, volumeBucketPair.getLeft(), - volumeBucketPair.getRight(), omMetadataManager.getSnapshotChainManager(), - bucketWithDeleteKeys.hasExpectedPathPreviousSnapshotUUID() - ? fromProtobuf(bucketWithDeleteKeys.getExpectedPathPreviousSnapshotUUID()) : null)); - if (isPreviousSnapshotValid) { - renamedKeysToBePurged.addAll(bucketWithDeleteKeys.getRenamedKeysList()); keysToBePurgedList.addAll(bucketWithDeleteKeys.getKeysList()); - } } if (keysToBePurgedList.isEmpty() && renamedKeysToBePurged.isEmpty()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index a140fdada763..966819b05f0d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -186,7 +186,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn // TODO: [SNAPSHOT] Assign actual data size once we have the // pre-replicated key size counter in OmBucketInfo. snapshotInfo.setReferencedSize(estimateBucketDataSize(omBucketInfo)); - SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); addSnapshotInfoToSnapshotChainAndCache(ozoneManager, omMetadataManager, termIndex.getIndex()); @@ -269,6 +268,7 @@ private void addSnapshotInfoToSnapshotChainAndCache( snapshotInfo.setPathPreviousSnapshotId(latestPathSnapshot); snapshotInfo.setGlobalPreviousSnapshotId(latestGlobalSnapshot); + Optional previousSnapshot = Optional.ofNullable(SnapshotUtils.getLatestSnapshotInfo( snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), ozoneManager, snapshotChainManager)); Optional previousPrevSnapshot = previousSnapshot.isPresent() ? diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index b298a5b8a97c..03f3f6fa42b2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -110,15 +110,15 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); - Optional previousSnapshot = Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(fromSnapshot, - snapshotChainManager, omSnapshotManager)); + Optional previousSnapshot = Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(ozoneManager, + snapshotChainManager, fromSnapshot)); Optional previousPrevSnapshot = previousSnapshot.isPresent() ? - Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(previousSnapshot.get(), snapshotChainManager, - omSnapshotManager)) : Optional.empty(); + Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(ozoneManager, snapshotChainManager, + previousSnapshot.get())) : Optional.empty(); // Step 1: Reset the deep clean flag for the next active snapshot if and only if the last 2 snapshots in the // chain are active, otherwise set it to prevent deep cleaning from running till the deleted snapshots don't // get purged. - updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, trxnLogIndex, + updateNextSnapshotInfoFields(nextSnapshot, omMetadataManager, trxnLogIndex, !(previousSnapshot.map(SnapshotInfo::getSnapshotStatus).orElse(SNAPSHOT_ACTIVE) == SNAPSHOT_ACTIVE && previousPrevSnapshot.map(SnapshotInfo::getSnapshotStatus).orElse(SNAPSHOT_ACTIVE) == SNAPSHOT_ACTIVE)); @@ -130,6 +130,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn .addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), CacheValue.get(trxnLogIndex)); } + for (SnapshotInfo snapshotInfo : updatedSnapshotInfos.values()) { + SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapshotInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), snapshotInfo)); + } + omClientResponse = new OMSnapshotPurgeResponse(omResponse.build(), snapshotDbKeys, updatedSnapshotInfos); omMetrics.incNumSnapshotPurges(); @@ -145,20 +151,21 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn return omClientResponse; } - private void updateSnapshotInfoAndCache(SnapshotInfo snapInfo, OmMetadataManagerImpl omMetadataManager, - long trxnLogIndex, boolean flagValue) throws IOException { - if (snapInfo != null) { + private void updateNextSnapshotInfoFields(SnapshotInfo nextSnapshot, OmMetadataManagerImpl omMetadataManager, + long trxnLogIndex, boolean deepCleanFlagValue) throws IOException { + if (nextSnapshot != null) { // Setting next snapshot deep clean to false, Since the // current snapshot is deleted. We can potentially // reclaim more keys in the next snapshot. - snapInfo.setDeepClean(flagValue); - snapInfo.setDeepCleanedDeletedDir(flagValue); + nextSnapshot.setDeepClean(deepCleanFlagValue); + nextSnapshot.setDeepCleanedDeletedDir(deepCleanFlagValue); // Update table cache first - omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapInfo.getTableKey()), - CacheValue.get(trxnLogIndex, snapInfo)); - updatedSnapshotInfos.put(snapInfo.getTableKey(), snapInfo); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextSnapshot.getTableKey()), + CacheValue.get(trxnLogIndex, nextSnapshot)); + updatedSnapshotInfos.put(nextSnapshot.getTableKey(), nextSnapshot); } + } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 937000273643..9ecdf9f25258 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -78,6 +78,10 @@ public OMDirectoriesPurgeResponseWithFSO(@Nonnull OMResponse omResponse, this.openKeyInfoMap = openKeyInfoMap; } + public OMDirectoriesPurgeResponseWithFSO(OMResponse omResponse) { + super(omResponse); + } + @Override public void addToDBBatch(OMMetadataManager metadataManager, BatchOperation batchOp) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java index 89bc2dfd2e26..eac0502bb826 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java @@ -122,7 +122,7 @@ private void processKeys(BatchOperation batchOp, } // Delete rename entries. for (String key : renamedList) { - metadataManager.getSnapshotInfoTable().deleteWithBatch(batchOp, key); + metadataManager.getSnapshotRenamedTable().deleteWithBatch(batchOp, key); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index 05e8a0935b33..8e5a86299b4d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ServiceException; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.utils.BackgroundService; @@ -40,6 +41,8 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; @@ -65,14 +68,12 @@ import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Queue; import java.util.UUID; @@ -82,8 +83,6 @@ import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; -import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; /** * Abstracts common code from KeyDeletingService and DirectoryDeletingService @@ -94,7 +93,7 @@ public abstract class AbstractKeyDeletingService extends BackgroundService private final OzoneManager ozoneManager; private final ScmBlockLocationProtocol scmClient; - private static ClientId clientId = ClientId.randomId(); + protected static ClientId clientId = ClientId.randomId(); private final AtomicLong deletedDirsCount; private final AtomicLong movedDirsCount; private final AtomicLong movedFilesCount; @@ -115,26 +114,65 @@ public AbstractKeyDeletingService(String serviceName, long interval, this.runCount = new AtomicLong(0); } - //Removes deleted file from AOS and moves if it is not referenced in the previous snapshot. - // Returns True if it can be deleted and false if it cannot be deleted. - private boolean deleteKeyIfNotReferencedInPreviousSnapshot(long volumeId, OmBucketInfo bucketInfo, - OmKeyInfo deletedKeyInfo, SnapshotInfo prevSnapshotInfo, SnapshotInfo prevPrevSnapshotInfo, - Table renamedTable, Table prevRenamedTable, - Table previousKeyTable, Table previousPrevKeyTable, - Map exclusiveSizeMap, Map exclusiveReplicatedSizeMap) throws IOException { - if (isKeyReclaimable(previousKeyTable, renamedTable, deletedKeyInfo, bucketInfo, volumeId, null)) { + // TODO: Move this util class. + public static boolean isBlockLocationInfoSame(OmKeyInfo prevKeyInfo, + OmKeyInfo deletedKeyInfo) { + + if (prevKeyInfo == null && deletedKeyInfo == null) { + LOG.debug("Both prevKeyInfo and deletedKeyInfo are null."); return true; } - calculateExclusiveSize(prevSnapshotInfo, prevPrevSnapshotInfo, deletedKeyInfo, bucketInfo, volumeId, - renamedTable, previousKeyTable, prevRenamedTable, previousPrevKeyTable, exclusiveSizeMap, - exclusiveReplicatedSizeMap); - return false; + if (prevKeyInfo == null || deletedKeyInfo == null) { + LOG.debug("prevKeyInfo: '{}' or deletedKeyInfo: '{}' is null.", + prevKeyInfo, deletedKeyInfo); + return false; + } + // For hsync, Though the blockLocationInfo of a key may not be same + // at the time of snapshot and key deletion as blocks can be appended. + // If the objectId is same then the key is same. + if (prevKeyInfo.isHsync() && deletedKeyInfo.isHsync()) { + return true; + } + + if (prevKeyInfo.getKeyLocationVersions().size() != + deletedKeyInfo.getKeyLocationVersions().size()) { + return false; + } + + OmKeyLocationInfoGroup deletedOmKeyLocation = + deletedKeyInfo.getLatestVersionLocations(); + OmKeyLocationInfoGroup prevOmKeyLocation = + prevKeyInfo.getLatestVersionLocations(); + + if (deletedOmKeyLocation == null || prevOmKeyLocation == null) { + return false; + } + + List deletedLocationList = + deletedOmKeyLocation.getLocationList(); + List prevLocationList = + prevOmKeyLocation.getLocationList(); + + if (deletedLocationList.size() != prevLocationList.size()) { + return false; + } + + for (int idx = 0; idx < deletedLocationList.size(); idx++) { + OmKeyLocationInfo deletedLocationInfo = deletedLocationList.get(idx); + OmKeyLocationInfo prevLocationInfo = prevLocationList.get(idx); + if (!deletedLocationInfo.hasSameBlockAs(prevLocationInfo)) { + return false; + } + } + + return true; } protected Pair processKeyDeletes(List keyBlocksList, KeyManager manager, Map keysToModify, - String snapTableKey) throws IOException { + List renameEntries, + String snapTableKey, UUID expectedPreviousSnapshotId) throws IOException { long startTime = Time.monotonicNow(); Pair purgeResult = Pair.of(0, false); @@ -156,7 +194,8 @@ protected Pair processKeyDeletes(List keyBlocksLis if (blockDeletionResults != null) { startTime = Time.monotonicNow(); if (isRatisEnabled()) { - purgeResult = submitPurgeKeysRequest(blockDeletionResults, keysToModify, snapTableKey); + purgeResult = submitPurgeKeysRequest(blockDeletionResults, keysToModify, renameEntries, + snapTableKey, expectedPreviousSnapshotId); } else { // TODO: Once HA and non-HA paths are merged, we should have // only one code path here. Purge keys should go through an @@ -211,9 +250,8 @@ private Pair deleteAllKeys(List result */ private Pair submitPurgeKeysRequest(List results, Map keysToModify, List renameEntriesToBeDeleted, - String snapTableKey) { - Map, Pair, List>> purgeKeysMapPerBucket = - new HashMap<>(); + String snapTableKey, UUID expectedPreviousSnapshotId) { + Map, List> purgeKeysMapPerBucket = new HashMap<>(); // Put all keys to be purged in a list int deletedCount = 0; @@ -246,18 +284,25 @@ private Pair submitPurgeKeysRequest(List, Pair, List>> entry : - purgeKeysMapPerBucket.entrySet()) { + for (Map.Entry, List> entry : purgeKeysMapPerBucket.entrySet()) { Pair volumeBucketPair = entry.getKey(); DeletedKeys deletedKeysInBucket = DeletedKeys.newBuilder() .setVolumeName(volumeBucketPair.getLeft()) .setBucketName(volumeBucketPair.getRight()) - .addAllKeys(entry.getValue().getKey()) - .addAllRenamedKeys(entry.getValue().getValue()) + .addAllKeys(entry.getValue()) .build(); purgeKeysRequest.addDeletedKeys(deletedKeysInBucket); } + // Adding rename entries to be purged. + if (renameEntriesToBeDeleted != null) { + purgeKeysRequest.addAllRenamedKeys(renameEntriesToBeDeleted); + } + List keysToUpdateList = new ArrayList<>(); if (keysToModify != null) { @@ -348,7 +393,7 @@ protected RaftClientRequest createRaftClientRequestForPurge( * Parse Volume and Bucket Name from ObjectKey and add it to given map of * keys to be purged per bucket. */ - private void addToMap(Map, Pair, List>> map, String objectKey) { + private void addToMap(Map, List> map, String objectKey) { // Parse volume and bucket name String[] split = objectKey.split(OM_KEY_PREFIX); Preconditions.assertTrue(split.length >= 3, "Volume and/or Bucket Name " + @@ -358,20 +403,24 @@ private void addToMap(Map, Pair, List> } Pair volumeBucketPair = Pair.of(split[1], split[2]); if (!map.containsKey(volumeBucketPair)) { - map.put(volumeBucketPair, Pair.of(new ArrayList<>(), new ArrayList<>())); + map.put(volumeBucketPair, new ArrayList<>()); } - map.get(volumeBucketPair).getKey().add(objectKey); - + map.get(volumeBucketPair).add(objectKey); } protected OzoneManagerProtocolProtos.OMResponse submitPurgePaths( - List requests, String snapTableKey) { + List requests, String snapTableKey, UUID expectedPreviousSnapshotId) { OzoneManagerProtocolProtos.PurgeDirectoriesRequest.Builder purgeDirRequest = OzoneManagerProtocolProtos.PurgeDirectoriesRequest.newBuilder(); if (snapTableKey != null) { purgeDirRequest.setSnapshotTableKey(snapTableKey); } + + if (expectedPreviousSnapshotId != null) { + purgeDirRequest.setExpectedPreviousSnapshotID(HddsUtils.toProtobuf(expectedPreviousSnapshotId)); + } + purgeDirRequest.addAllDeletedPath(requests); OzoneManagerProtocolProtos.OMRequest omRequest = @@ -398,20 +447,6 @@ protected OzoneManagerProtocolProtos.OMResponse submitPurgePaths( return null; } - protected OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( - final long volumeId, final long bucketId, final List purgeDeletedFiles) { - // Put all keys to be purged in a list - PurgePathRequest.Builder purgePathsRequest = PurgePathRequest.newBuilder(); - purgePathsRequest.setVolumeId(volumeId); - purgePathsRequest.setBucketId(bucketId); - - for (OzoneManagerProtocolProtos.KeyInfo purgeFile : purgeDeletedFiles) { - purgePathsRequest.addDeletedSubFiles(purgeFile); - } - - return purgePathsRequest.build(); - } - private OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( final long volumeId, final long bucketId, @@ -512,7 +547,8 @@ public Pair> optimizeDirD List purgePathRequestList, String snapTableKey, long startTime, int remainingBufLimit, KeyManager keyManager, CheckExceptionOperation, Boolean, IOException> subDirPurgeChecker, - CheckExceptionOperation, Boolean, IOException> fileDeletionChecker) { + CheckExceptionOperation, Boolean, IOException> fileDeletionChecker, + UUID expectedPreviousSnapshotId) { // Optimization to handle delete sub-dir and keys to remove quickly // This case will be useful to handle when depth of directory is high @@ -558,7 +594,7 @@ public Pair> optimizeDirD } OzoneManagerProtocolProtos.OMResponse response = null; if (!purgePathRequestList.isEmpty()) { - response = submitPurgePaths(purgePathRequestList, snapTableKey); + response = submitPurgePaths(purgePathRequestList, snapTableKey, expectedPreviousSnapshotId); } if (dirNum != 0 || subDirNum != 0 || subFileNum != 0) { @@ -681,20 +717,6 @@ protected boolean isBufferLimitCrossed( return cLimit + increment >= maxLimit; } - protected SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo, - SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager) - throws IOException { - SnapshotInfo currSnapInfo = snapInfo; - if (chainManager.hasPreviousPathSnapshot(currSnapInfo.getSnapshotPath(), currSnapInfo.getSnapshotId())) { - - UUID prevPathSnapshot = chainManager.previousPathSnapshot( - currSnapInfo.getSnapshotPath(), currSnapInfo.getSnapshotId()); - String tableKey = chainManager.getTableKey(prevPathSnapshot); - return omSnapshotManager.getSnapshotInfo(tableKey); - } - return null; - } - protected boolean isKeyReclaimable( Table previousKeyTable, Table renamedTable, @@ -1003,8 +1025,7 @@ private List getLastNSnapshotInChain(String volume, String bucket) return snapshotInfos; } - // Initialize the last N snapshots in the chain by acquiring locks. Throw IOException if it fails. - private void initializePreviousSnapshotsFromChain(String volume, String bucket) throws IOException { + private boolean validateExistingLastNSnapshotsInChain(String volume, String bucket) throws IOException { List expectedLastNSnapshotsInChain = getLastNSnapshotInChain(volume, bucket); List expectedSnapshotIds = expectedLastNSnapshotsInChain.stream() .map(snapshotInfo -> snapshotInfo == null ? null : snapshotInfo.getSnapshotId()) @@ -1012,16 +1033,24 @@ private void initializePreviousSnapshotsFromChain(String volume, String bucket) List existingSnapshotIds = previousOmSnapshots.stream() .map(omSnapshotReferenceCounted -> omSnapshotReferenceCounted == null ? null : omSnapshotReferenceCounted.get().getSnapshotID()).collect(Collectors.toList()); - // If snapshotIds don't match then close all snapshots and reopen the previous N snapshots. - if (!expectedSnapshotIds.equals(existingSnapshotIds)) { + return expectedSnapshotIds.equals(existingSnapshotIds); + } + + // Initialize the last N snapshots in the chain by acquiring locks. Throw IOException if it fails. + private void initializePreviousSnapshotsFromChain(String volume, String bucket) throws IOException { + // If existing snapshotIds don't match then close all snapshots and reopen the previous N snapshots. + if (!validateExistingLastNSnapshotsInChain(volume, bucket)) { close(); try { // Acquire lock only on last N-1 snapshot & current snapshot(AOS if it is null). - List lockIds = new ArrayList<>(expectedSnapshotIds.subList(1, existingSnapshotIds.size())); + List expectedLastNSnapshotsInChain = getLastNSnapshotInChain(volume, bucket); + List expectedSnapshotIds = expectedLastNSnapshotsInChain.stream() + .map(snapshotInfo -> snapshotInfo == null ? null : snapshotInfo.getSnapshotId()) + .collect(Collectors.toList()); + List lockIds = new ArrayList<>(expectedSnapshotIds.subList(1, expectedSnapshotIds.size())); lockIds.add(currentSnapshotInfo == null ? null : currentSnapshotInfo.getSnapshotId()); if (snapshotIdLocks.acquireLock(lockIds).isLockAcquired()) { - expectedLastNSnapshotsInChain = getLastNSnapshotInChain(volume, bucket); for (SnapshotInfo snapshotInfo : expectedLastNSnapshotsInChain) { if (snapshotInfo != null) { // For AOS fail operation if any of the previous snapshots are not active. currentSnapshotInfo for @@ -1056,8 +1085,13 @@ private void initializePreviousSnapshotsFromChain(String volume, String bucket) @Override public Boolean apply(Table.KeyValue keyValue) throws IOException { - initializePreviousSnapshotsFromChain(getVolumeName(keyValue), getBucketName(keyValue)); - return isReclaimable(keyValue); + String volume = getVolumeName(keyValue); + String bucket = getBucketName(keyValue); + initializePreviousSnapshotsFromChain(volume, bucket); + boolean isReclaimable = isReclaimable(keyValue); + // This is to ensure the reclamation ran on the same previous snapshot and no change occurred in the chain + // while processing the entry. + return isReclaimable && validateExistingLastNSnapshotsInChain(volume, bucket); } protected abstract String getVolumeName(Table.KeyValue keyValue) throws IOException; @@ -1247,5 +1281,4 @@ protected String getBucketName(Table.KeyValue keyValue) throws I return getMetadataManager().splitRenameKey(keyValue.getKey())[1]; } } - } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index 4fb5fca248c5..a5dda6680dd9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -87,8 +87,6 @@ public class DirectoryDeletingService extends AbstractKeyDeletingService { public static final Logger LOG = LoggerFactory.getLogger(DirectoryDeletingService.class); - private static ClientId clientId = ClientId.randomId(); - // Use only a single thread for DirDeletion. Multiple threads would read // or write to same tables and can send deletion requests for same key // multiple times. @@ -219,78 +217,87 @@ private long processDeletedDirsForStore(SnapshotInfo currentSnapshotInfo, KeyMan OmSnapshotManager omSnapshotManager = getOzoneManager().getOmSnapshotManager(); SnapshotChainManager snapshotChainManager = ((OmMetadataManagerImpl)getOzoneManager().getMetadataManager()) .getSnapshotChainManager(); + // This is to avoid race condition b/w purge request and snapshot chain updation. For AOS taking the global + // snapshotId since AOS could process multiple buckets in one iteration. While using path previous snapshot + // Id for a snapshot since it would process only one bucket. + UUID expectedPreviousSnapshotId = currentSnapshotInfo == null ? + snapshotChainManager.getLatestGlobalSnapshotId() : + SnapshotUtils.getPreviousSnapshotId(currentSnapshotInfo, snapshotChainManager); IOzoneManagerLock lock = getOzoneManager().getMetadataManager().getLock(); - ReclaimableDirFilter reclaimableDirFilter = new ReclaimableDirFilter(omSnapshotManager, snapshotChainManager, + + try(ReclaimableDirFilter reclaimableDirFilter = new ReclaimableDirFilter(omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); - ReclaimableKeyFilter reclaimableSubFileFilter = new ReclaimableKeyFilter(omSnapshotManager, - snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); - long startTime = Time.monotonicNow(); - while (remainNum > 0 && deletedDirsIterator.hasNext()) { - pendingDeletedDirInfo = deletedDirsIterator.next(); - // Always perform listing on AOS. - Optional request = prepareDeleteDirRequest( - remainNum, pendingDeletedDirInfo.getValue(), - pendingDeletedDirInfo.getKey(), allSubDirList, - getOzoneManager().getKeyManager(), reclaimableDirFilter.apply(pendingDeletedDirInfo), - reclaimableSubFileFilter); - - if (request.isPresent() && isBufferLimitCrossed(ratisByteLimit, consumedSize, - request.get().getSerializedSize())) { - if (purgePathRequestList.size() != 0) { - // if message buffer reaches max limit, avoid sending further - remainNum = 0; - break; - } - // if directory itself is having a lot of keys / files, - // reduce capacity to minimum level - remainNum = MIN_ERR_LIMIT_PER_TASK; - request = prepareDeleteDirRequest( + ReclaimableKeyFilter reclaimableSubFileFilter = new ReclaimableKeyFilter(omSnapshotManager, + snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock)) { + long startTime = Time.monotonicNow(); + while (remainNum > 0 && deletedDirsIterator.hasNext()) { + pendingDeletedDirInfo = deletedDirsIterator.next(); + // Always perform listing on AOS. + Optional request = prepareDeleteDirRequest( remainNum, pendingDeletedDirInfo.getValue(), pendingDeletedDirInfo.getKey(), allSubDirList, getOzoneManager().getKeyManager(), reclaimableDirFilter.apply(pendingDeletedDirInfo), reclaimableSubFileFilter); - } - if (!request.isPresent()) { - continue; - } - PurgePathRequest purgePathRequest = request.get(); - consumedSize += purgePathRequest.getSerializedSize(); - purgePathRequestList.add(purgePathRequest); - remainNum = remainNum - purgePathRequest.getDeletedSubFilesCount(); - remainNum = remainNum - purgePathRequest.getMarkDeletedSubDirsCount(); - // Count up the purgeDeletedDir, subDirs and subFiles - if (purgePathRequest.getDeletedDir() != null - && !purgePathRequest.getDeletedDir().isEmpty()) { - dirNum++; - } - subDirNum += purgePathRequest.getMarkDeletedSubDirsCount(); - subFileNum += purgePathRequest.getDeletedSubFilesCount(); - } - Pair> retVal = optimizeDirDeletesAndSubmitRequest( - remainNum, dirNum, subDirNum, subFileNum, - allSubDirList, purgePathRequestList, snapshotTableKey, startTime, - ratisByteLimit - consumedSize, - getOzoneManager().getKeyManager(), reclaimableDirFilter, reclaimableSubFileFilter); - remainNum = retVal.getKey(); - - if (remainNum == initialRemainNum && - retVal.getValue().map(OzoneManagerProtocolProtos.OMResponse::getSuccess).orElse(true)) { - List setSnapshotPropertyRequests = new ArrayList<>(); - Map exclusiveReplicatedSizeMap = reclaimableSubFileFilter.getExclusiveReplicatedSizeMap(); - Map exclusiveSizeMap = reclaimableSubFileFilter.getExclusiveSizeMap(); - for (String snapshot : Stream.of(exclusiveSizeMap.keySet(), - exclusiveReplicatedSizeMap.keySet()).flatMap(Collection::stream).distinct().collect(Collectors.toList())) { - setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(exclusiveSizeMap, - exclusiveReplicatedSizeMap, snapshot)); - } - //Updating directory deep clean flag of snapshot. - if (currentSnapshotInfo != null) { - setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir( - currentSnapshotInfo.getTableKey())); + if (request.isPresent() && isBufferLimitCrossed(ratisByteLimit, consumedSize, + request.get().getSerializedSize())) { + if (purgePathRequestList.size() != 0) { + // if message buffer reaches max limit, avoid sending further + remainNum = 0; + break; + } + // if directory itself is having a lot of keys / files, + // reduce capacity to minimum level + remainNum = MIN_ERR_LIMIT_PER_TASK; + request = prepareDeleteDirRequest( + remainNum, pendingDeletedDirInfo.getValue(), + pendingDeletedDirInfo.getKey(), allSubDirList, + getOzoneManager().getKeyManager(), reclaimableDirFilter.apply(pendingDeletedDirInfo), + reclaimableSubFileFilter); + } + if (!request.isPresent()) { + continue; + } + PurgePathRequest purgePathRequest = request.get(); + consumedSize += purgePathRequest.getSerializedSize(); + purgePathRequestList.add(purgePathRequest); + remainNum = remainNum - purgePathRequest.getDeletedSubFilesCount(); + remainNum = remainNum - purgePathRequest.getMarkDeletedSubDirsCount(); + // Count up the purgeDeletedDir, subDirs and subFiles + if (purgePathRequest.getDeletedDir() != null + && !purgePathRequest.getDeletedDir().isEmpty()) { + dirNum++; + } + subDirNum += purgePathRequest.getMarkDeletedSubDirsCount(); + subFileNum += purgePathRequest.getDeletedSubFilesCount(); } - submitSetSnapshotRequest(setSnapshotPropertyRequests); + Pair> retVal = optimizeDirDeletesAndSubmitRequest( + remainNum, dirNum, subDirNum, subFileNum, + allSubDirList, purgePathRequestList, snapshotTableKey, startTime, + ratisByteLimit - consumedSize, + getOzoneManager().getKeyManager(), reclaimableDirFilter, reclaimableSubFileFilter, + expectedPreviousSnapshotId); + remainNum = retVal.getKey(); + + if (remainNum == initialRemainNum && + retVal.getValue().map(OzoneManagerProtocolProtos.OMResponse::getSuccess).orElse(true)) { + List setSnapshotPropertyRequests = new ArrayList<>(); + Map exclusiveReplicatedSizeMap = reclaimableSubFileFilter.getExclusiveReplicatedSizeMap(); + Map exclusiveSizeMap = reclaimableSubFileFilter.getExclusiveSizeMap(); + for (String snapshot : Stream.of(exclusiveSizeMap.keySet(), + exclusiveReplicatedSizeMap.keySet()).flatMap(Collection::stream).distinct().collect(Collectors.toList())) { + setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(exclusiveSizeMap, + exclusiveReplicatedSizeMap, snapshot)); + } + //Updating directory deep clean flag of snapshot. + if (currentSnapshotInfo != null) { + setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir( + currentSnapshotInfo.getTableKey())); + } + + submitSetSnapshotRequest(setSnapshotPropertyRequests); + } } } catch (IOException e) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 3702ee7fbfdc..c6c0df869997 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.service; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -104,7 +105,6 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private final Map exclusiveSizeMap; private final Map exclusiveReplicatedSizeMap; private final Map snapshotSeekMap; - private static ClientId clientId = ClientId.randomId(); public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, @@ -230,6 +230,7 @@ private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyMana String volume = currentSnapshotInfo == null ? null : currentSnapshotInfo.getVolumeName(); String bucket = currentSnapshotInfo == null ? null : currentSnapshotInfo.getBucketName(); String snapshotTableKey = currentSnapshotInfo == null ? null : currentSnapshotInfo.getTableKey(); + String startKey = ""; int initialRemainNum = remainNum; boolean successStatus = true; @@ -242,50 +243,71 @@ private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyMana OmSnapshotManager omSnapshotManager = getOzoneManager().getOmSnapshotManager(); SnapshotChainManager snapshotChainManager = ((OmMetadataManagerImpl)getOzoneManager().getMetadataManager()) .getSnapshotChainManager(); + // This is to avoid race condition b/w purge request and snapshot chain updation. For AOS taking the global + // snapshotId since AOS could process multiple buckets in one iteration. While using path previous snapshot + // Id for a snapshot since it would process only one bucket. + UUID expectedPreviousSnapshotId = currentSnapshotInfo == null ? + snapshotChainManager.getLatestGlobalSnapshotId() : + SnapshotUtils.getPreviousSnapshotId(currentSnapshotInfo, snapshotChainManager); + IOzoneManagerLock lock = getOzoneManager().getMetadataManager().getLock(); - ReclaimableKeyFilter reclaimableKeyFilter = new ReclaimableKeyFilter(omSnapshotManager, snapshotChainManager, + // Purge deleted Keys in the deletedTable && rename entries in the snapshotRenamedTable which doesn't have a + // reference in the previous snapshot. + try (ReclaimableKeyFilter reclaimableKeyFilter = new ReclaimableKeyFilter(omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); - ReclaimableRenameEntryFilter renameEntryFilter = new ReclaimableRenameEntryFilter( - omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); - List> renamedTableEntries = keyManager.getRenamesKeyEntries(volume, bucket, - startKey, renameEntryFilter, remainNum); - submitPurgeKeysRequest() - - PendingKeysDeletion pendingKeysDeletion = keyManager.getPendingDeletionKeys(volume, bucket, startKey, - reclaimableKeyFilter, remainNum); - List keyBlocksList = pendingKeysDeletion.getKeyBlocksList(); - if (keyBlocksList != null && !keyBlocksList.isEmpty()) { - Pair purgeResult = processKeyDeletes(keyBlocksList, getOzoneManager().getKeyManager(), - pendingKeysDeletion.getKeysToModify(), snapshotTableKey); - remainNum -= purgeResult.getKey(); - successStatus = purgeResult.getValue(); - } - - // Checking remainNum is greater than zero and not equal to the initial value if there were some keys to - // reclaim. This is to check if - if (remainNum > 0 && successStatus) { - List setSnapshotPropertyRequests = new ArrayList<>(); - Map exclusiveReplicatedSizeMap = reclaimableKeyFilter.getExclusiveReplicatedSizeMap(); - Map exclusiveSizeMap = reclaimableKeyFilter.getExclusiveSizeMap(); - for (String snapshot : Stream.of(exclusiveSizeMap.keySet(), - exclusiveReplicatedSizeMap.keySet()).flatMap(Collection::stream).distinct().collect(Collectors.toList())) { - setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(exclusiveSizeMap, - exclusiveReplicatedSizeMap, snapshot)); - } - - //Updating directory deep clean flag of snapshot. - if (currentSnapshotInfo != null) { - setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir( - snapshotTableKey)); + ReclaimableRenameEntryFilter renameEntryFilter = new ReclaimableRenameEntryFilter( + omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock)) { + List renamedTableEntries = + keyManager.getRenamesKeyEntries(volume, bucket, startKey, renameEntryFilter, remainNum).stream() + .map(entry -> { + try { + return entry.getKey(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).collect(Collectors.toList()); + remainNum -= renamedTableEntries.size(); + + // Get pending keys that can be deleted + PendingKeysDeletion pendingKeysDeletion = keyManager.getPendingDeletionKeys(volume, bucket, startKey, + reclaimableKeyFilter, remainNum); + List keyBlocksList = pendingKeysDeletion.getKeyBlocksList(); + //submit purge requests if there are renamed entries to be purged or keys to be purged. + if (!renamedTableEntries.isEmpty() || keyBlocksList != null && !keyBlocksList.isEmpty()) { + Pair purgeResult = processKeyDeletes(keyBlocksList, getOzoneManager().getKeyManager(), + pendingKeysDeletion.getKeysToModify(), renamedTableEntries, snapshotTableKey, expectedPreviousSnapshotId); + remainNum -= purgeResult.getKey(); + successStatus = purgeResult.getValue(); } + // Checking remainNum is greater than zero and not equal to the initial value if there were some keys to + // reclaim. This is to check if + if (remainNum > 0 && successStatus) { + List setSnapshotPropertyRequests = new ArrayList<>(); + Map exclusiveReplicatedSizeMap = reclaimableKeyFilter.getExclusiveReplicatedSizeMap(); + Map exclusiveSizeMap = reclaimableKeyFilter.getExclusiveSizeMap(); + List previousPathSnapshotsInChain = + Stream.of(exclusiveSizeMap.keySet(), exclusiveReplicatedSizeMap.keySet()) + .flatMap(Collection::stream).distinct().collect(Collectors.toList()); + for (String snapshot : previousPathSnapshotsInChain) { + setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(exclusiveSizeMap, + exclusiveReplicatedSizeMap, snapshot)); + } - submitSetSnapshotRequest(setSnapshotPropertyRequests); + //Updating directory deep clean flag of snapshot. + if (currentSnapshotInfo != null) { + setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir( + snapshotTableKey)); + } + submitSetSnapshotRequest(setSnapshotPropertyRequests); + } } } catch (IOException e) { throw e; + } catch (UncheckedIOException e) { + throw e.getCause(); } return remainNum; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index d553c157c0ea..0586e92fc166 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ServiceException; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; @@ -29,29 +28,20 @@ import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; -import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientRequest; @@ -59,7 +49,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.UUID; @@ -88,11 +77,13 @@ public class SnapshotDeletingService extends AbstractKeyDeletingService { private final OzoneManager ozoneManager; private final OmSnapshotManager omSnapshotManager; - private final SnapshotChainManager chainManager; + private final SnapshotChainManager snapshotChainManager; private final AtomicBoolean suspended; private final OzoneConfiguration conf; private final AtomicLong successRunCount; - private final long snapshotDeletionPerTask; + private final int keyLimitPerTask; + private final int snapshotDeletionPerTask; + private final int ratisByteLimit; public SnapshotDeletingService(long interval, long serviceTimeout, OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient) @@ -104,13 +95,21 @@ public SnapshotDeletingService(long interval, long serviceTimeout, this.omSnapshotManager = ozoneManager.getOmSnapshotManager(); OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); - this.chainManager = omMetadataManager.getSnapshotChainManager(); + this.snapshotChainManager = omMetadataManager.getSnapshotChainManager(); this.successRunCount = new AtomicLong(0); this.suspended = new AtomicBoolean(false); this.conf = ozoneManager.getConfiguration(); - this.snapshotDeletionPerTask = conf - .getLong(SNAPSHOT_DELETING_LIMIT_PER_TASK, + this.snapshotDeletionPerTask = conf.getInt(SNAPSHOT_DELETING_LIMIT_PER_TASK, SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT); + int limit = (int) conf.getStorageSize( + OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, + OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT, + StorageUnit.BYTES); + // always go to 90% of max limit for request as other header will be added + this.ratisByteLimit = (int) (limit * 0.9); + this.keyLimitPerTask = conf.getInt( + OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK, + OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); } private class SnapshotDeletingTask implements BackgroundTask { @@ -123,34 +122,34 @@ public BackgroundTaskResult call() throws InterruptedException { } getRunCount().incrementAndGet(); - SnapshotChainManager snapshotChainManager = - ((OmMetadataManagerImpl)ozoneManager.getMetadataManager()).getSnapshotChainManager(); try { Iterator iterator = snapshotChainManager.iterator(true); long snapshotLimit = snapshotDeletionPerTask; while (iterator.hasNext() && snapshotLimit > 0) { - SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, chainManager, iterator.next()); + SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, snapshotChainManager, iterator.next()); // Only Iterate in deleted snapshot & only if all the changes have been flushed into disk. if (shouldIgnoreSnapshot(snapInfo)) { continue; } - SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, chainManager, snapInfo); + SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, snapInfo); + SnapshotInfo nextToNextSnapshot = nextSnapshot == null ? null : SnapshotUtils.getNextSnapshot(ozoneManager, + snapshotChainManager, snapInfo); // Continue if the next snapshot is not active. This is to avoid unnecessary copies from one snapshot to // another. if (nextSnapshot != null && nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { continue; } - // Wait for the next iteration if the next snapshot is still running deep cleaning or AOS is running - // cleaning, since purge transaction will add entries and it could be processed by mistake. - if ((nextSnapshot == null && isDeepCleaningRunningOnAOS()) || - (nextSnapshot != null && isSnapshotDeepCleaningServiceRunning(nextSnapshot))) { + // Wait for the next iteration if the next snapshot or next to next snapshot is still not deep cleaned + // since purge transaction will add entries and it could be processed by mistake. + if ((nextSnapshot != null && isSnapshotDeepCleaned(nextSnapshot)) || + (nextToNextSnapshot != null && isSnapshotDeepCleaned(nextToNextSnapshot))) { continue; } successRunCount.incrementAndGet(); - submitSnapshotPurgeAndMoveRequest(snapInfo, nextSnapshot); + submitSnapshotMoveDeletedKeys(snapInfo, nextSnapshot); snapshotLimit--; } } catch (IOException e) { @@ -159,19 +158,43 @@ public BackgroundTaskResult call() throws InterruptedException { return BackgroundTaskResult.EmptyTaskResult.newResult(); } - public void submitSnapshotPurgeAndMoveRequest(SnapshotInfo snapInfo, SnapshotInfo nextSnapshotInfo) { + private void submitSnapshotPurgeRequest(List purgeSnapshotKeys) { + if (!purgeSnapshotKeys.isEmpty()) { + SnapshotPurgeRequest snapshotPurgeRequest = SnapshotPurgeRequest + .newBuilder() + .addAllSnapshotDBKeys(purgeSnapshotKeys) + .build(); - OzoneManagerProtocolProtos.SnapshotPurgeAndMoveRequest.Builder requestBuilder = - OzoneManagerProtocolProtos.SnapshotPurgeAndMoveRequest.newBuilder().setSnapshotID( - HddsUtils.toProtobuf(snapInfo.getSnapshotId())); + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(Type.SnapshotPurge) + .setSnapshotPurgeRequest(snapshotPurgeRequest) + .setClientId(clientId.toString()) + .build(); - // When null it means next target is the AOS. - if (nextSnapshotInfo != null) { - requestBuilder.setExpectedNextSnapshotID(HddsUtils.toProtobuf(nextSnapshotInfo.getSnapshotId())); + submitRequest(omRequest); } + } + + public void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, + List toReclaimList, + List toNextDBList, + List renamedList, + List dirsToMove) throws InterruptedException { + + OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder = + OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest.newBuilder() + .setFromSnapshot(snapInfo.getProtobuf()); + + OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest moveDeletedKeys = moveDeletedKeysBuilder + .addAllReclaimKeys(toReclaimList) + .addAllNextDBKeys(toNextDBList) + .addAllRenamedKeys(renamedList) + .addAllDeletedDirsToMove(dirsToMove) + .build(); + OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(Type.SnapshotPurgeAndMove) - .setSnapshotPurgeAndMoveRequest(requestBuilder.build()) + .setCmdType(Type.SnapshotMoveDeletedKeys) + .setSnapshotMoveDeletedKeysRequest(moveDeletedKeys) .setClientId(clientId.toString()) .build(); @@ -211,72 +234,12 @@ boolean shouldIgnoreSnapshot(SnapshotInfo snapInfo) throws IOException { SnapshotInfo.SnapshotStatus snapshotStatus = snapInfo.getSnapshotStatus(); return snapshotStatus != SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED || OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapInfo) - && isSnapshotDeepCleaningServiceRunning(snapInfo); + && isSnapshotDeepCleaned(snapInfo); } - private boolean isSnapshotDeepCleaningServiceRunning(SnapshotInfo snapInfo) { + private boolean isSnapshotDeepCleaned(SnapshotInfo snapInfo) { return (!snapInfo.getDeepClean() || !snapInfo.getDeepCleanedDeletedDir()); } - private boolean isDeepCleaningRunningOnAOS() { - return getOzoneManager().getKeyManager().getDeletingService().isRunningOnAOS() || - getOzoneManager().getKeyManager().getDirDeletingService().isRunningOnAOS(); - } - - - // TODO: Move this util class. - public static boolean isBlockLocationInfoSame(OmKeyInfo prevKeyInfo, - OmKeyInfo deletedKeyInfo) { - - if (prevKeyInfo == null && deletedKeyInfo == null) { - LOG.debug("Both prevKeyInfo and deletedKeyInfo are null."); - return true; - } - if (prevKeyInfo == null || deletedKeyInfo == null) { - LOG.debug("prevKeyInfo: '{}' or deletedKeyInfo: '{}' is null.", - prevKeyInfo, deletedKeyInfo); - return false; - } - // For hsync, Though the blockLocationInfo of a key may not be same - // at the time of snapshot and key deletion as blocks can be appended. - // If the objectId is same then the key is same. - if (prevKeyInfo.isHsync() && deletedKeyInfo.isHsync()) { - return true; - } - - if (prevKeyInfo.getKeyLocationVersions().size() != - deletedKeyInfo.getKeyLocationVersions().size()) { - return false; - } - - OmKeyLocationInfoGroup deletedOmKeyLocation = - deletedKeyInfo.getLatestVersionLocations(); - OmKeyLocationInfoGroup prevOmKeyLocation = - prevKeyInfo.getLatestVersionLocations(); - - if (deletedOmKeyLocation == null || prevOmKeyLocation == null) { - return false; - } - - List deletedLocationList = - deletedOmKeyLocation.getLocationList(); - List prevLocationList = - prevOmKeyLocation.getLocationList(); - - if (deletedLocationList.size() != prevLocationList.size()) { - return false; - } - - for (int idx = 0; idx < deletedLocationList.size(); idx++) { - OmKeyLocationInfo deletedLocationInfo = deletedLocationList.get(idx); - OmKeyLocationInfo prevLocationInfo = prevLocationList.get(idx); - if (!deletedLocationInfo.hasSameBlockAs(prevLocationInfo)) { - return false; - } - } - - return true; - } - @Override public BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index db6d9b7b9084..ca6689ff51e8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -46,7 +46,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithObjectID; import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; -import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; +import org.apache.hadoop.ozone.om.service.AbstractKeyDeletingService; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; @@ -1416,7 +1416,7 @@ long generateDiffReport( private boolean isKeyModified(OmKeyInfo fromKey, OmKeyInfo toKey) { return !fromKey.isKeyInfoSame(toKey, false, false, false, false, true) - || !SnapshotDeletingService.isBlockLocationInfoSame( + || !AbstractKeyDeletingService.isBlockLocationInfoSame( fromKey, toKey); } @@ -1465,7 +1465,7 @@ private boolean isBlockLocationSame( "OmKeyInfo"); } - return SnapshotDeletingService.isBlockLocationInfoSame( + return AbstractKeyDeletingService.isBlockLocationInfoSame( (OmKeyInfo) fromObject, (OmKeyInfo) toObject); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index f24d979ef71d..6e589ee92a30 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.snapshot; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -29,6 +30,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; +import org.apache.hadoop.ozone.om.response.key.OMKeyPurgeResponse; import org.apache.ratis.server.protocol.TermIndex; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; @@ -288,6 +290,13 @@ public static String getOzonePathKeyForFso(OMMetadataManager metadataManager, return OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId + OM_KEY_PREFIX; } + public static SnapshotInfo getLatestGlobalSnapshotInfo(OzoneManager ozoneManager, + SnapshotChainManager snapshotChainManager) throws IOException { + Optional latestGlobalSnapshot = Optional.ofNullable(snapshotChainManager.getLatestGlobalSnapshotId()); + return latestGlobalSnapshot.isPresent() ? getSnapshotInfo(ozoneManager, snapshotChainManager, + latestGlobalSnapshot.get()) : null; + } + public static SnapshotInfo getLatestSnapshotInfo(String volumeName, String bucketName, OzoneManager ozoneManager, SnapshotChainManager snapshotChainManager) throws IOException { @@ -304,24 +313,23 @@ public static UUID getLatestSnapshotId(String volumeName, String bucketName, } // Validates previous snapshotId given a snapshotInfo or volumeName & bucketName. Incase snapshotInfo is null, this - // would be considered as AOS and previous snapshot becomes the latest snapshot for the bucket in the snapshot chain. - public static boolean validatePreviousSnapshotId(SnapshotInfo snapshotInfo, String volumeName, String bucketName, - SnapshotChainManager snapshotChainManager, - UUID expectedPreviousSnapshotId) { + // would be considered as AOS and previous snapshot becomes the latest snapshot in the global snapshot chain. + // Would throw OMException if validation fails otherwise function would pass. + public static void validatePreviousSnapshotId(SnapshotInfo snapshotInfo, + SnapshotChainManager snapshotChainManager, + UUID expectedPreviousSnapshotId) throws IOException { try { - if (snapshotInfo != null && (!snapshotInfo.getVolumeName().equals(volumeName) || - !snapshotInfo.getBucketName().equals(bucketName))) { - LOG.error("Volume & Bucket mismatch expected volume: {}, expected bucket: {} for snapshot: {}", - volumeName, bucketName, snapshotInfo); - return false; + UUID previousSnapshotId = snapshotInfo == null ? snapshotChainManager.getLatestGlobalSnapshotId() : + SnapshotUtils.getPreviousSnapshotId(snapshotInfo, snapshotChainManager); + if (!Objects.equals(expectedPreviousSnapshotId, previousSnapshotId)) { + throw new OMException("Snapshot validation failed. Expected previous snapshotId : " + + expectedPreviousSnapshotId + " but was " + previousSnapshotId, + OMException.ResultCodes.INVALID_REQUEST); } - UUID previousSnapshotId = snapshotInfo == null ? SnapshotUtils.getLatestSnapshotId(volumeName, bucketName, - snapshotChainManager) : SnapshotUtils.getPreviousSnapshotId(snapshotInfo, snapshotChainManager); - return Objects.equals(expectedPreviousSnapshotId, previousSnapshotId); } catch (IOException e) { - LOG.error("Error while validating previous snapshot for volume: {}, bucket: {}, snapshot: {}", volumeName, - bucketName, snapshotInfo == null ? null : snapshotInfo.getName(), e); + LOG.error("Error while validating previous snapshot for snapshot: {}", + snapshotInfo == null ? null : snapshotInfo.getName(), e); + throw e; } - return false; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 0f2ab6150669..9483ac0c3747 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -49,6 +49,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; +import org.apache.hadoop.ozone.om.service.AbstractKeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.om.snapshot.SnapshotTestUtils.StubbedPersistentMap; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; @@ -799,7 +800,7 @@ public void testGenerateDiffReport() throws IOException { mockedSnapshotDeletingService = mockStatic( SnapshotDeletingService.class)) { mockedSnapshotDeletingService.when(() -> - SnapshotDeletingService.isBlockLocationInfoSame(any(OmKeyInfo.class), + AbstractKeyDeletingService.isBlockLocationInfoSame(any(OmKeyInfo.class), any(OmKeyInfo.class))) .thenAnswer(i -> { int keyVal = Integer.parseInt(((OmKeyInfo)i.getArgument(0)) From 2640d2700c57a1669bb4067ca683ec633f03f658 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 9 Sep 2024 10:54:34 -0700 Subject: [PATCH 05/22] Push interim merge in progress Change-Id: I235052073fb814cdc8818b37882d4a047483971d --- .../java/org/apache/hadoop/ozone/OmUtils.java | 1 - .../apache/hadoop/ozone/om/KeyManager.java | 14 +++ .../hadoop/ozone/om/KeyManagerImpl.java | 28 +----- .../om/ratis/OzoneManagerDoubleBuffer.java | 2 +- .../snapshot/OMSnapshotPurgeRequest.java | 47 +++++++--- .../om/service/SnapshotDeletingService.java | 94 +++++++++++++++++-- 6 files changed, 137 insertions(+), 49 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index c74104351052..8fa8921cc9a9 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -334,7 +334,6 @@ public static boolean isReadOnly( case RenameSnapshot: case SnapshotMoveDeletedKeys: case SnapshotPurge: - case SnapshotPurgeAndMove: case RecoverLease: case SetTimes: case AbortExpiredMultiPartUploads: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 986fd9beef97..05df15425109 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; import java.util.List; /** @@ -273,6 +274,19 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, TableIterator> getPendingDeletionDirs(String volume, String bucket) throws IOException; + default List> getDeletedDirEntries(String volume, String bucket, int count) + throws IOException { + List> deletedDirEntries = new ArrayList<>(count); + try (TableIterator> iterator = + getPendingDeletionDirs(volume, bucket)) { + while (deletedDirEntries.size() < count && iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + deletedDirEntries.add(Table.newKeyValue(kv.getKey(), kv.getValue())); + } + return deletedDirEntries; + } + } + /** * Returns all sub directories under the given parent directory. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 2395722a9d9e..1bb57d2e78cb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -755,36 +755,10 @@ public List>> getDeletedKeyEntries( Table.KeyValue kv = delKeyIter.next(); if (kv != null && filter.apply(kv)) { deletedKeyEntries.add(Table.newKeyValue(kv.getKey(), kv.getValue().cloneOmKeyInfoList())); - - for (OmKeyInfo info : infoList.cloneOmKeyInfoList()) { - - // Skip the key if the filter doesn't allow the file to be deleted. - if (filter.apply(Table.newKeyValue(kv.getKey(), info))) { - List blockIDS = info.getKeyLocationVersions().stream() - .flatMap(versionLocations -> versionLocations.getLocationList().stream() - .map(b -> new BlockID(b.getContainerID(), b.getLocalID()))).collect(Collectors.toList()); - BlockGroup keyBlocks = BlockGroup.newBuilder().setKeyName(kv.getKey()) - .addAllBlockIDs(blockIDS).build(); - blockGroupList.add(keyBlocks); - currentCount++; - } else { - notReclaimableKeyInfo.addOmKeyInfo(info); - } - } - - List notReclaimableKeyInfoList = notReclaimableKeyInfo.getOmKeyInfoList(); - - // If all the versions are not reclaimable, then modify key by just purging the key that can be purged. - if (notReclaimableKeyInfoList.size() > 0 && - notReclaimableKeyInfoList.size() != infoList.getOmKeyInfoList().size()) { - keysToModify.put(kv.getKey(), notReclaimableKeyInfo); - } - keyBlocksList.addAll(blockGroupList); } } - nextPageStartKey = delKeyIter.hasNext() ? delKeyIter.next().getKey() : null; } - return new PendingKeysDeletion(keyBlocksList, keysToModify, nextPageStartKey); + return deletedKeyEntries; } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java index 59e8861ef3b5..cb37b915093c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java @@ -449,7 +449,7 @@ private String addToBatch(Queue buffer, BatchOperation batchOperation) { private List> splitReadyBufferAtCreateSnapshot() { final List> response = new ArrayList<>(); final Set standaloneBatchCmdTypes = ImmutableSet.of( - OzoneManagerProtocolProtos.Type.SnapshotPurge, OzoneManagerProtocolProtos.Type.SnapshotPurgeAndMove); + OzoneManagerProtocolProtos.Type.SnapshotPurge, OzoneManagerProtocolProtos.Type.CreateSnapshot); final List> standaloneBatchConditions = ImmutableList.of(OMResponse::hasCreateSnapshotResponse, (omResponse) -> standaloneBatchCmdTypes.contains(omResponse.getCmdType())); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index 03f3f6fa42b2..99b9dd3d5e38 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -110,17 +110,17 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); - Optional previousSnapshot = Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(ozoneManager, - snapshotChainManager, fromSnapshot)); - Optional previousPrevSnapshot = previousSnapshot.isPresent() ? - Optional.ofNullable(SnapshotUtils.getPreviousSnapshot(ozoneManager, snapshotChainManager, - previousSnapshot.get())) : Optional.empty(); + SnapshotInfo nextToNextSnapshot = nextSnapshot == null ? null + : SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, nextSnapshot); + SnapshotInfo previousSnapshot = SnapshotUtils.getPreviousSnapshot(ozoneManager, + snapshotChainManager, fromSnapshot); + SnapshotInfo previousPrevSnapshot = previousSnapshot == null ? null + : SnapshotUtils.getPreviousSnapshot(ozoneManager, snapshotChainManager, previousSnapshot); // Step 1: Reset the deep clean flag for the next active snapshot if and only if the last 2 snapshots in the // chain are active, otherwise set it to prevent deep cleaning from running till the deleted snapshots don't // get purged. - updateNextSnapshotInfoFields(nextSnapshot, omMetadataManager, trxnLogIndex, - !(previousSnapshot.map(SnapshotInfo::getSnapshotStatus).orElse(SNAPSHOT_ACTIVE) == SNAPSHOT_ACTIVE && - previousPrevSnapshot.map(SnapshotInfo::getSnapshotStatus).orElse(SNAPSHOT_ACTIVE) == SNAPSHOT_ACTIVE)); + updateNextSnapshotInfoFields(nextSnapshot, nextToNextSnapshot, previousSnapshot, previousPrevSnapshot, + omMetadataManager, trxnLogIndex); // Step 2: Update the snapshot chain. @@ -151,12 +151,18 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn return omClientResponse; } - private void updateNextSnapshotInfoFields(SnapshotInfo nextSnapshot, OmMetadataManagerImpl omMetadataManager, - long trxnLogIndex, boolean deepCleanFlagValue) throws IOException { + private void updateNextSnapshotInfoFields(SnapshotInfo nextSnapshot, + SnapshotInfo nextToNextSnapshot, + SnapshotInfo previousSnapshot, + SnapshotInfo previousToPreviousSnapshot, + OmMetadataManagerImpl omMetadataManager, + long trxnLogIndex) throws IOException { if (nextSnapshot != null) { - // Setting next snapshot deep clean to false, Since the - // current snapshot is deleted. We can potentially - // reclaim more keys in the next snapshot. + // Reset the deep clean flag for the next active snapshot if and only if the last 2 snapshots in the + // chain are active, otherwise set it to prevent deep cleaning from running till the deleted snapshots don't + // get purged. There could be potentially more keys to be reclaimed. + boolean deepCleanFlagValue = (previousSnapshot == null || previousSnapshot.getSnapshotStatus() == SNAPSHOT_ACTIVE) + && (previousToPreviousSnapshot == null || previousToPreviousSnapshot.getSnapshotStatus() == SNAPSHOT_ACTIVE); nextSnapshot.setDeepClean(deepCleanFlagValue); nextSnapshot.setDeepCleanedDeletedDir(deepCleanFlagValue); @@ -166,6 +172,21 @@ private void updateNextSnapshotInfoFields(SnapshotInfo nextSnapshot, OmMetadataM updatedSnapshotInfos.put(nextSnapshot.getTableKey(), nextSnapshot); } + if (nextToNextSnapshot != null) { + // Reset the deep clean flag for the next active snapshot if and only if the last 2 snapshots in the + // chain are active, otherwise set it to prevent deep cleaning from running till the deleted snapshots don't + // get purged. + boolean deepCleanFlagValue = (previousSnapshot == null || previousSnapshot.getSnapshotStatus() == SNAPSHOT_ACTIVE) + && (nextSnapshot == null || nextSnapshot.getSnapshotStatus() == SNAPSHOT_ACTIVE); + nextToNextSnapshot.setDeepClean(deepCleanFlagValue); + nextToNextSnapshot.setDeepCleanedDeletedDir(deepCleanFlagValue); + + // Update table cache first + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextToNextSnapshot.getTableKey()), + CacheValue.get(trxnLogIndex, nextSnapshot)); + updatedSnapshotInfos.put(nextToNextSnapshot.getTableKey(), nextSnapshot); + } + } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 0586e92fc166..e61bfab7e6b3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -28,18 +28,25 @@ import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.lock.BootstrapStateHandler; +import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.ratis.protocol.ClientId; @@ -49,12 +56,15 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT; @@ -124,7 +134,9 @@ public BackgroundTaskResult call() throws InterruptedException { getRunCount().incrementAndGet(); try { + int remaining = keyLimitPerTask; Iterator iterator = snapshotChainManager.iterator(true); + List snapshotsToBePurged = new ArrayList<>(); long snapshotLimit = snapshotDeletionPerTask; while (iterator.hasNext() && snapshotLimit > 0) { SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, snapshotChainManager, iterator.next()); @@ -148,10 +160,65 @@ public BackgroundTaskResult call() throws InterruptedException { (nextToNextSnapshot != null && isSnapshotDeepCleaned(nextToNextSnapshot))) { continue; } + try(ReferenceCounted snapshot = omSnapshotManager.getSnapshot(snapInfo.getVolumeName(), + snapInfo.getBucketName(), + snapInfo.getName())) { + KeyManager snapshotKeyManager = snapshot.get().getKeyManager(); + int moveCount = 0; + // Get all entries from deletedKeyTable. + List>> deletedKeyEntries = + snapshotKeyManager.getDeletedKeyEntries(snapInfo.getVolumeName(), snapInfo.getBucketName(), + null, (kv) -> true, remaining); + moveCount += deletedKeyEntries.size(); + // Get all entries from deletedDirTable. + List> deletedDirEntries = snapshotKeyManager.getDeletedDirEntries( + snapInfo.getVolumeName(), snapInfo.getBucketName(), remaining - moveCount); + moveCount += deletedDirEntries.size(); + // Get all entries from snapshotRenamedTable. + List> renameEntries = snapshotKeyManager.getRenamesKeyEntries( + snapInfo.getVolumeName(), snapInfo.getBucketName(), null , (kv) -> true, remaining - moveCount); + moveCount += renameEntries.size(); + if (moveCount > 0) { + try { + submitSnapshotMoveDeletedKeys(snapInfo, deletedKeyEntries.stream().map(kv -> { + try { + return SnapshotMoveKeyInfos.newBuilder().setKey(kv.getKey()).addAllKeyInfos(kv.getValue() + .stream().map(val -> val.getProtobuf(ClientVersion.CURRENT_VERSION)) + .collect(Collectors.toList())).build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).collect(Collectors.toList()), + renameEntries.stream().map(kv -> { + try { + return HddsProtos.KeyValue.newBuilder().setKey(kv.getKey()).setValue(kv.getValue()).build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).collect(Collectors.toList()), + deletedDirEntries.stream() + .map(kv -> { + try { + return kv.getKey(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).collect(Collectors.toList())); + remaining -= moveCount; + } catch (UncheckedIOException e) { + throw e.getCause(); + } + } else { + snapshotsToBePurged.add(snapInfo); + } + } successRunCount.incrementAndGet(); - submitSnapshotMoveDeletedKeys(snapInfo, nextSnapshot); snapshotLimit--; } + if (!snapshotsToBePurged.isEmpty()) { + submitSnapshotPurgeRequest(snapshotsToBePurged.stream().map(SnapshotInfo::getTableKey) + .collect(Collectors.toList())); + } } catch (IOException e) { LOG.error("Error while running Snapshot Deleting Service", e); } @@ -176,21 +243,34 @@ private void submitSnapshotPurgeRequest(List purgeSnapshotKeys) { } public void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, - List toReclaimList, - List toNextDBList, - List renamedList, - List dirsToMove) throws InterruptedException { + List keysToMove, List renamedList, + List dirsToMove) { OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder = OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest.newBuilder() .setFromSnapshot(snapInfo.getProtobuf()); OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest moveDeletedKeys = moveDeletedKeysBuilder - .addAllReclaimKeys(toReclaimList) - .addAllNextDBKeys(toNextDBList) + .addAllNextDBKeys(keysToMove) .addAllRenamedKeys(renamedList) .addAllDeletedDirsToMove(dirsToMove) + .setPurgeMovedKeys(true) .build(); + if (isBufferLimitCrossed(ratisByteLimit, 0, moveDeletedKeys.getSerializedSize())) { + int remaining = MIN_ERR_LIMIT_PER_TASK; + keysToMove = keysToMove.subList(0, Math.min(remaining, keysToMove.size())); + remaining -= keysToMove.size(); + renamedList = renamedList.subList(0, Math.min(remaining, renamedList.size())); + remaining -= renamedList.size(); + dirsToMove = dirsToMove.subList(0, Math.min(remaining, dirsToMove.size())); + remaining -= dirsToMove.size(); + moveDeletedKeys = moveDeletedKeysBuilder + .addAllNextDBKeys(keysToMove) + .addAllRenamedKeys(renamedList) + .addAllDeletedDirsToMove(dirsToMove) + .setPurgeMovedKeys(true) + .build(); + } OMRequest omRequest = OMRequest.newBuilder() .setCmdType(Type.SnapshotMoveDeletedKeys) From d44686a1e6a2f38795e7cbc79d0ecc819d60f3ea Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 9 Sep 2024 14:01:09 -0700 Subject: [PATCH 06/22] Push interim merge in progress Change-Id: I40d1c664a0dddd455ba00479457f4b7204bc9d85 --- .../om/service/SnapshotDeletingService.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index e61bfab7e6b3..2509461a55b7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ServiceException; -import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -41,6 +40,8 @@ import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; +import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; @@ -58,8 +59,10 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -94,6 +97,7 @@ public class SnapshotDeletingService extends AbstractKeyDeletingService { private final int keyLimitPerTask; private final int snapshotDeletionPerTask; private final int ratisByteLimit; + private MultiLocks snapshotIdLocks; public SnapshotDeletingService(long interval, long serviceTimeout, OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient) @@ -120,6 +124,8 @@ public SnapshotDeletingService(long interval, long serviceTimeout, this.keyLimitPerTask = conf.getInt( OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK, OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); + IOzoneManagerLock lock = getOzoneManager().getMetadataManager().getLock(); + this.snapshotIdLocks = new MultiLocks<>(lock, OzoneManagerLock.Resource.SNAPSHOT_GC_LOCK, true); } private class SnapshotDeletingTask implements BackgroundTask { @@ -160,6 +166,12 @@ public BackgroundTaskResult call() throws InterruptedException { (nextToNextSnapshot != null && isSnapshotDeepCleaned(nextToNextSnapshot))) { continue; } + // Acquire write lock on current snapshot and next snapshot in chain. + if (!snapshotIdLocks.acquireLock(Arrays.asList(snapInfo.getSnapshotId(), + Optional.ofNullable(nextSnapshot).map(SnapshotInfo::getSnapshotId).orElse(null))) + .isLockAcquired()) { + continue; + } try(ReferenceCounted snapshot = omSnapshotManager.getSnapshot(snapInfo.getVolumeName(), snapInfo.getBucketName(), snapInfo.getName())) { @@ -211,6 +223,8 @@ public BackgroundTaskResult call() throws InterruptedException { } else { snapshotsToBePurged.add(snapInfo); } + } finally { + snapshotIdLocks.releaseLock(); } successRunCount.incrementAndGet(); snapshotLimit--; From 74d1ebc029dc1f1b70d73c5d3d50182c91bac6b4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 9 Sep 2024 15:08:16 -0700 Subject: [PATCH 07/22] Test fixes Change-Id: I3df0ba8346d2573a5d403b3f58fab8aedb365bda --- .../org/apache/hadoop/ozone/om/TestKeyPurging.java | 2 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 2 +- .../request/key/TestOMKeyPurgeRequestAndResponse.java | 6 +++--- .../ozone/om/service/TestKeyDeletingService.java | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java index e3bb5b5bccb8..29efc1caeb10 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java @@ -140,7 +140,7 @@ public void testKeysPurgingByKeyDeletingService() throws Exception { GenericTestUtils.waitFor( () -> { try { - return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) + return keyManager.getPendingDeletionKeys(null, Integer.MAX_VALUE) .getKeyBlocksList().size() == 0; } catch (IOException e) { return false; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 1bb57d2e78cb..052941d57ffc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -669,7 +669,7 @@ public PendingKeysDeletion getPendingDeletionKeys( for (OmKeyInfo info : infoList.cloneOmKeyInfoList()) { // Skip the key if the filter doesn't allow the file to be deleted. - if (filter.apply(Table.newKeyValue(kv.getKey(), info))) { + if (filter == null || filter.apply(Table.newKeyValue(kv.getKey(), info))) { List blockIDS = info.getKeyLocationVersions().stream() .flatMap(versionLocations -> versionLocations.getLocationList().stream() .map(b -> new BlockID(b.getContainerID(), b.getLocalID()))).collect(Collectors.toList()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java index 7299f6968e68..60ccdc656efc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java @@ -103,10 +103,9 @@ private OMRequest createPurgeKeysRequest(List deletedKeys, List .setVolumeName(volumeName) .setBucketName(bucketName) .addAllKeys(deletedKeys) - .addAllRenamedKeys(renamedEntries) .build(); PurgeKeysRequest.Builder purgeKeysRequest = PurgeKeysRequest.newBuilder() - .addDeletedKeys(deletedKeysInBucket); + .addDeletedKeys(deletedKeysInBucket).addAllRenamedKeys(renamedEntries); if (snapshotDbKey != null) { purgeKeysRequest.setSnapshotTableKey(snapshotDbKey); @@ -261,7 +260,8 @@ public void testKeyPurgeInSnapshot() throws Exception { try (BatchOperation batchOperation = omMetadataManager.getStore().initBatchOperation()) { - OMKeyPurgeResponse omKeyPurgeResponse = new OMKeyPurgeResponse(omResponse, deletedKeyNames, snapInfo, null); + OMKeyPurgeResponse omKeyPurgeResponse = new OMKeyPurgeResponse(omResponse, deleteKeysAndRenamedEntry.getKey(), + deleteKeysAndRenamedEntry.getValue(), snapInfo, null); omKeyPurgeResponse.addToDBBatch(omMetadataManager, batchOperation); // Do manual commit and see whether addToBatch is successful or not. diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java index 8163592cfc6d..05ffcc273276 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java @@ -191,7 +191,7 @@ void checkIfDeleteServiceIsDeletingKeys() () -> getDeletedKeyCount() >= initialDeletedCount + keyCount, 100, 10000); assertThat(getRunCount()).isGreaterThan(initialRunCount); - assertThat(keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).getKeyBlocksList()) + assertThat(keyManager.getPendingDeletionKeys((kv) -> true, Integer.MAX_VALUE).getKeyBlocksList()) .isEmpty(); } @@ -220,7 +220,7 @@ void checkDeletionForKeysWithMultipleVersions() throws Exception { 1000, 10000); assertThat(getRunCount()) .isGreaterThan(initialRunCount); - assertThat(keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).getKeyBlocksList()) + assertThat(keyManager.getPendingDeletionKeys((kv) -> true, Integer.MAX_VALUE).getKeyBlocksList()) .isEmpty(); // The 1st version of the key has 1 block and the 2nd version has 2 @@ -262,7 +262,7 @@ void checkDeletedTableCleanUpForSnapshot() throws Exception { 1000, 10000); assertThat(getRunCount()) .isGreaterThan(initialRunCount); - assertThat(keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).getKeyBlocksList()) + assertThat(keyManager.getPendingDeletionKeys(null, Integer.MAX_VALUE).getKeyBlocksList()) .isEmpty(); // deletedTable should have deleted key of the snapshot bucket @@ -758,7 +758,7 @@ private long getRunCount() { private int countKeysPendingDeletion() { try { - final int count = keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) + final int count = keyManager.getPendingDeletionKeys(null, Integer.MAX_VALUE) .getKeyBlocksList().size(); LOG.debug("KeyManager keys pending deletion: {}", count); return count; @@ -769,7 +769,7 @@ private int countKeysPendingDeletion() { private long countBlocksPendingDeletion() { try { - return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) + return keyManager.getPendingDeletionKeys(null, Integer.MAX_VALUE) .getKeyBlocksList() .stream() .map(BlockGroup::getBlockIDList) From da5ff0a5b78835217117a04365fff71845455075 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 9 Sep 2024 16:24:06 -0700 Subject: [PATCH 08/22] Flush snapshot info Change-Id: Ibaf5b526c4f821087212307d07c3847d2d15aa59 --- .../apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java index eac0502bb826..26b872dd7496 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java @@ -93,6 +93,7 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, fromSnapshotStore.commitBatchOperation(writeBatch); } } + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); } else { processKeys(batchOperation, omMetadataManager); processKeysToUpdate(batchOperation, omMetadataManager); From 831cd46ff48d75c0ea4f1c21f907deebbb5f3262 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 9 Sep 2024 17:24:14 -0700 Subject: [PATCH 09/22] HDDS-11440. Add a lastTransactionInfo field in SnapshotInfo to check for transactions in flight on the snapshot Change-Id: Ifa01e118427bd37000eef801a6dc8b12ccfe5d0a --- .../hadoop/hdds/utils/TransactionInfo.java | 11 +++- .../hadoop/ozone/om/helpers/SnapshotInfo.java | 62 ++++++++++++------- .../src/main/proto/OmClientProtocol.proto | 1 + .../hadoop/ozone/om/OmSnapshotManager.java | 35 +++++++++++ .../key/OMDirectoriesPurgeRequestWithFSO.java | 11 ++-- .../om/request/key/OMKeyPurgeRequest.java | 28 ++++++--- .../snapshot/OMSnapshotCreateRequest.java | 3 +- .../OMSnapshotMoveDeletedKeysRequest.java | 25 +++++--- .../snapshot/OMSnapshotPurgeRequest.java | 9 ++- .../OMDirectoriesPurgeResponseWithFSO.java | 4 +- .../om/response/key/OMKeyPurgeResponse.java | 4 +- .../OMSnapshotMoveDeletedKeysResponse.java | 5 ++ .../ozone/om/snapshot/SnapshotUtils.java | 10 +++ .../ozone/om/snapshot/TestSnapshotInfo.java | 44 +++++++++++++ 14 files changed, 203 insertions(+), 49 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java index e7c4ec4ce3d6..68d05af0c6ff 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java @@ -162,7 +162,16 @@ public String toString() { */ public static TransactionInfo readTransactionInfo( DBStoreHAManager metadataManager) throws IOException { - return metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); + return readTransactionInfo(metadataManager, false); + } + + /** + * Return transaction info persisted in OM DB skipping cache. + */ + public static TransactionInfo readTransactionInfo( + DBStoreHAManager metadataManager, boolean skipCache) throws IOException { + return skipCache ? metadataManager.getTransactionInfoTable().getSkipCache(TRANSACTION_INFO_KEY) : + metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); } public SnapshotInfo toSnapshotInfo() { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index 47a48c37e8e0..54af86348937 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -19,6 +19,7 @@ */ import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ByteString; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.CopyObject; @@ -124,6 +125,7 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) { private long exclusiveSize; private long exclusiveReplicatedSize; private boolean deepCleanedDeletedDir; + private ByteString lastTransactionInfo; private SnapshotInfo(Builder b) { this.snapshotId = b.snapshotId; @@ -145,6 +147,7 @@ private SnapshotInfo(Builder b) { this.exclusiveSize = b.exclusiveSize; this.exclusiveReplicatedSize = b.exclusiveReplicatedSize; this.deepCleanedDeletedDir = b.deepCleanedDeletedDir; + this.lastTransactionInfo = b.lastTransactionInfo; } public void setName(String name) { @@ -261,13 +264,15 @@ public SnapshotInfo.Builder toBuilder() { .setGlobalPreviousSnapshotId(globalPreviousSnapshotId) .setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) + .setDbTxSequenceNumber(dbTxSequenceNumber) .setDeepClean(deepClean) .setSstFiltered(sstFiltered) .setReferencedSize(referencedSize) .setReferencedReplicatedSize(referencedReplicatedSize) .setExclusiveSize(exclusiveSize) .setExclusiveReplicatedSize(exclusiveReplicatedSize) - .setDeepCleanedDeletedDir(deepCleanedDeletedDir); + .setDeepCleanedDeletedDir(deepCleanedDeletedDir) + .setLastTransactionInfo(lastTransactionInfo); } /** @@ -293,6 +298,7 @@ public static class Builder { private long exclusiveSize; private long exclusiveReplicatedSize; private boolean deepCleanedDeletedDir; + private ByteString lastTransactionInfo; public Builder() { // default values @@ -411,6 +417,16 @@ public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) { return this; } + public Builder setLastTransactionInfo(byte[] lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo == null ? null : ByteString.copyFrom(lastTransactionInfo); + return this; + } + + public Builder setLastTransactionInfo(ByteString lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo; + return this; + } + public SnapshotInfo build() { Preconditions.checkNotNull(name); return new SnapshotInfo(this); @@ -445,6 +461,10 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() { sib.setGlobalPreviousSnapshotID(toProtobuf(globalPreviousSnapshotId)); } + if (lastTransactionInfo != null) { + sib.setLastTransactionInfo(lastTransactionInfo); + } + sib.setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) .setDbTxSequenceNumber(dbTxSequenceNumber) @@ -513,6 +533,10 @@ public static SnapshotInfo getFromProtobuf( snapshotInfoProto.getDeepCleanedDeletedDir()); } + if (snapshotInfoProto.hasLastTransactionInfo()) { + osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo().toByteArray()); + } + osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) .setCheckpointDir(snapshotInfoProto.getCheckpointDir()) .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber()); @@ -605,6 +629,14 @@ public void setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) { this.deepCleanedDeletedDir = deepCleanedDeletedDir; } + public ByteString getLastTransactionInfo() { + return lastTransactionInfo; + } + + public void setLastTransactionInfo(byte[] lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo == null ? null : ByteString.copyFrom(lastTransactionInfo); + } + /** * Generate default name of snapshot, (used if user doesn't provide one). */ @@ -673,7 +705,8 @@ public boolean equals(Object o) { referencedReplicatedSize == that.referencedReplicatedSize && exclusiveSize == that.exclusiveSize && exclusiveReplicatedSize == that.exclusiveReplicatedSize && - deepCleanedDeletedDir == that.deepCleanedDeletedDir; + deepCleanedDeletedDir == that.deepCleanedDeletedDir && + Objects.equals(lastTransactionInfo, that.lastTransactionInfo); } @Override @@ -684,7 +717,7 @@ public int hashCode() { globalPreviousSnapshotId, snapshotPath, checkpointDir, deepClean, sstFiltered, referencedSize, referencedReplicatedSize, - exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir); + exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir, lastTransactionInfo); } /** @@ -692,27 +725,7 @@ public int hashCode() { */ @Override public SnapshotInfo copyObject() { - return new Builder() - .setSnapshotId(snapshotId) - .setName(name) - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setSnapshotStatus(snapshotStatus) - .setCreationTime(creationTime) - .setDeletionTime(deletionTime) - .setPathPreviousSnapshotId(pathPreviousSnapshotId) - .setGlobalPreviousSnapshotId(globalPreviousSnapshotId) - .setSnapshotPath(snapshotPath) - .setCheckpointDir(checkpointDir) - .setDbTxSequenceNumber(dbTxSequenceNumber) - .setDeepClean(deepClean) - .setSstFiltered(sstFiltered) - .setReferencedSize(referencedSize) - .setReferencedReplicatedSize(referencedReplicatedSize) - .setExclusiveSize(exclusiveSize) - .setExclusiveReplicatedSize(exclusiveReplicatedSize) - .setDeepCleanedDeletedDir(deepCleanedDeletedDir) - .build(); + return this.toBuilder().build(); } @Override @@ -737,6 +750,7 @@ public String toString() { ", exclusiveSize: '" + exclusiveSize + '\'' + ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' + ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' + + ", lastTransactionInfo: '" + lastTransactionInfo + '\'' + '}'; } } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index e79797993c13..eefcfa7552ca 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -886,6 +886,7 @@ message SnapshotInfo { optional uint64 exclusiveReplicatedSize = 18; // note: shared sizes can be calculated from: referenced - exclusive optional bool deepCleanedDeletedDir = 19; + optional bytes lastTransactionInfo = 20; } message SnapshotDiffJobProto { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 18b29118182d..5c9333859a2c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.CodecRegistry; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; @@ -674,6 +675,40 @@ private ReferenceCounted getSnapshot(String snapshotTableKey, boolea return snapshotCache.get(snapshotInfo.getSnapshotId()); } + /** + * Checks if the last transaction performed on the snapshot has been flushed to disk. + * @param metadataManager Metadatamanager of Active OM. + * @param snapshotTableKey table key corresponding to snapshot in snapshotInfoTable. + * @return True if the changes have been flushed to DB otherwise false + * @throws IOException + */ + public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataManager, String snapshotTableKey) + throws IOException { + // Need this info from cache since the snapshot could have been updated only on cache and not on disk. + SnapshotInfo snapshotInfo = metadataManager.getSnapshotInfoTable().get(snapshotTableKey); + return areSnapshotChangesFlushedToDB(metadataManager, snapshotInfo); + } + + /** + * Checks if the last transaction performed on the snapshot has been flushed to disk. + * @param metadataManager Metadatamanager of Active OM. + * @param snapshotInfo table key corresponding to snapshot in snapshotInfoTable, this should be a value from cache + * and not from disk. + * @return True if the changes have been flushed to DB otherwise false + * @throws IOException + */ + public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataManager, SnapshotInfo snapshotInfo) + throws IOException { + if (snapshotInfo != null) { + TransactionInfo snapshotTransactionInfo = snapshotInfo.getLastTransactionInfo() != null ? + TransactionInfo.getCodec().fromPersistedFormat(snapshotInfo.getLastTransactionInfo().toByteArray()) : null; + TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager, true); + return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; + } + return false; + } + + /** * Returns OmSnapshot object and skips active check. * This should only be used for API calls initiated by background service e.g. purgeKeys, purgeSnapshot, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index cb10c0d2e40a..422c872b4e18 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -42,7 +42,6 @@ import org.apache.hadoop.ozone.om.response.key.OMDirectoriesPurgeResponseWithFSO; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; @@ -71,7 +70,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn Map, OmBucketInfo> volBucketInfoMap = new HashMap<>(); OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); Map openKeyInfoMap = new HashMap<>(); - + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); OMMetrics omMetrics = ozoneManager.getMetrics(); try { if (fromSnapshot != null) { @@ -149,6 +149,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } } } + if (fromSnapshotInfo != null) { + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshotInfo, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshotInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshotInfo)); + } } catch (IOException ex) { // Case of IOException for fromProtobuf will not happen // as this is created and send within OM @@ -164,8 +169,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } } - OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( - getOmRequest()); OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO( omResponse.build(), purgeRequests, ozoneManager.isRatisEnabled(), getBucketLayout(), volBucketInfoMap, fromSnapshotInfo, openKeyInfoMap); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java index 5738d7945bfe..ab3692bcdd17 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.util.ArrayList; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OzoneManager; @@ -61,6 +64,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn String fromSnapshot = purgeKeysRequest.hasSnapshotTableKey() ? purgeKeysRequest.getSnapshotTableKey() : null; List keysToBePurgedList = new ArrayList<>(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); @@ -71,17 +75,27 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn keysToBePurgedList.add(deletedKey); } } + final SnapshotInfo fromSnapshotInfo; try { - SnapshotInfo fromSnapshotInfo = null; - if (fromSnapshot != null) { - fromSnapshotInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot); - } - omClientResponse = new OMKeyPurgeResponse(omResponse.build(), - keysToBePurgedList, fromSnapshotInfo, keysToUpdateList); + fromSnapshotInfo = fromSnapshot == null ? null : SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot); } catch (IOException ex) { - omClientResponse = new OMKeyPurgeResponse(createErrorOMResponse(omResponse, ex)); + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, ex)); + } + + // Setting transaction info for snapshot, this is to prevent duplicate purge requests to OM from background + // services. + try { + if (fromSnapshotInfo != null) { + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshotInfo, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshotInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshotInfo)); + } + } catch (IOException e) { + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, e)); } + omClientResponse = new OMKeyPurgeResponse(omResponse.build(), keysToBePurgedList, fromSnapshotInfo, + keysToUpdateList); return omClientResponse; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 3aa4151cea32..c55539070a5e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -166,7 +167,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn ((RDBStore) omMetadataManager.getStore()).getDb() .getLatestSequenceNumber(); snapshotInfo.setDbTxSequenceNumber(dbLatestSequenceNumber); - + SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); // Snapshot referenced size should be bucket's used bytes OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index df4781bb0ca6..f2fe481b0750 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.om.request.snapshot; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; @@ -82,15 +84,20 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn nextSnapshot = SnapshotUtils.getNextActiveSnapshot(fromSnapshot, snapshotChainManager, ozoneManager); // Get next non-deleted snapshot. - List nextDBKeysList = - moveDeletedKeysRequest.getNextDBKeysList(); - List reclaimKeysList = - moveDeletedKeysRequest.getReclaimKeysList(); - List renamedKeysList = - moveDeletedKeysRequest.getRenamedKeysList(); - List movedDirs = - moveDeletedKeysRequest.getDeletedDirsToMoveList(); - + List nextDBKeysList = moveDeletedKeysRequest.getNextDBKeysList(); + List reclaimKeysList = moveDeletedKeysRequest.getReclaimKeysList(); + List renamedKeysList = moveDeletedKeysRequest.getRenamedKeysList(); + List movedDirs = moveDeletedKeysRequest.getDeletedDirsToMoveList(); + + // Update lastTransactionInfo for fromSnapshot and the nextSnapshot. + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshot, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshot)); + if (nextSnapshot != null) { + SnapshotUtils.setTransactionInfoInSnapshot(nextSnapshot, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), nextSnapshot)); + } omClientResponse = new OMSnapshotMoveDeletedKeysResponse( omResponse.build(), fromSnapshot, nextSnapshot, nextDBKeysList, reclaimKeysList, renamedKeysList, movedDirs); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index 47304b416aeb..ef266cc06c0e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -110,9 +110,16 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, trxnLogIndex); // Step 2: Update the snapshot chain. updateSnapshotChainAndCache(omMetadataManager, fromSnapshot, trxnLogIndex); - // Step 3: Purge the snapshot from SnapshotInfoTable cache. + // Step 3: Purge the snapshot from SnapshotInfoTable cache and also remove from the map. omMetadataManager.getSnapshotInfoTable() .addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), CacheValue.get(trxnLogIndex)); + updatedSnapshotInfos.remove(fromSnapshot.getTableKey()); + } + + for (SnapshotInfo snapshotInfo : updatedSnapshotInfos.values()) { + SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapshotInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), snapshotInfo)); } omClientResponse = new OMSnapshotPurgeResponse(omResponse.build(), snapshotDbKeys, updatedSnapshotInfos); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 138e942e2b60..28c3e3d758e2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -48,12 +48,13 @@ import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; /** * Response for {@link OMDirectoriesPurgeRequestWithFSO} request. */ @CleanupTableInfo(cleanupTables = {DELETED_TABLE, DELETED_DIR_TABLE, - DIRECTORY_TABLE, FILE_TABLE}) + DIRECTORY_TABLE, FILE_TABLE, SNAPSHOT_INFO_TABLE}) public class OMDirectoriesPurgeResponseWithFSO extends OmKeyResponse { private static final Logger LOG = LoggerFactory.getLogger(OMDirectoriesPurgeResponseWithFSO.class); @@ -97,6 +98,7 @@ public void addToDBBatch(OMMetadataManager metadataManager, fromSnapshotStore.commitBatchOperation(writeBatch); } } + metadataManager.getSnapshotInfoTable().putWithBatch(batchOp, fromSnapshotInfo.getTableKey(), fromSnapshotInfo); } else { processPaths(metadataManager, batchOp); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java index b59c7d18d408..cd2f7d190f45 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java @@ -39,12 +39,13 @@ import jakarta.annotation.Nonnull; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; import static org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveDeletedKeysResponse.createRepeatedOmKeyInfo; /** * Response for {@link OMKeyPurgeRequest} request. */ -@CleanupTableInfo(cleanupTables = {DELETED_TABLE}) +@CleanupTableInfo(cleanupTables = {DELETED_TABLE, SNAPSHOT_INFO_TABLE}) public class OMKeyPurgeResponse extends OmKeyResponse { private List purgeKeyList; private SnapshotInfo fromSnapshot; @@ -90,6 +91,7 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, fromSnapshotStore.commitBatchOperation(writeBatch); } } + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); } else { processKeys(batchOperation, omMetadataManager); processKeysToUpdate(batchOperation, omMetadataManager); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index 3b5a7454f9db..f39d5827a0cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -133,6 +133,11 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, } } + // Flush snapshot info to rocksDB. + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); + if (nextSnapshot != null) { + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, nextSnapshot.getTableKey(), nextSnapshot); + } } private void deleteDirsFromSnapshot(BatchOperation batchOp, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index e0f40dabd8a7..94f1a4e7fce8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.snapshot; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -26,6 +27,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; +import org.apache.ratis.server.protocol.TermIndex; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.slf4j.Logger; @@ -138,6 +140,14 @@ public static void checkSnapshotActive(SnapshotInfo snapInfo, } } + /** + * Set transactionInfo in snapshotInfo. + */ + public static void setTransactionInfoInSnapshot(SnapshotInfo snapshot, TermIndex termIndex) throws IOException { + TransactionInfo transactionInfo = TransactionInfo.valueOf(termIndex); + snapshot.setLastTransactionInfo(TransactionInfo.getCodec().toPersistedFormat(transactionInfo)); + } + /** * Get the next non deleted snapshot in the snapshot chain. */ diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java index dc00433e179b..39d81bf898b5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java @@ -19,12 +19,18 @@ package org.apache.hadoop.ozone.om.snapshot; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; import org.apache.hadoop.util.Time; +import org.apache.ratis.server.protocol.TermIndex; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -114,4 +120,42 @@ public void testSnapshotSSTFilteredFlag() throws Exception { snapshotInfo.put(EXPECTED_SNAPSHOT_KEY, info); assertTrue(snapshotInfo.get(EXPECTED_SNAPSHOT_KEY).isSstFiltered()); } + + @Test + public void testLastTransactionInfo() throws Exception { + Table snapshotInfo = + omMetadataManager.getSnapshotInfoTable(); + SnapshotInfo info = createSnapshotInfo(); + snapshotInfo.put(EXPECTED_SNAPSHOT_KEY, info); + assertEquals(snapshotInfo.get(EXPECTED_SNAPSHOT_KEY).getLastTransactionInfo(), null); + omMetadataManager.getTransactionInfoTable().put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.valueOf(0, 0)); + // Checking if changes have been flushed when lastTransactionInfo is null + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, info)); + TermIndex termIndex = TermIndex.valueOf(1, 1); + SnapshotUtils.setTransactionInfoInSnapshot(info, termIndex); + // Checking if changes to snapshot object has been updated but not updated on cache or disk. + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + snapshotInfo.addCacheEntry(new CacheKey<>(EXPECTED_SNAPSHOT_KEY), CacheValue.get(termIndex.getIndex(), info)); + + assertEquals(snapshotInfo.get(EXPECTED_SNAPSHOT_KEY).getLastTransactionInfo(), info.getLastTransactionInfo()); + + // Checking if changes have not been flushed when snapshot last transaction info is behind OmTransactionTable value. + assertFalse(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + omMetadataManager.getTransactionInfoTable().addCacheEntry(new CacheKey<>(OzoneConsts.TRANSACTION_INFO_KEY), + CacheValue.get(termIndex.getIndex(), TransactionInfo.valueOf(1, 1))); + assertFalse(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + + // Checking changes are flushed when transaction is equal. + omMetadataManager.getTransactionInfoTable().put(OzoneConsts.TRANSACTION_INFO_KEY, + TransactionInfo.valueOf(1, 1)); + + + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + // Checking changes are flushed when transactionIndex is greater . + omMetadataManager.getTransactionInfoTable().put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.valueOf(1, 2)); + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + // Checking changes are flushed when both term & transactionIndex is greater. + omMetadataManager.getTransactionInfoTable().put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.valueOf(2, 2)); + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + } } From eebd1ce537d5b6a1b76b78f6264d5a4513cf6a9f Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 19:04:04 -0700 Subject: [PATCH 10/22] Fix OmSnapshotMoveTableKeys request Change-Id: I02ee0ab044a234eff4ead00d1426032060edd44b --- .../hadoop/hdds/utils/TransactionInfo.java | 11 +- .../hadoop/ozone/om/helpers/SnapshotInfo.java | 7 +- .../src/main/proto/OmClientProtocol.proto | 17 +- .../hadoop/ozone/om/OmSnapshotManager.java | 2 +- .../ratis/utils/OzoneManagerRatisUtils.java | 3 + .../OMSnapshotMoveDeletedKeysRequest.java | 58 +++-- .../OMSnapshotMoveTableKeysRequest.java | 118 +++++++++++ .../om/response/key/OMKeyPurgeResponse.java | 3 +- .../OMSnapshotMoveDeletedKeysResponse.java | 99 ++++----- .../OMSnapshotMoveTableKeysResponse.java | 198 ++++++++++++++++++ .../om/service/SnapshotDeletingService.java | 33 ++- 11 files changed, 428 insertions(+), 121 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java index 68d05af0c6ff..e7c4ec4ce3d6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java @@ -162,16 +162,7 @@ public String toString() { */ public static TransactionInfo readTransactionInfo( DBStoreHAManager metadataManager) throws IOException { - return readTransactionInfo(metadataManager, false); - } - - /** - * Return transaction info persisted in OM DB skipping cache. - */ - public static TransactionInfo readTransactionInfo( - DBStoreHAManager metadataManager, boolean skipCache) throws IOException { - return skipCache ? metadataManager.getTransactionInfoTable().getSkipCache(TRANSACTION_INFO_KEY) : - metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); + return metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); } public SnapshotInfo toSnapshotInfo() { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index 2bd222ca9195..631de9224ddf 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -418,11 +418,6 @@ public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) { return this; } - public Builder setLastTransactionInfo(byte[] lastTransactionInfo) { - this.lastTransactionInfo = lastTransactionInfo == null? null : ByteString.copyFrom(lastTransactionInfo); - return this; - } - public Builder setLastTransactionInfo(ByteString lastTransactionInfo) { this.lastTransactionInfo = lastTransactionInfo; return this; @@ -535,7 +530,7 @@ public static SnapshotInfo getFromProtobuf( } if (snapshotInfoProto.hasLastTransactionInfo()) { - osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo().toByteArray()); + osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo()); } osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index b85dff849010..77df8a1e1a2f 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -153,6 +153,7 @@ enum Type { GetServerDefaults = 134; GetQuotaRepairStatus = 135; StartQuotaRepair = 136; + SnapshotMoveTableKeys = 137; } enum SafeMode { @@ -273,7 +274,7 @@ message OMRequest { optional ListSnapshotRequest ListSnapshotRequest = 113; optional SnapshotDiffRequest snapshotDiffRequest = 114; optional DeleteSnapshotRequest DeleteSnapshotRequest = 115; - optional SnapshotMoveDeletedKeysRequest SnapshotMoveDeletedKeysRequest = 116; + optional SnapshotMoveDeletedKeysRequest SnapshotMoveDeletedKeysRequest = 116 [deprecated = true]; optional hdds.TransferLeadershipRequestProto TransferOmLeadershipRequest = 117; optional SnapshotPurgeRequest SnapshotPurgeRequest = 118; @@ -296,6 +297,7 @@ message OMRequest { optional GetQuotaRepairStatusRequest GetQuotaRepairStatusRequest = 133; optional StartQuotaRepairRequest StartQuotaRepairRequest = 134; repeated SetSnapshotPropertyRequest SetSnapshotPropertyRequests = 135; + optional SnapshotMoveTableKeysRequest SnapshotMoveTableKeysRequest = 136; } message OMResponse { @@ -1979,13 +1981,18 @@ message PrintCompactionLogDagRequest { } message SnapshotMoveDeletedKeysRequest { - optional SnapshotInfo fromSnapshot = 1 [deprecated = true]; + optional SnapshotInfo fromSnapshot = 1; repeated SnapshotMoveKeyInfos nextDBKeys = 2; - repeated SnapshotMoveKeyInfos reclaimKeys = 3 [deprecated = true]; + repeated SnapshotMoveKeyInfos reclaimKeys = 3; repeated hadoop.hdds.KeyValue renamedKeys = 4; repeated string deletedDirsToMove = 5; - optional bool purgeMovedKeys = 6; - optional hadoop.hdds.UUID fromSnapshotID = 7; +} + +message SnapshotMoveTableKeysRequest { + optional hadoop.hdds.UUID fromSnapshotID = 1; + repeated SnapshotMoveKeyInfos deletedKeys = 2; + repeated SnapshotMoveKeyInfos deletedDirs = 3; + repeated hadoop.hdds.KeyValue renamedKeys = 4; } message SnapshotMoveKeyInfos { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 98b0ace13eaa..8426e8809d2f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -703,7 +703,7 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa if (snapshotInfo != null && snapshotInfo.getLastTransactionInfo() != null) { snapshotTransactionInfo = TransactionInfo.getCodec() .fromPersistedFormat(snapshotInfo.getLastTransactionInfo().toByteArray()); - TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager, true); + TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager); return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; } return false; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 5dc640c742cc..06185a86def4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -77,6 +77,7 @@ import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotDeleteRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveDeletedKeysRequest; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveTableKeysRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotPurgeRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotRenameRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotSetPropertyRequest; @@ -229,6 +230,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMSnapshotRenameRequest(omRequest); case SnapshotMoveDeletedKeys: return new OMSnapshotMoveDeletedKeysRequest(omRequest); + case SnapshotMoveTableKeys: + return new OMSnapshotMoveTableKeysRequest(omRequest); case SnapshotPurge: return new OMSnapshotPurgeRequest(omRequest); case SetSnapshotProperty: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index 0c64e8cc3b00..7769d177ae6f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -19,14 +19,11 @@ package org.apache.hadoop.ozone.om.request.snapshot; -import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -46,7 +43,6 @@ import java.io.IOException; import java.util.List; -import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.FILESYSTEM_SNAPSHOT; /** @@ -65,38 +61,37 @@ public OMSnapshotMoveDeletedKeysRequest(OMRequest omRequest) { @Override @DisallowedUntilLayoutVersion(FILESYSTEM_SNAPSHOT) public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIndex termIndex) { - OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); - OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); - SnapshotChainManager snapshotChainManager = omMetadataManager.getSnapshotChainManager(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) + ozoneManager.getMetadataManager(); + SnapshotChainManager snapshotChainManager = + omMetadataManager.getSnapshotChainManager(); - SnapshotMoveDeletedKeysRequest moveDeletedKeysRequest = getOmRequest().getSnapshotMoveDeletedKeysRequest(); - SnapshotInfo fromSnapshot = null; + SnapshotMoveDeletedKeysRequest moveDeletedKeysRequest = + getOmRequest().getSnapshotMoveDeletedKeysRequest(); + SnapshotInfo fromSnapshot = SnapshotInfo.getFromProtobuf( + moveDeletedKeysRequest.getFromSnapshot()); // If there is no Non-Deleted Snapshot move the // keys to Active Object Store. SnapshotInfo nextSnapshot = null; OMClientResponse omClientResponse = null; - OzoneManagerProtocolProtos.OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); try { - fromSnapshot = moveDeletedKeysRequest.hasFromSnapshotID() ? SnapshotUtils.getSnapshotInfo(ozoneManager, - snapshotChainManager, fromProtobuf(moveDeletedKeysRequest.getFromSnapshotID())) : - SnapshotInfo.getFromProtobuf(moveDeletedKeysRequest.getFromSnapshot()); - nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager,fromSnapshot); + // Check the snapshot exists. + SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot.getTableKey()); - // Only skip the move operation for newer requests. Newer requests would always have the purgeMovedKeys in the - // request. Skip move operation if the next snapshot is not active. - if (moveDeletedKeysRequest.hasPurgeMovedKeys() && nextSnapshot != null && - nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { - throw new OMException("Next snapshot : " + nextSnapshot + " in chain is not active.", - OMException.ResultCodes.INVALID_REQUEST); - } + nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); // Get next non-deleted snapshot. - List nextDBKeysList = moveDeletedKeysRequest.getNextDBKeysList(); - List reclaimKeysList = moveDeletedKeysRequest.getReclaimKeysList(); - List renamedKeysList = moveDeletedKeysRequest.getRenamedKeysList(); - List movedDirs = moveDeletedKeysRequest.getDeletedDirsToMoveList(); - + List nextDBKeysList = + moveDeletedKeysRequest.getNextDBKeysList(); + List reclaimKeysList = + moveDeletedKeysRequest.getReclaimKeysList(); + List renamedKeysList = + moveDeletedKeysRequest.getRenamedKeysList(); + List movedDirs = + moveDeletedKeysRequest.getDeletedDirsToMoveList(); // Update lastTransactionInfo for fromSnapshot and the nextSnapshot. SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshot, termIndex); omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), @@ -106,14 +101,13 @@ snapshotChainManager, fromProtobuf(moveDeletedKeysRequest.getFromSnapshotID())) omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextSnapshot.getTableKey()), CacheValue.get(termIndex.getIndex(), nextSnapshot)); } - - - omClientResponse = new OMSnapshotMoveDeletedKeysResponse(omResponse.build(), fromSnapshot, nextSnapshot, - nextDBKeysList, reclaimKeysList, renamedKeysList, movedDirs, - moveDeletedKeysRequest.hasPurgeMovedKeys() && moveDeletedKeysRequest.getPurgeMovedKeys()); + omClientResponse = new OMSnapshotMoveDeletedKeysResponse( + omResponse.build(), fromSnapshot, nextSnapshot, + nextDBKeysList, reclaimKeysList, renamedKeysList, movedDirs); } catch (IOException ex) { - omClientResponse = new OMSnapshotMoveDeletedKeysResponse(createErrorOMResponse(omResponse, ex)); + omClientResponse = new OMSnapshotMoveDeletedKeysResponse( + createErrorOMResponse(omResponse, ex)); } return omClientResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java new file mode 100644 index 000000000000..2cd7fe76a036 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.request.snapshot; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveDeletedKeysResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveTableKeysResponse; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveTableKeysRequest; +import org.apache.ratis.server.protocol.TermIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.FILESYSTEM_SNAPSHOT; + +/** + * Handles OMSnapshotMoveTableKeysRequest Request. + * This is an OM internal request. Does not need @RequireSnapshotFeatureState. + */ +public class OMSnapshotMoveTableKeysRequest extends OMClientRequest { + + private static final Logger LOG = LoggerFactory.getLogger(OMSnapshotMoveTableKeysRequest.class); + + public OMSnapshotMoveTableKeysRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + @DisallowedUntilLayoutVersion(FILESYSTEM_SNAPSHOT) + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIndex termIndex) { + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); + SnapshotChainManager snapshotChainManager = omMetadataManager.getSnapshotChainManager(); + + SnapshotMoveTableKeysRequest moveTableKeysRequest = getOmRequest().getSnapshotMoveTableKeysRequest(); + + OMClientResponse omClientResponse; + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); + try { + SnapshotInfo fromSnapshot = SnapshotUtils.getSnapshotInfo(ozoneManager, + snapshotChainManager, fromProtobuf(moveTableKeysRequest.getFromSnapshotID())); + // If there is no Non-Deleted Snapshot move the + // keys to Active Object Store. + SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager,fromSnapshot); + + // If next snapshot is not active then ignore move. Since this could be a redundant operations. + if (nextSnapshot != null && nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { + throw new OMException("Next snapshot : " + nextSnapshot + " in chain is not active.", + OMException.ResultCodes.INVALID_REQUEST); + } + + // Filter only deleted keys with atlest one keyInfo per key. + List deletedKeys = + moveTableKeysRequest.getDeletedKeysList().stream() + .filter(snapshotMoveKeyInfos -> !snapshotMoveKeyInfos.getKeyInfosList().isEmpty()) + .collect(Collectors.toList()); + List renamedKeysList = moveTableKeysRequest.getRenamedKeysList(); + // Filter only deleted dirs with only one keyInfo per key. + List deletedDirs = moveTableKeysRequest.getDeletedDirsList().stream() + .filter(snapshotMoveKeyInfos -> snapshotMoveKeyInfos.getKeyInfosList().size() == 1) + .collect(Collectors.toList()); + + // Update lastTransactionInfo for fromSnapshot and the nextSnapshot. + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshot, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshot)); + if (nextSnapshot != null) { + SnapshotUtils.setTransactionInfoInSnapshot(nextSnapshot, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), nextSnapshot)); + } + omClientResponse = new OMSnapshotMoveTableKeysResponse(omResponse.build(), fromSnapshot, nextSnapshot, + deletedKeys, deletedDirs, renamedKeysList); + } catch (IOException ex) { + omClientResponse = new OMSnapshotMoveTableKeysResponse(createErrorOMResponse(omResponse, ex)); + } + + return omClientResponse; + } +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java index 26b872dd7496..d1408ccb65fa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java @@ -108,8 +108,7 @@ private void processKeysToUpdate(BatchOperation batchOp, for (SnapshotMoveKeyInfos keyToUpdate : keysToUpdateList) { List keyInfosList = keyToUpdate.getKeyInfosList(); - RepeatedOmKeyInfo repeatedOmKeyInfo = - createRepeatedOmKeyInfo(keyInfosList); + RepeatedOmKeyInfo repeatedOmKeyInfo = createRepeatedOmKeyInfo(keyInfosList); metadataManager.getDeletedTable().putWithBatch(batchOp, keyToUpdate.getKey(), repeatedOmKeyInfo); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index c68bd7ad2ec6..f39d5827a0cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -54,13 +54,13 @@ public class OMSnapshotMoveDeletedKeysResponse extends OMClientResponse { private List renamedKeysList; private List movedDirs; - private boolean purgeMovedKeys; - public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, - @Nonnull SnapshotInfo fromSnapshot, SnapshotInfo nextSnapshot, - List nextDBKeysList, List reclaimKeysList, - List renamedKeysList, List movedDirs, - boolean purgeMovedKeys) { + @Nonnull SnapshotInfo fromSnapshot, + SnapshotInfo nextSnapshot, + List nextDBKeysList, + List reclaimKeysList, + List renamedKeysList, + List movedDirs) { super(omResponse); this.fromSnapshot = fromSnapshot; this.nextSnapshot = nextSnapshot; @@ -68,7 +68,6 @@ public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, this.reclaimKeysList = reclaimKeysList; this.renamedKeysList = renamedKeysList; this.movedDirs = movedDirs; - this.purgeMovedKeys = purgeMovedKeys; } /** @@ -81,16 +80,18 @@ public OMSnapshotMoveDeletedKeysResponse(@Nonnull OMResponse omResponse) { } @Override - protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { + protected void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { // Note: a trick to get // To do this properly, refactor OzoneManagerDoubleBuffer#addToBatch and // add OmSnapshotManager as a parameter. - OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) omMetadataManager) - .getOzoneManager().getOmSnapshotManager(); + OmSnapshotManager omSnapshotManager = + ((OmMetadataManagerImpl) omMetadataManager) + .getOzoneManager().getOmSnapshotManager(); try (ReferenceCounted rcOmFromSnapshot = - omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) { + omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) { OmSnapshot fromOmSnapshot = rcOmFromSnapshot.get(); @@ -99,11 +100,14 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation rcOmNextSnapshot = omSnapshotManager.getSnapshot(nextSnapshot.getSnapshotId())) { OmSnapshot nextOmSnapshot = rcOmNextSnapshot.get(); - RDBStore nextSnapshotStore = (RDBStore) nextOmSnapshot.getMetadataManager().getStore(); + RDBStore nextSnapshotStore = + (RDBStore) nextOmSnapshot.getMetadataManager().getStore(); // Init Batch Operation for snapshot db. - try (BatchOperation writeBatch = nextSnapshotStore.initBatchOperation()) { + try (BatchOperation writeBatch = + nextSnapshotStore.initBatchOperation()) { processKeys(writeBatch, nextOmSnapshot.getMetadataManager()); - processDirs(writeBatch, nextOmSnapshot.getMetadataManager(), fromOmSnapshot); + processDirs(writeBatch, nextOmSnapshot.getMetadataManager(), + fromOmSnapshot); nextSnapshotStore.commitBatchOperation(writeBatch); nextSnapshotStore.getDb().flushWal(true); nextSnapshotStore.getDb().flush(); @@ -116,15 +120,13 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation } // Update From Snapshot Deleted Table. - RDBStore fromSnapshotStore = (RDBStore) fromOmSnapshot.getMetadataManager().getStore(); - try (BatchOperation fromSnapshotBatchOp = fromSnapshotStore.initBatchOperation()) { - deleteDirsFromSnapshot(fromSnapshotBatchOp, fromOmSnapshot.getMetadataManager()); - // if purgeMovedKeys is true, reclaimKeys would be always empty. - if (purgeMovedKeys) { - deleteKeysFromSnapshot(fromSnapshotBatchOp, fromOmSnapshot.getMetadataManager()); - } else { - processReclaimKeys(fromSnapshotBatchOp, fromOmSnapshot.getMetadataManager()); - } + RDBStore fromSnapshotStore = + (RDBStore) fromOmSnapshot.getMetadataManager().getStore(); + try (BatchOperation fromSnapshotBatchOp = + fromSnapshotStore.initBatchOperation()) { + processReclaimKeys(fromSnapshotBatchOp, + fromOmSnapshot.getMetadataManager()); + deleteDirsFromSnapshot(fromSnapshotBatchOp, fromOmSnapshot); fromSnapshotStore.commitBatchOperation(fromSnapshotBatchOp); fromSnapshotStore.getDb().flushWal(true); fromSnapshotStore.getDb().flush(); @@ -138,54 +140,52 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation } } - private void deleteDirsFromSnapshot(BatchOperation batchOp, OMMetadataManager fromSnapshotMetadataManager) + private void deleteDirsFromSnapshot(BatchOperation batchOp, + OmSnapshot fromOmSnapshot) throws IOException { for (String movedDirsKey : movedDirs) { // Delete dirs from current snapshot that are moved to next snapshot. - fromSnapshotMetadataManager + fromOmSnapshot + .getMetadataManager() .getDeletedDirTable() .deleteWithBatch(batchOp, movedDirsKey); } } - private void deleteKeysFromSnapshot(BatchOperation batchOp, OMMetadataManager fromSnapshotMetadataManager) + private void processReclaimKeys(BatchOperation batchOp, + OMMetadataManager metadataManager) throws IOException { - for (SnapshotMoveKeyInfos movedKey : nextDBKeysList) { - // Delete keys from current snapshot that are moved to next snapshot. - fromSnapshotMetadataManager.getDeletedTable().deleteWithBatch(batchOp, movedKey.getKey()); - } - - // Delete renamed keys from current snapshot that are moved to next snapshot. - for (HddsProtos.KeyValue renamedKey: renamedKeysList) { - fromSnapshotMetadataManager.getSnapshotRenamedTable().deleteWithBatch(batchOp, renamedKey.getKey()); - } - } - - @Deprecated - private void processReclaimKeys(BatchOperation batchOp, OMMetadataManager metadataManager) throws IOException { for (SnapshotMoveKeyInfos dBKey : reclaimKeysList) { - RepeatedOmKeyInfo omKeyInfos = createRepeatedOmKeyInfo(dBKey.getKeyInfosList()); + RepeatedOmKeyInfo omKeyInfos = + createRepeatedOmKeyInfo(dBKey.getKeyInfosList()); // omKeyInfos can be null, because everything from RepeatedOmKeyInfo // is moved to next snapshot which means this key can be deleted in // the current snapshot processed by SDS. The reclaim key here indicates // the key can be removed from the deleted current snapshot if (omKeyInfos == null) { - metadataManager.getDeletedTable().deleteWithBatch(batchOp, dBKey.getKey()); + metadataManager.getDeletedTable().deleteWithBatch(batchOp, + dBKey.getKey()); continue; } - metadataManager.getDeletedTable().putWithBatch(batchOp, dBKey.getKey(), omKeyInfos); + metadataManager.getDeletedTable().putWithBatch(batchOp, + dBKey.getKey(), omKeyInfos); } } - private void processDirs(BatchOperation batchOp, OMMetadataManager omMetadataManager, OmSnapshot fromOmSnapshot) + private void processDirs(BatchOperation batchOp, + OMMetadataManager omMetadataManager, + OmSnapshot fromOmSnapshot) throws IOException { for (String movedDirsKey : movedDirs) { - OmKeyInfo keyInfo = fromOmSnapshot.getMetadataManager().getDeletedDirTable().get(movedDirsKey); + OmKeyInfo keyInfo = fromOmSnapshot.getMetadataManager() + .getDeletedDirTable() + .get(movedDirsKey); if (keyInfo == null) { continue; } // Move deleted dirs to next snapshot or active DB - omMetadataManager.getDeletedDirTable().putWithBatch(batchOp, movedDirsKey, keyInfo); + omMetadataManager.getDeletedDirTable().putWithBatch( + batchOp, movedDirsKey, keyInfo); } } @@ -194,15 +194,18 @@ private void processKeys(BatchOperation batchOp, // Move renamed keys to only the next snapshot or active DB. for (HddsProtos.KeyValue renamedKey: renamedKeysList) { - metadataManager.getSnapshotRenamedTable().putWithBatch(batchOp, renamedKey.getKey(), renamedKey.getValue()); + metadataManager.getSnapshotRenamedTable() + .putWithBatch(batchOp, renamedKey.getKey(), renamedKey.getValue()); } for (SnapshotMoveKeyInfos dBKey : nextDBKeysList) { - RepeatedOmKeyInfo omKeyInfos = createRepeatedOmKeyInfo(dBKey, metadataManager); + RepeatedOmKeyInfo omKeyInfos = + createRepeatedOmKeyInfo(dBKey, metadataManager); if (omKeyInfos == null) { continue; } - metadataManager.getDeletedTable().putWithBatch(batchOp, dBKey.getKey(), omKeyInfos); + metadataManager.getDeletedTable().putWithBatch(batchOp, + dBKey.getKey(), omKeyInfos); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java new file mode 100644 index 000000000000..221cea5ce769 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.response.snapshot; + +import jakarta.annotation.Nonnull; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; + +/** + * Response for OMSnapshotMoveDeletedKeysRequest. + */ +@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) +public class OMSnapshotMoveTableKeysResponse extends OMClientResponse { + + private SnapshotInfo fromSnapshot; + private SnapshotInfo nextSnapshot; + private List deletedKeys; + private List renamedKeysList; + private List deletedDirs; + + public OMSnapshotMoveTableKeysResponse(OMResponse omResponse, + @Nonnull SnapshotInfo fromSnapshot, SnapshotInfo nextSnapshot, + List deletedKeys, + List deletedDirs, + List renamedKeys) { + super(omResponse); + this.fromSnapshot = fromSnapshot; + this.nextSnapshot = nextSnapshot; + this.deletedKeys = deletedKeys; + this.renamedKeysList = renamedKeys; + this.deletedDirs = deletedDirs; + } + + /** + * For when the request is not successful. + * For a successful request, the other constructor should be used. + */ + public OMSnapshotMoveTableKeysResponse(@Nonnull OMResponse omResponse) { + super(omResponse); + checkStatusNotOK(); + } + + @Override + protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { + + // Note: a trick to get + // To do this properly, refactor OzoneManagerDoubleBuffer#addToBatch and + // add OmSnapshotManager as a parameter. + OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) omMetadataManager) + .getOzoneManager().getOmSnapshotManager(); + + try (ReferenceCounted rcOmFromSnapshot = + omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) { + + OmSnapshot fromOmSnapshot = rcOmFromSnapshot.get(); + + if (nextSnapshot != null) { + try (ReferenceCounted + rcOmNextSnapshot = omSnapshotManager.getSnapshot(nextSnapshot.getSnapshotId())) { + + OmSnapshot nextOmSnapshot = rcOmNextSnapshot.get(); + RDBStore nextSnapshotStore = (RDBStore) nextOmSnapshot.getMetadataManager().getStore(); + // Init Batch Operation for snapshot db. + try (BatchOperation writeBatch = nextSnapshotStore.initBatchOperation()) { + addKeysToNextSnapshot(writeBatch, nextOmSnapshot.getMetadataManager()); + nextSnapshotStore.commitBatchOperation(writeBatch); + nextSnapshotStore.getDb().flushWal(true); + nextSnapshotStore.getDb().flush(); + } + } + } else { + // Handle the case where there is no next Snapshot. + addKeysToNextSnapshot(batchOperation, omMetadataManager); + } + + // Update From Snapshot Deleted Table. + RDBStore fromSnapshotStore = (RDBStore) fromOmSnapshot.getMetadataManager().getStore(); + try (BatchOperation fromSnapshotBatchOp = fromSnapshotStore.initBatchOperation()) { + deleteKeysFromSnapshot(fromSnapshotBatchOp, fromOmSnapshot.getMetadataManager()); + fromSnapshotStore.commitBatchOperation(fromSnapshotBatchOp); + fromSnapshotStore.getDb().flushWal(true); + fromSnapshotStore.getDb().flush(); + } + } + + // Flush snapshot info to rocksDB. + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); + if (nextSnapshot != null) { + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, nextSnapshot.getTableKey(), nextSnapshot); + } + } + + private void deleteKeysFromSnapshot(BatchOperation batchOp, OMMetadataManager fromSnapshotMetadataManager) + throws IOException { + for (SnapshotMoveKeyInfos deletedKey : deletedKeys) { + // Delete keys from current snapshot that are moved to next snapshot. + fromSnapshotMetadataManager.getDeletedTable().deleteWithBatch(batchOp, deletedKey.getKey()); + } + + // Delete renamed keys from current snapshot that are moved to next snapshot. + for (HddsProtos.KeyValue renamedKey: renamedKeysList) { + fromSnapshotMetadataManager.getSnapshotRenamedTable().deleteWithBatch(batchOp, renamedKey.getKey()); + } + + // Delete deletedDir from current snapshot that are moved to next snapshot. + for (SnapshotMoveKeyInfos deletedDir : deletedDirs) { + fromSnapshotMetadataManager.getDeletedDirTable().deleteWithBatch(batchOp, deletedDir.getKey()); + } + + } + + private void addKeysToNextSnapshot(BatchOperation batchOp, OMMetadataManager metadataManager) throws IOException { + + // Add renamed keys to the next snapshot or active DB. + for (HddsProtos.KeyValue renamedKey: renamedKeysList) { + metadataManager.getSnapshotRenamedTable().putWithBatch(batchOp, renamedKey.getKey(), renamedKey.getValue()); + } + // Add deleted keys to the next snapshot or active DB. + for (SnapshotMoveKeyInfos dBKey : deletedKeys) { + RepeatedOmKeyInfo omKeyInfos = createRepeatedOmKeyInfo(dBKey, metadataManager); + metadataManager.getDeletedTable().putWithBatch(batchOp, dBKey.getKey(), omKeyInfos); + } + // Add deleted dir keys to the next snapshot or active DB. + for (SnapshotMoveKeyInfos dBKey : deletedDirs) { + metadataManager.getDeletedDirTable().putWithBatch(batchOp, dBKey.getKey(), + OmKeyInfo.getFromProtobuf(dBKey.getKeyInfosList().get(0))); + } + } + + private RepeatedOmKeyInfo createRepeatedOmKeyInfo( + SnapshotMoveKeyInfos snapshotMoveKeyInfos, + OMMetadataManager metadataManager) throws IOException { + String dbKey = snapshotMoveKeyInfos.getKey(); + List keyInfoList = new ArrayList<>(); + for (KeyInfo info : snapshotMoveKeyInfos.getKeyInfosList()) { + OmKeyInfo fromProtobuf = OmKeyInfo.getFromProtobuf(info); + keyInfoList.add(fromProtobuf); + } + // When older version of keys are moved to the next snapshot's deletedTable + // The newer version might also be in the next snapshot's deletedTable and + // it might overwrite. This is to avoid that and also avoid having + // orphans blocks. Checking the last keyInfoList size omKeyInfo versions, + // this is to avoid redundant additions if the last n versions match. + RepeatedOmKeyInfo result = metadataManager.getDeletedTable().get(dbKey); + if (result == null) { + result = new RepeatedOmKeyInfo(keyInfoList); + } else if (!isSameAsLatestOmKeyInfo(keyInfoList, result)){ + keyInfoList.forEach(result::addOmKeyInfo); + } + return result; + } + + private boolean isSameAsLatestOmKeyInfo(List omKeyInfos, + RepeatedOmKeyInfo result) { + int size = result.getOmKeyInfoList().size(); + if (size >= omKeyInfos.size()) { + return omKeyInfos.equals(result.getOmKeyInfoList().subList(size - omKeyInfos.size(), size)); + } + return false; + } +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 2509461a55b7..9511ebb10063 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -48,6 +48,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveTableKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.ratis.protocol.ClientId; @@ -69,6 +70,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import static org.apache.hadoop.hdds.HddsUtils.toProtobuf; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK; @@ -211,7 +213,8 @@ public BackgroundTaskResult call() throws InterruptedException { deletedDirEntries.stream() .map(kv -> { try { - return kv.getKey(); + return SnapshotMoveKeyInfos.newBuilder().setKey(kv.getKey()) + .addKeyInfos(kv.getValue().getProtobuf(ClientVersion.CURRENT_VERSION)).build(); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -257,38 +260,34 @@ private void submitSnapshotPurgeRequest(List purgeSnapshotKeys) { } public void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, - List keysToMove, List renamedList, - List dirsToMove) { + List deletedKeys, List renamedList, + List dirsToMove) { - OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder = - OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest.newBuilder() - .setFromSnapshot(snapInfo.getProtobuf()); + SnapshotMoveTableKeysRequest.Builder moveDeletedKeysBuilder = SnapshotMoveTableKeysRequest.newBuilder() + .setFromSnapshotID(toProtobuf(snapInfo.getSnapshotId())); - OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest moveDeletedKeys = moveDeletedKeysBuilder - .addAllNextDBKeys(keysToMove) + SnapshotMoveTableKeysRequest moveDeletedKeys = moveDeletedKeysBuilder + .addAllDeletedKeys(deletedKeys) .addAllRenamedKeys(renamedList) - .addAllDeletedDirsToMove(dirsToMove) - .setPurgeMovedKeys(true) + .addAllDeletedDirs(dirsToMove) .build(); if (isBufferLimitCrossed(ratisByteLimit, 0, moveDeletedKeys.getSerializedSize())) { int remaining = MIN_ERR_LIMIT_PER_TASK; - keysToMove = keysToMove.subList(0, Math.min(remaining, keysToMove.size())); - remaining -= keysToMove.size(); + deletedKeys = deletedKeys.subList(0, Math.min(remaining, deletedKeys.size())); + remaining -= deletedKeys.size(); renamedList = renamedList.subList(0, Math.min(remaining, renamedList.size())); remaining -= renamedList.size(); dirsToMove = dirsToMove.subList(0, Math.min(remaining, dirsToMove.size())); - remaining -= dirsToMove.size(); moveDeletedKeys = moveDeletedKeysBuilder - .addAllNextDBKeys(keysToMove) + .addAllDeletedKeys(deletedKeys) .addAllRenamedKeys(renamedList) - .addAllDeletedDirsToMove(dirsToMove) - .setPurgeMovedKeys(true) + .addAllDeletedDirs(dirsToMove) .build(); } OMRequest omRequest = OMRequest.newBuilder() .setCmdType(Type.SnapshotMoveDeletedKeys) - .setSnapshotMoveDeletedKeysRequest(moveDeletedKeys) + .setSnapshotMoveTableKeysRequest(moveDeletedKeys) .setClientId(clientId.toString()) .build(); From 4ff6a6eaebb7c74445ed4f5a74a7fc40a6ed7edc Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 19:38:04 -0700 Subject: [PATCH 11/22] HDDS-11440. Address review comments Change-Id: I414e862601ceab9dff051484c3bba72546e0bf5a --- .../apache/hadoop/hdds/utils/TransactionInfo.java | 12 ++++-------- .../apache/hadoop/ozone/om/helpers/SnapshotInfo.java | 11 +++-------- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 2 +- .../key/OMDirectoriesPurgeRequestWithFSO.java | 6 ++++-- .../hadoop/ozone/om/snapshot/SnapshotUtils.java | 2 +- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java index 68d05af0c6ff..d7b55fc7ae69 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Objects; +import com.google.protobuf.ByteString; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.DelegatedCodec; import org.apache.hadoop.hdds.utils.db.StringCodec; @@ -162,16 +163,11 @@ public String toString() { */ public static TransactionInfo readTransactionInfo( DBStoreHAManager metadataManager) throws IOException { - return readTransactionInfo(metadataManager, false); + return metadataManager.getTransactionInfoTable().getSkipCache(TRANSACTION_INFO_KEY); } - /** - * Return transaction info persisted in OM DB skipping cache. - */ - public static TransactionInfo readTransactionInfo( - DBStoreHAManager metadataManager, boolean skipCache) throws IOException { - return skipCache ? metadataManager.getTransactionInfoTable().getSkipCache(TRANSACTION_INFO_KEY) : - metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); + public ByteString toByteString() throws IOException { + return ByteString.copyFrom(getCodec().toPersistedFormat(this)); } public SnapshotInfo toSnapshotInfo() { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index 54af86348937..8584796c2e95 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -417,11 +417,6 @@ public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) { return this; } - public Builder setLastTransactionInfo(byte[] lastTransactionInfo) { - this.lastTransactionInfo = lastTransactionInfo == null ? null : ByteString.copyFrom(lastTransactionInfo); - return this; - } - public Builder setLastTransactionInfo(ByteString lastTransactionInfo) { this.lastTransactionInfo = lastTransactionInfo; return this; @@ -534,7 +529,7 @@ public static SnapshotInfo getFromProtobuf( } if (snapshotInfoProto.hasLastTransactionInfo()) { - osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo().toByteArray()); + osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo()); } osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) @@ -633,8 +628,8 @@ public ByteString getLastTransactionInfo() { return lastTransactionInfo; } - public void setLastTransactionInfo(byte[] lastTransactionInfo) { - this.lastTransactionInfo = lastTransactionInfo == null ? null : ByteString.copyFrom(lastTransactionInfo); + public void setLastTransactionInfo(ByteString lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 5c9333859a2c..f946335da649 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -702,7 +702,7 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa if (snapshotInfo != null) { TransactionInfo snapshotTransactionInfo = snapshotInfo.getLastTransactionInfo() != null ? TransactionInfo.getCodec().fromPersistedFormat(snapshotInfo.getLastTransactionInfo().toByteArray()) : null; - TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager, true); + TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager); return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; } return false; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index 422c872b4e18..632b78f77ba8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.om.response.key.OMDirectoriesPurgeResponseWithFSO; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; @@ -70,8 +71,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn Map, OmBucketInfo> volBucketInfoMap = new HashMap<>(); OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); Map openKeyInfoMap = new HashMap<>(); - OzoneManagerProtocolProtos.OMResponse.Builder omResponse = - OmResponseUtil.getOMResponseBuilder(getOmRequest()); + OMMetrics omMetrics = ozoneManager.getMetrics(); try { if (fromSnapshot != null) { @@ -169,6 +169,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } } + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( + getOmRequest()); OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO( omResponse.build(), purgeRequests, ozoneManager.isRatisEnabled(), getBucketLayout(), volBucketInfoMap, fromSnapshotInfo, openKeyInfoMap); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index 94f1a4e7fce8..aa7cd6e3ba2b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -145,7 +145,7 @@ public static void checkSnapshotActive(SnapshotInfo snapInfo, */ public static void setTransactionInfoInSnapshot(SnapshotInfo snapshot, TermIndex termIndex) throws IOException { TransactionInfo transactionInfo = TransactionInfo.valueOf(termIndex); - snapshot.setLastTransactionInfo(TransactionInfo.getCodec().toPersistedFormat(transactionInfo)); + snapshot.setLastTransactionInfo(transactionInfo.toByteString()); } /** From dc55f37442c68eb0dd6ea4561e84c58a73291562 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 19:47:11 -0700 Subject: [PATCH 12/22] HDDS-11440. Address review comments Change-Id: I55b6d5f4503d3a427ddd13c6e6ae4eac091655e0 --- .../java/org/apache/hadoop/hdds/utils/TransactionInfo.java | 4 ++++ .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java index d7b55fc7ae69..29531f315184 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java @@ -170,6 +170,10 @@ public ByteString toByteString() throws IOException { return ByteString.copyFrom(getCodec().toPersistedFormat(this)); } + public static TransactionInfo fromByteString(ByteString byteString) throws IOException { + return byteString == null ? null : getCodec().fromPersistedFormat(byteString.toByteArray()); + } + public SnapshotInfo toSnapshotInfo() { return snapshotInfo; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index f946335da649..b63f129a412c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -701,7 +701,7 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa throws IOException { if (snapshotInfo != null) { TransactionInfo snapshotTransactionInfo = snapshotInfo.getLastTransactionInfo() != null ? - TransactionInfo.getCodec().fromPersistedFormat(snapshotInfo.getLastTransactionInfo().toByteArray()) : null; + TransactionInfo.fromByteString(snapshotInfo.getLastTransactionInfo()) : null; TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager); return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; } From e55612959663bf46b217604bc74cee94c14964be Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 19:59:15 -0700 Subject: [PATCH 13/22] HDDS-11440. Address review comments Change-Id: I34decc69bf5b4a1c5d5a5e58865c9ede74dae2dc --- .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index b63f129a412c..02db3adbc949 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -694,7 +694,8 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa * @param metadataManager Metadatamanager of Active OM. * @param snapshotInfo table key corresponding to snapshot in snapshotInfoTable, this should be a value from cache * and not from disk. - * @return True if the changes have been flushed to DB otherwise false + * @return True if the changes have been flushed to DB otherwise false. It would return true if the snapshot + * provided is null meaning the snapshot doesn't exist. * @throws IOException */ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataManager, SnapshotInfo snapshotInfo) @@ -703,9 +704,10 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa TransactionInfo snapshotTransactionInfo = snapshotInfo.getLastTransactionInfo() != null ? TransactionInfo.fromByteString(snapshotInfo.getLastTransactionInfo()) : null; TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager); + // If transactionInfo field is null then return true to keep things backward compatible. return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; } - return false; + return true; } From b32d074af41f13bb8c5975612939be2b9871edd4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 20:01:06 -0700 Subject: [PATCH 14/22] HDDS-11440. Address review comments Change-Id: Idbaf521397c170b381530f86f7fe9f215db5169d --- .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 02db3adbc949..dde5b22e7937 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -692,8 +692,7 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa /** * Checks if the last transaction performed on the snapshot has been flushed to disk. * @param metadataManager Metadatamanager of Active OM. - * @param snapshotInfo table key corresponding to snapshot in snapshotInfoTable, this should be a value from cache - * and not from disk. + * @param snapshotInfo SnapshotInfo value. * @return True if the changes have been flushed to DB otherwise false. It would return true if the snapshot * provided is null meaning the snapshot doesn't exist. * @throws IOException From dc83f8522fd4128c78a469a07d7527aef170dcc4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 20:21:10 -0700 Subject: [PATCH 15/22] Merge with HDDS-11440 Change-Id: Ia5140a522ae658684068dbe3dca2f5b617d3e904 --- .../hadoop/ozone/om/service/AbstractKeyDeletingService.java | 3 +-- .../hadoop/ozone/om/service/DirectoryDeletingService.java | 3 +-- .../hadoop/ozone/om/service/SnapshotDeletingService.java | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index 8e5a86299b4d..64eb2977380c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -1014,8 +1014,7 @@ private List getLastNSnapshotInChain(String volume, String bucket) snapshotInfos.add(snapshotInfo); // If changes made to the snapshot have not been flushed to disk, throw exception immediately, next run of // garbage collection would process the snapshot. - if (snapshotInfo != null && - !OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapshotInfo)) { + if (!OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapshotInfo)) { throw new IOException("Changes made to the snapshot " + snapshotInfo + " have not been flushed to the disk "); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index a5dda6680dd9..1d11ca9173aa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -340,8 +340,7 @@ public BackgroundTaskResult call() { try { SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(getOzoneManager(), chainManager, snapshotId); // Wait for snapshot changes to be flushed to disk. - if (!OmSnapshotManager.areSnapshotChangesFlushedToDB( - getOzoneManager().getMetadataManager(), snapInfo)) { + if (!OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapInfo)) { LOG.info("Skipping snapshot processing since changes to snapshot {} have not been flushed to disk", snapInfo); continue; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 9511ebb10063..d389f6e867be 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -148,7 +148,8 @@ public BackgroundTaskResult call() throws InterruptedException { long snapshotLimit = snapshotDeletionPerTask; while (iterator.hasNext() && snapshotLimit > 0) { SnapshotInfo snapInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, snapshotChainManager, iterator.next()); - // Only Iterate in deleted snapshot & only if all the changes have been flushed into disk. + // Only Iterate in deleted snapshot & only if all the changes have been flushed into disk & snapshots have + // been deep cleaned. if (shouldIgnoreSnapshot(snapInfo)) { continue; } From b5f21a0074e078ff3df3a8c7cdd5933b6312ee30 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 12 Sep 2024 15:26:34 -0700 Subject: [PATCH 16/22] HDDS-11411. Snapshot garbage collection should not run when the keys are moved from a deleted snapshot to the next snapshot in the chain Change-Id: Ib48b8aae241efff80fbc8636d3c5e72a49a08a30 --- .../src/main/resources/ozone-default.xml | 9 ++ .../apache/hadoop/ozone/om/OMConfigKeys.java | 2 + .../src/main/proto/OmClientProtocol.proto | 8 ++ .../hadoop/ozone/om/KeyManagerImpl.java | 8 +- .../ozone/om/OmMetadataManagerImpl.java | 75 ++++-------- .../key/OMDirectoriesPurgeRequestWithFSO.java | 45 +++++-- .../om/request/key/OMKeyPurgeRequest.java | 58 ++++++--- .../OMSnapshotMoveDeletedKeysRequest.java | 4 +- .../snapshot/OMSnapshotPurgeRequest.java | 6 +- .../OMDirectoriesPurgeResponseWithFSO.java | 4 + .../service/AbstractKeyDeletingService.java | 50 ++++---- .../om/service/DirectoryDeletingService.java | 36 +++++- .../ozone/om/service/KeyDeletingService.java | 26 ++-- .../om/service/SnapshotDeletingService.java | 7 +- .../SnapshotDirectoryCleaningService.java | 5 +- .../ozone/om/snapshot/SnapshotUtils.java | 112 +++++++++++++++--- 16 files changed, 303 insertions(+), 152 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index fd601e1a7d3e..2d8dfa212db1 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3743,6 +3743,15 @@ + + ozone.snapshot.deep.cleaning.enabled + true + OZONE, PERFORMANCE, OM + + Flag to enable/disable snapshot deep cleaning. + + + ozone.scm.event.ContainerReport.thread.pool.size 10 diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 46becc9e64b5..161b1c555ff2 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -403,6 +403,8 @@ private OMConfigKeys() { /** * Configuration properties for Snapshot Directory Service. */ + public static final String OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED = "ozone.snapshot.deep.cleaning.enabled"; + public static final boolean OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED_DEFAULT = true; public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL = "ozone.snapshot.directory.service.interval"; public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL_DEFAULT diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index eefcfa7552ca..04fec5399ddf 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1379,6 +1379,8 @@ message PurgeKeysRequest { // if set, will purge keys in a snapshot DB instead of active DB optional string snapshotTableKey = 2; repeated SnapshotMoveKeyInfos keysToUpdate = 3; + // previous snapshotID can also be null & this field would be absent in older requests. + optional NullableUUID expectedPreviousSnapshotID = 4; } message PurgeKeysResponse { @@ -1401,6 +1403,12 @@ message PurgePathsResponse { message PurgeDirectoriesRequest { repeated PurgePathRequest deletedPath = 1; optional string snapshotTableKey = 2; + // previous snapshotID can also be null & this field would be absent in older requests. + optional NullableUUID expectedPreviousSnapshotID = 3; +} + +message NullableUUID { + optional hadoop.hdds.UUID uuid = 1; } message PurgeDirectoriesResponse { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 6d276d95284e..9c58da899502 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -141,6 +141,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_TIMEOUT_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DIRECTORY_SERVICE_TIMEOUT; @@ -228,6 +230,8 @@ public KeyManagerImpl(OzoneManager om, ScmClient scmClient, @Override public void start(OzoneConfiguration configuration) { + boolean isSnapshotDeepCleaningEnabled = configuration.getBoolean(OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED, + OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED_DEFAULT); if (keyDeletingService == null) { long blockDeleteInterval = configuration.getTimeDuration( OZONE_BLOCK_DELETING_SERVICE_INTERVAL, @@ -239,7 +243,7 @@ public void start(OzoneConfiguration configuration) { TimeUnit.MILLISECONDS); keyDeletingService = new KeyDeletingService(ozoneManager, scmClient.getBlockClient(), this, blockDeleteInterval, - serviceTimeout, configuration); + serviceTimeout, configuration, isSnapshotDeepCleaningEnabled); keyDeletingService.start(); } @@ -312,7 +316,7 @@ public void start(OzoneConfiguration configuration) { } } - if (snapshotDirectoryCleaningService == null && + if (isSnapshotDeepCleaningEnabled && snapshotDirectoryCleaningService == null && ozoneManager.isFilesystemSnapshotEnabled()) { long dirDeleteInterval = configuration.getTimeDuration( OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index ee92dbc2fde9..9f5a488e3c34 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -113,6 +113,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; @@ -1595,11 +1596,20 @@ public PendingKeysDeletion getPendingDeletionKeys(final int keyCount, String[] keySplit = kv.getKey().split(OM_KEY_PREFIX); String bucketKey = getBucketKey(keySplit[1], keySplit[2]); OmBucketInfo bucketInfo = getBucketTable().get(bucketKey); - + SnapshotInfo previousSnapshotInfo = SnapshotUtils.getLatestSnapshotInfo(bucketInfo.getVolumeName(), + bucketInfo.getBucketName(), ozoneManager, snapshotChainManager); + // previous snapshot is not active or it has not been flushed to disk then don't process the key in this + // iteration. + if (previousSnapshotInfo != null && + (previousSnapshotInfo.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE || + !OmSnapshotManager.areSnapshotChangesFlushedToDB(ozoneManager.getMetadataManager(), + previousSnapshotInfo))) { + continue; + } // Get the latest snapshot in snapshot path. - try (ReferenceCounted - rcLatestSnapshot = getLatestActiveSnapshot( - keySplit[1], keySplit[2], omSnapshotManager)) { + try (ReferenceCounted rcLatestSnapshot = previousSnapshotInfo == null ? null : + omSnapshotManager.getSnapshot(previousSnapshotInfo.getVolumeName(), + previousSnapshotInfo.getBucketName(), previousSnapshotInfo.getName())) { // Multiple keys with the same path can be queued in one DB entry RepeatedOmKeyInfo infoList = kv.getValue(); @@ -1688,6 +1698,14 @@ public PendingKeysDeletion getPendingDeletionKeys(final int keyCount, infoList.getOmKeyInfoList().size()) { keyBlocksList.addAll(blockGroupList); } + SnapshotInfo newPreviousSnapshotInfo = SnapshotUtils.getLatestSnapshotInfo(bucketInfo.getVolumeName(), + bucketInfo.getBucketName(), ozoneManager, snapshotChainManager); + if (!Objects.equals(Optional.ofNullable(newPreviousSnapshotInfo).map(SnapshotInfo::getSnapshotId), + Optional.ofNullable(previousSnapshotInfo).map(SnapshotInfo::getSnapshotId))) { + throw new OMException("Snapshot chain has changed while checking for key reference " + + "Previous snapshot in chain : " + previousSnapshotInfo + " new snapshot in chain : " + + newPreviousSnapshotInfo, INTERNAL_ERROR); + } } } } @@ -1703,55 +1721,6 @@ private boolean versionExistsInPreviousSnapshot(OmKeyInfo omKeyInfo, delOmKeyInfo != null; } - /** - * Get the latest OmSnapshot for a snapshot path. - */ - public ReferenceCounted getLatestActiveSnapshot( - String volumeName, String bucketName, - OmSnapshotManager snapshotManager) - throws IOException { - - String snapshotPath = volumeName + OM_KEY_PREFIX + bucketName; - Optional latestPathSnapshot = Optional.ofNullable( - snapshotChainManager.getLatestPathSnapshotId(snapshotPath)); - - Optional snapshotInfo = Optional.empty(); - - while (latestPathSnapshot.isPresent()) { - Optional snapTableKey = latestPathSnapshot - .map(uuid -> snapshotChainManager.getTableKey(uuid)); - - snapshotInfo = snapTableKey.isPresent() ? - Optional.ofNullable(getSnapshotInfoTable().get(snapTableKey.get())) : - Optional.empty(); - - if (snapshotInfo.isPresent() && snapshotInfo.get().getSnapshotStatus() == - SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { - break; - } - - // Update latestPathSnapshot if current snapshot is deleted. - if (snapshotChainManager.hasPreviousPathSnapshot(snapshotPath, - latestPathSnapshot.get())) { - latestPathSnapshot = Optional.ofNullable(snapshotChainManager - .previousPathSnapshot(snapshotPath, latestPathSnapshot.get())); - } else { - latestPathSnapshot = Optional.empty(); - } - } - - Optional> rcOmSnapshot = - snapshotInfo.isPresent() ? - Optional.ofNullable( - snapshotManager.getSnapshot(volumeName, - bucketName, - snapshotInfo.get().getName()) - ) : - Optional.empty(); - - return rcOmSnapshot.orElse(null); - } - /** * Decide whether the open key is a multipart upload related key. * @param openKeyInfo open key related to multipart upload diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index dd08ff171654..a63b63725b12 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -23,13 +23,21 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.UUID; + +import com.google.common.collect.Maps; +import org.apache.commons.compress.utils.Lists; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -45,8 +53,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.validatePreviousSnapshotId; /** * Handles purging of keys from OM DB. @@ -66,19 +76,34 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn List purgeRequests = purgeDirsRequest.getDeletedPathList(); - - SnapshotInfo fromSnapshotInfo = null; Set> lockSet = new HashSet<>(); Map, OmBucketInfo> volBucketInfoMap = new HashMap<>(); - OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); Map openKeyInfoMap = new HashMap<>(); - OMMetrics omMetrics = ozoneManager.getMetrics(); + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( + getOmRequest()); + final SnapshotInfo fromSnapshotInfo; try { - if (fromSnapshot != null) { - fromSnapshotInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot); + fromSnapshotInfo = fromSnapshot != null ? SnapshotUtils.getSnapshotInfo(ozoneManager, + fromSnapshot) : null; + // Checking if this request is an old request or new one. + if (purgeDirsRequest.hasExpectedPreviousSnapshotID()) { + // Validating previous snapshot since while purging deletes, a snapshot create request could make this purge + // directory request invalid on AOS since the deletedDirectory would be in the newly created snapshot. Adding + // subdirectories could lead to not being able to reclaim sub-files and subdirectories since the + // file/directory would be present in the newly created snapshot. + // Validating previous snapshot can ensure the chain hasn't changed. + UUID expectedPreviousSnapshotId = purgeDirsRequest.getExpectedPreviousSnapshotID().hasUuid() + ? fromProtobuf(purgeDirsRequest.getExpectedPreviousSnapshotID().getUuid()) : null; + validatePreviousSnapshotId(fromSnapshotInfo, omMetadataManager.getSnapshotChainManager(), + expectedPreviousSnapshotId); } - + } catch (IOException e) { + LOG.error("Error occured while performing OMDirectoriesPurge. ", e); + return new OMDirectoriesPurgeResponseWithFSO(createErrorOMResponse(omResponse, e)); + } + try { for (OzoneManagerProtocolProtos.PurgePathRequest path : purgeRequests) { for (OzoneManagerProtocolProtos.KeyInfo key : path.getMarkDeletedSubDirsList()) { @@ -170,12 +195,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } } - OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( - getOmRequest()); - OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO( + return new OMDirectoriesPurgeResponseWithFSO( omResponse.build(), purgeRequests, ozoneManager.isRatisEnabled(), getBucketLayout(), volBucketInfoMap, fromSnapshotInfo, openKeyInfoMap); - - return omClientResponse; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java index 14c80bb7a93b..0719d6909b67 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OzoneManager; @@ -42,6 +43,10 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.UUID; + +import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.validatePreviousSnapshotId; /** * Handles purging of keys from OM DB. @@ -58,30 +63,46 @@ public OMKeyPurgeRequest(OMRequest omRequest) { @Override public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIndex termIndex) { PurgeKeysRequest purgeKeysRequest = getOmRequest().getPurgeKeysRequest(); - List bucketDeletedKeysList = purgeKeysRequest - .getDeletedKeysList(); - List keysToUpdateList = purgeKeysRequest - .getKeysToUpdateList(); - String fromSnapshot = purgeKeysRequest.hasSnapshotTableKey() ? - purgeKeysRequest.getSnapshotTableKey() : null; - List keysToBePurgedList = new ArrayList<>(); + List bucketDeletedKeysList = purgeKeysRequest.getDeletedKeysList(); + List keysToUpdateList = purgeKeysRequest.getKeysToUpdateList(); + String fromSnapshot = purgeKeysRequest.hasSnapshotTableKey() ? purgeKeysRequest.getSnapshotTableKey() : null; OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); - OMClientResponse omClientResponse = null; - for (DeletedKeys bucketWithDeleteKeys : bucketDeletedKeysList) { - for (String deletedKey : bucketWithDeleteKeys.getKeysList()) { - keysToBePurgedList.add(deletedKey); + + final SnapshotInfo fromSnapshotInfo; + try { + fromSnapshotInfo = fromSnapshot != null ? SnapshotUtils.getSnapshotInfo(ozoneManager, + fromSnapshot) : null; + // Checking if this request is an old request or new one. + if (purgeKeysRequest.hasExpectedPreviousSnapshotID()) { + // Validating previous snapshot since while purging deletes, a snapshot create request could make this purge + // directory request invalid on AOS since the deletedDirectory would be in the newly created snapshot. Adding + // subdirectories could lead to not being able to reclaim sub-files and subdirectories since the + // file/directory would be present in the newly created snapshot. + // Validating previous snapshot can ensure the chain hasn't changed. + UUID expectedPreviousSnapshotId = purgeKeysRequest.getExpectedPreviousSnapshotID().hasUuid() + ? fromProtobuf(purgeKeysRequest.getExpectedPreviousSnapshotID().getUuid()) : null; + validatePreviousSnapshotId(fromSnapshotInfo, omMetadataManager.getSnapshotChainManager(), + expectedPreviousSnapshotId); } + } catch (IOException e) { + LOG.error("Error occured while performing OMDirectoriesPurge. ", e); + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, e)); } - final SnapshotInfo fromSnapshotInfo; - try { - fromSnapshotInfo = fromSnapshot == null ? null : SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot); - } catch (IOException ex) { - return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, ex)); + List keysToBePurgedList = new ArrayList<>(); + + for (DeletedKeys bucketWithDeleteKeys : bucketDeletedKeysList) { + keysToBePurgedList.addAll(bucketWithDeleteKeys.getKeysList()); + } + + if (keysToBePurgedList.isEmpty()) { + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, + new OMException("None of the keys can be purged be purged since a new snapshot was created for all the " + + "buckets, making this request invalid", OMException.ResultCodes.KEY_DELETION_ERROR))); } // Setting transaction info for snapshot, this is to prevent duplicate purge requests to OM from background @@ -95,10 +116,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } catch (IOException e) { return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, e)); } - omClientResponse = new OMKeyPurgeResponse(omResponse.build(), keysToBePurgedList, fromSnapshotInfo, - keysToUpdateList); - return omClientResponse; + return new OMKeyPurgeResponse(omResponse.build(), + keysToBePurgedList, fromSnapshotInfo, keysToUpdateList); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index 58fdb1232d31..18055bdda40c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -80,9 +80,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OmResponseUtil.getOMResponseBuilder(getOmRequest()); try { // Check the snapshot exists. - SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot.getTableKey()); + SnapshotInfo snapshotInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot.getTableKey()); - nextSnapshot = SnapshotUtils.getNextActiveSnapshot(fromSnapshot, snapshotChainManager, ozoneManager); + nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, snapshotInfo); // Get next non-deleted snapshot. List nextDBKeysList = moveDeletedKeysRequest.getNextDBKeysList(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index 6602f52514b5..38c51d4de5c0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -103,9 +103,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn "Snapshot purge request.", snapTableKey); continue; } - - SnapshotInfo nextSnapshot = - SnapshotUtils.getNextActiveSnapshot(fromSnapshot, snapshotChainManager, ozoneManager); + SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); // Step 1: Update the deep clean flag for the next active snapshot updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, trxnLogIndex); @@ -116,7 +114,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn .addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), CacheValue.get(trxnLogIndex)); updatedSnapshotInfos.remove(fromSnapshot.getTableKey()); } - + // Update the snapshotInfo lastTransactionInfo. for (SnapshotInfo snapshotInfo : updatedSnapshotInfos.values()) { snapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapshotInfo.getTableKey()), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 28c3e3d758e2..782063d32446 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -78,6 +78,10 @@ public OMDirectoriesPurgeResponseWithFSO(@Nonnull OMResponse omResponse, this.openKeyInfoMap = openKeyInfoMap; } + public OMDirectoriesPurgeResponseWithFSO(OMResponse omResponse) { + super(omResponse); + } + @Override public void addToDBBatch(OMMetadataManager metadataManager, BatchOperation batchOp) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index 2c2d16bf14c7..845ae34ed0b9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ServiceException; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.utils.BackgroundService; @@ -97,7 +98,7 @@ public AbstractKeyDeletingService(String serviceName, long interval, protected int processKeyDeletes(List keyBlocksList, KeyManager manager, HashMap keysToModify, - String snapTableKey) throws IOException { + String snapTableKey, UUID expectedPreviousSnapshotId) throws IOException { long startTime = Time.monotonicNow(); int delCount = 0; @@ -120,7 +121,7 @@ protected int processKeyDeletes(List keyBlocksList, startTime = Time.monotonicNow(); if (isRatisEnabled()) { delCount = submitPurgeKeysRequest(blockDeletionResults, - keysToModify, snapTableKey); + keysToModify, snapTableKey, expectedPreviousSnapshotId); } else { // TODO: Once HA and non-HA paths are merged, we should have // only one code path here. Purge keys should go through an @@ -172,7 +173,7 @@ private int deleteAllKeys(List results, * @param keysToModify Updated list of RepeatedOmKeyInfo */ private int submitPurgeKeysRequest(List results, - HashMap keysToModify, String snapTableKey) { + HashMap keysToModify, String snapTableKey, UUID expectedPreviousSnapshotId) { Map, List> purgeKeysMapPerBucket = new HashMap<>(); @@ -203,6 +204,15 @@ private int submitPurgeKeysRequest(List results, if (snapTableKey != null) { purgeKeysRequest.setSnapshotTableKey(snapTableKey); } + OzoneManagerProtocolProtos.NullableUUID.Builder expectedPreviousSnapshotNullableUUID = + OzoneManagerProtocolProtos.NullableUUID.newBuilder(); + if (expectedPreviousSnapshotId != null) { + expectedPreviousSnapshotNullableUUID.setUuid(HddsUtils.toProtobuf(expectedPreviousSnapshotId)); + } + + if (expectedPreviousSnapshotId != null) { + purgeKeysRequest.setExpectedPreviousSnapshotID(expectedPreviousSnapshotNullableUUID); + } // Add keys to PurgeKeysRequest bucket wise. for (Map.Entry, List> entry : @@ -274,13 +284,21 @@ private void addToMap(Map, List> map, String object } protected void submitPurgePaths(List requests, - String snapTableKey) { + String snapTableKey, + UUID expectedPreviousSnapshotId) { OzoneManagerProtocolProtos.PurgeDirectoriesRequest.Builder purgeDirRequest = OzoneManagerProtocolProtos.PurgeDirectoriesRequest.newBuilder(); if (snapTableKey != null) { purgeDirRequest.setSnapshotTableKey(snapTableKey); } + OzoneManagerProtocolProtos.NullableUUID.Builder expectedPreviousSnapshotNullableUUID = + OzoneManagerProtocolProtos.NullableUUID.newBuilder(); + if (expectedPreviousSnapshotId != null) { + expectedPreviousSnapshotNullableUUID.setUuid(HddsUtils.toProtobuf(expectedPreviousSnapshotId)); + } + purgeDirRequest.setExpectedPreviousSnapshotID(expectedPreviousSnapshotNullableUUID.build()); + purgeDirRequest.addAllDeletedPath(requests); OzoneManagerProtocolProtos.OMRequest omRequest = @@ -386,7 +404,8 @@ public long optimizeDirDeletesAndSubmitRequest(long remainNum, List> allSubDirList, List purgePathRequestList, String snapTableKey, long startTime, - int remainingBufLimit, KeyManager keyManager) { + int remainingBufLimit, KeyManager keyManager, + UUID expectedPreviousSnapshotId) { // Optimization to handle delete sub-dir and keys to remove quickly // This case will be useful to handle when depth of directory is high @@ -426,7 +445,7 @@ public long optimizeDirDeletesAndSubmitRequest(long remainNum, } if (!purgePathRequestList.isEmpty()) { - submitPurgePaths(purgePathRequestList, snapTableKey); + submitPurgePaths(purgePathRequestList, snapTableKey, expectedPreviousSnapshotId); } if (dirNum != 0 || subDirNum != 0 || subFileNum != 0) { @@ -549,25 +568,6 @@ protected boolean isBufferLimitCrossed( return cLimit + increment >= maxLimit; } - protected SnapshotInfo getPreviousActiveSnapshot(SnapshotInfo snapInfo, SnapshotChainManager chainManager) - throws IOException { - SnapshotInfo currSnapInfo = snapInfo; - while (chainManager.hasPreviousPathSnapshot( - currSnapInfo.getSnapshotPath(), currSnapInfo.getSnapshotId())) { - - UUID prevPathSnapshot = chainManager.previousPathSnapshot( - currSnapInfo.getSnapshotPath(), currSnapInfo.getSnapshotId()); - String tableKey = chainManager.getTableKey(prevPathSnapshot); - SnapshotInfo prevSnapInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, tableKey); - if (prevSnapInfo.getSnapshotStatus() == - SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { - return prevSnapInfo; - } - currSnapInfo = prevSnapInfo; - } - return null; - } - protected boolean isKeyReclaimable( Table previousKeyTable, Table renamedTable, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index c8703c3c4c62..494457e8bffa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -31,9 +31,12 @@ import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; @@ -43,11 +46,15 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR; /** * This is a background service to delete orphan directories and its @@ -155,9 +162,15 @@ public BackgroundTaskResult call() { = new ArrayList<>((int) remainNum); Table.KeyValue pendingDeletedDirInfo; + try (TableIterator> deleteTableIterator = getOzoneManager().getMetadataManager(). getDeletedDirTable().iterator()) { + // This is to avoid race condition b/w purge request and snapshot chain updation. For AOS taking the global + // snapshotId since AOS could process multiple buckets in one iteration. + UUID expectedPreviousSnapshotId = + ((OmMetadataManagerImpl)getOzoneManager().getMetadataManager()).getSnapshotChainManager() + .getLatestGlobalSnapshotId(); long startTime = Time.monotonicNow(); while (remainNum > 0 && deleteTableIterator.hasNext()) { @@ -204,7 +217,7 @@ public BackgroundTaskResult call() { remainNum, dirNum, subDirNum, subFileNum, allSubDirList, purgePathRequestList, null, startTime, ratisByteLimit - consumedSize, - getOzoneManager().getKeyManager()); + getOzoneManager().getKeyManager(), expectedPreviousSnapshotId); } catch (IOException e) { LOG.error("Error while running delete directories and files " + @@ -224,12 +237,21 @@ private boolean previousSnapshotHasDir( getOzoneManager().getOmSnapshotManager(); OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) getOzoneManager().getMetadataManager(); - + SnapshotInfo previousSnapshotInfo = SnapshotUtils.getLatestSnapshotInfo(deletedDirInfo.getVolumeName(), + deletedDirInfo.getBucketName(), getOzoneManager(), metadataManager.getSnapshotChainManager()); + // previous snapshot is not active or it has not been flushed to disk then don't process the key in this + // iteration. + if (previousSnapshotInfo != null && + (previousSnapshotInfo.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE || + !OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), + previousSnapshotInfo))) { + return true; + } try (ReferenceCounted rcLatestSnapshot = - metadataManager.getLatestActiveSnapshot( + omSnapshotManager.getSnapshot( deletedDirInfo.getVolumeName(), deletedDirInfo.getBucketName(), - omSnapshotManager)) { + previousSnapshotInfo.getName())) { if (rcLatestSnapshot != null) { String dbRenameKey = metadataManager @@ -250,7 +272,11 @@ private boolean previousSnapshotHasDir( String prevDbKey = prevDirTableDBKey == null ? metadataManager.getOzoneDeletePathDirKey(key) : prevDirTableDBKey; OmDirectoryInfo prevDirInfo = prevDirTable.get(prevDbKey); - return prevDirInfo != null && + //Check the snapshot chain hasn't changed while the checking the previous snapshot. + SnapshotInfo newPreviousSnapshotInfo = SnapshotUtils.getLatestSnapshotInfo(deletedDirInfo.getVolumeName(), + deletedDirInfo.getBucketName(), getOzoneManager(), metadataManager.getSnapshotChainManager()); + return Objects.equals(Optional.ofNullable(newPreviousSnapshotInfo).map(SnapshotInfo::getSnapshotId), + Optional.ofNullable(previousSnapshotInfo).map(SnapshotInfo::getSnapshotId)) && prevDirInfo != null && prevDirInfo.getObjectID() == deletedDirInfo.getObjectID(); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 5e622cb17019..2d886769efbe 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -38,12 +39,14 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; @@ -92,11 +95,13 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private final Map exclusiveReplicatedSizeMap; private final Set completedExclusiveSizeSet; private final Map snapshotSeekMap; + private final boolean deepCleanSnapshots; public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, KeyManager manager, long serviceInterval, - long serviceTimeout, ConfigurationSource conf) { + long serviceTimeout, ConfigurationSource conf, + boolean deepCleanSnapshots) { super(KeyDeletingService.class.getSimpleName(), serviceInterval, TimeUnit.MILLISECONDS, KEY_DELETING_CORE_POOL_SIZE, serviceTimeout, ozoneManager, scmClient); @@ -111,6 +116,7 @@ public KeyDeletingService(OzoneManager ozoneManager, this.exclusiveReplicatedSizeMap = new HashMap<>(); this.completedExclusiveSizeSet = new HashSet<>(); this.snapshotSeekMap = new HashMap<>(); + this.deepCleanSnapshots = deepCleanSnapshots; } /** @@ -191,7 +197,11 @@ public BackgroundTaskResult call() { // doesn't have enough entries left. // OM would have to keep track of which snapshot the key is coming // from if the above would be done inside getPendingDeletionKeys(). - + // This is to avoid race condition b/w purge request and snapshot chain updation. For AOS taking the global + // snapshotId since AOS could process multiple buckets in one iteration. + UUID expectedPreviousSnapshotId = + ((OmMetadataManagerImpl)manager.getMetadataManager()).getSnapshotChainManager() + .getLatestGlobalSnapshotId(); PendingKeysDeletion pendingKeysDeletion = manager .getPendingDeletionKeys(getKeyLimitPerTask()); List keyBlocksList = pendingKeysDeletion @@ -199,7 +209,7 @@ public BackgroundTaskResult call() { if (keyBlocksList != null && !keyBlocksList.isEmpty()) { delCount = processKeyDeletes(keyBlocksList, getOzoneManager().getKeyManager(), - pendingKeysDeletion.getKeysToModify(), null); + pendingKeysDeletion.getKeysToModify(), null, expectedPreviousSnapshotId); deletedKeyCount.addAndGet(delCount); } } catch (IOException e) { @@ -208,7 +218,7 @@ public BackgroundTaskResult call() { } try { - if (delCount < keyLimitPerTask) { + if (deepCleanSnapshots && delCount < keyLimitPerTask) { processSnapshotDeepClean(delCount); } } catch (Exception e) { @@ -276,11 +286,13 @@ private void processSnapshotDeepClean(int delCount) } String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; - SnapshotInfo previousSnapshot = getPreviousActiveSnapshot(currSnapInfo, snapChainManager); + SnapshotInfo previousSnapshot = SnapshotUtils.getPreviousSnapshot(getOzoneManager(), snapChainManager, + currSnapInfo); SnapshotInfo previousToPrevSnapshot = null; if (previousSnapshot != null) { - previousToPrevSnapshot = getPreviousActiveSnapshot(previousSnapshot, snapChainManager); + previousToPrevSnapshot = SnapshotUtils.getPreviousSnapshot(getOzoneManager(), snapChainManager, + previousSnapshot); } Table previousKeyTable = null; @@ -409,7 +421,7 @@ private void processSnapshotDeepClean(int delCount) if (!keysToPurge.isEmpty()) { processKeyDeletes(keysToPurge, currOmSnapshot.getKeyManager(), - keysToModify, currSnapInfo.getTableKey()); + keysToModify, currSnapInfo.getTableKey(), null); } } finally { IOUtils.closeQuietly(rcPrevOmSnapshot, rcPrevToPrevOmSnapshot); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index f85bd781b050..d7e7f68946ed 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -49,6 +49,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; @@ -204,7 +205,7 @@ public BackgroundTaskResult call() throws InterruptedException { } //TODO: [SNAPSHOT] Add lock to deletedTable and Active DB. - SnapshotInfo previousSnapshot = getPreviousActiveSnapshot(snapInfo, chainManager); + SnapshotInfo previousSnapshot = SnapshotUtils.getPreviousSnapshot(ozoneManager, chainManager, snapInfo); Table previousKeyTable = null; Table previousDirTable = null; OmSnapshot omPreviousSnapshot = null; @@ -298,7 +299,7 @@ public BackgroundTaskResult call() throws InterruptedException { // Delete keys From deletedTable processKeyDeletes(keysToPurge, omSnapshot.getKeyManager(), - null, snapInfo.getTableKey()); + null, snapInfo.getTableKey(), null); successRunCount.incrementAndGet(); } catch (IOException ex) { LOG.error("Error while running Snapshot Deleting Service for " + @@ -436,7 +437,7 @@ private long handleDirectoryCleanUp( remainNum = optimizeDirDeletesAndSubmitRequest(remainNum, dirNum, subDirNum, subFileNum, allSubDirList, purgePathRequestList, snapInfo.getTableKey(), startTime, ratisByteLimit - consumedSize, - omSnapshot.getKeyManager()); + omSnapshot.getKeyManager(), null); } catch (IOException e) { LOG.error("Error while running delete directories and files for " + "snapshot " + snapInfo.getTableKey() + " in snapshot deleting " + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java index 26d5d24a8a03..828eee34283b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java @@ -61,6 +61,7 @@ import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getDirectoryInfo; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getOzonePathKeyForFso; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getPreviousSnapshot; /** * Snapshot BG Service for deleted directory deep clean and exclusive size @@ -173,7 +174,7 @@ public BackgroundTaskResult call() { "unexpected state."); } - SnapshotInfo previousSnapshot = getPreviousActiveSnapshot(currSnapInfo, snapChainManager); + SnapshotInfo previousSnapshot = getPreviousSnapshot(getOzoneManager(), snapChainManager, currSnapInfo); SnapshotInfo previousToPrevSnapshot = null; Table previousKeyTable = null; @@ -190,7 +191,7 @@ public BackgroundTaskResult call() { .getKeyTable(bucketInfo.getBucketLayout()); prevRenamedTable = omPreviousSnapshot .getMetadataManager().getSnapshotRenamedTable(); - previousToPrevSnapshot = getPreviousActiveSnapshot(previousSnapshot, snapChainManager); + previousToPrevSnapshot = getPreviousSnapshot(getOzoneManager(), snapChainManager, previousSnapshot); } Table previousToPrevKeyTable = null; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index e0f40dabd8a7..180ac9952a3d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -36,6 +36,8 @@ import java.util.NoSuchElementException; import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.UUID; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; @@ -86,6 +88,13 @@ public static SnapshotInfo getSnapshotInfo(final OzoneManager ozoneManager, return snapshotInfo; } + public static SnapshotInfo getSnapshotInfo(OzoneManager ozoneManager, + SnapshotChainManager chainManager, + UUID snapshotId) throws IOException { + String tableKey = chainManager.getTableKey(snapshotId); + return SnapshotUtils.getSnapshotInfo(ozoneManager, tableKey); + } + public static void dropColumnFamilyHandle( final ManagedRocksDB rocksDB, final ColumnFamilyHandle columnFamilyHandle) { @@ -139,36 +148,60 @@ public static void checkSnapshotActive(SnapshotInfo snapInfo, } /** - * Get the next non deleted snapshot in the snapshot chain. + * Get the next in the snapshot chain. */ - public static SnapshotInfo getNextActiveSnapshot(SnapshotInfo snapInfo, - SnapshotChainManager chainManager, OzoneManager ozoneManager) + public static SnapshotInfo getNextSnapshot(OzoneManager ozoneManager, + SnapshotChainManager chainManager, + SnapshotInfo snapInfo) throws IOException { - // If the snapshot is deleted in the previous run, then the in-memory // SnapshotChainManager might throw NoSuchElementException as the snapshot // is removed in-memory but OMDoubleBuffer has not flushed yet. if (snapInfo == null) { throw new OMException("Snapshot Info is null. Cannot get the next snapshot", INVALID_SNAPSHOT_ERROR); } - try { - while (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), + if (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId())) { + UUID nextPathSnapshot = chainManager.nextPathSnapshot(snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); + return getSnapshotInfo(ozoneManager, chainManager, nextPathSnapshot); + } + } catch (NoSuchElementException ex) { + LOG.error("The snapshot {} is not longer in snapshot chain, It " + + "maybe removed in the previous Snapshot purge request.", + snapInfo.getTableKey()); + } + return null; + } - UUID nextPathSnapshot = - chainManager.nextPathSnapshot( - snapInfo.getSnapshotPath(), snapInfo.getSnapshotId()); - - String tableKey = chainManager.getTableKey(nextPathSnapshot); - SnapshotInfo nextSnapshotInfo = getSnapshotInfo(ozoneManager, tableKey); - - if (nextSnapshotInfo.getSnapshotStatus().equals( - SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)) { - return nextSnapshotInfo; - } + /** + * Get the previous in the snapshot chain. + */ + public static SnapshotInfo getPreviousSnapshot(OzoneManager ozoneManager, + SnapshotChainManager chainManager, + SnapshotInfo snapInfo) + throws IOException { + UUID previousSnapshotId = getPreviousSnapshotId(snapInfo, chainManager); + return previousSnapshotId == null ? null : getSnapshotInfo(ozoneManager, chainManager, previousSnapshotId); + } - snapInfo = nextSnapshotInfo; + /** + * Get the previous in the snapshot chain. + */ + public static UUID getPreviousSnapshotId(SnapshotInfo snapInfo, + SnapshotChainManager chainManager) + throws IOException { + // If the snapshot is deleted in the previous run, then the in-memory + // SnapshotChainManager might throw NoSuchElementException as the snapshot + // is removed in-memory but OMDoubleBuffer has not flushed yet. + if (snapInfo == null) { + throw new OMException("Snapshot Info is null. Cannot get the next snapshot", INVALID_SNAPSHOT_ERROR); + } + try { + if (chainManager.hasPreviousPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotId())) { + return chainManager.previousPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotId()); } } catch (NoSuchElementException ex) { LOG.error("The snapshot {} is not longer in snapshot chain, It " + @@ -242,4 +275,47 @@ public static String getOzonePathKeyForFso(OMMetadataManager metadataManager, final long bucketId = metadataManager.getBucketId(volumeName, bucketName); return OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId + OM_KEY_PREFIX; } + + public static SnapshotInfo getLatestGlobalSnapshotInfo(OzoneManager ozoneManager, + SnapshotChainManager snapshotChainManager) throws IOException { + Optional latestGlobalSnapshot = Optional.ofNullable(snapshotChainManager.getLatestGlobalSnapshotId()); + return latestGlobalSnapshot.isPresent() ? getSnapshotInfo(ozoneManager, snapshotChainManager, + latestGlobalSnapshot.get()) : null; + } + + public static SnapshotInfo getLatestSnapshotInfo(String volumeName, String bucketName, + OzoneManager ozoneManager, + SnapshotChainManager snapshotChainManager) throws IOException { + Optional latestPathSnapshot = Optional.ofNullable( + getLatestSnapshotId(volumeName, bucketName, snapshotChainManager)); + return latestPathSnapshot.isPresent() ? + getSnapshotInfo(ozoneManager, snapshotChainManager, latestPathSnapshot.get()) : null; + } + + public static UUID getLatestSnapshotId(String volumeName, String bucketName, + SnapshotChainManager snapshotChainManager) throws IOException { + String snapshotPath = volumeName + OM_KEY_PREFIX + bucketName; + return snapshotChainManager.getLatestPathSnapshotId(snapshotPath); + } + + // Validates previous snapshotId given a snapshotInfo or volumeName & bucketName. Incase snapshotInfo is null, this + // would be considered as AOS and previous snapshot becomes the latest snapshot in the global snapshot chain. + // Would throw OMException if validation fails otherwise function would pass. + public static void validatePreviousSnapshotId(SnapshotInfo snapshotInfo, + SnapshotChainManager snapshotChainManager, + UUID expectedPreviousSnapshotId) throws IOException { + try { + UUID previousSnapshotId = snapshotInfo == null ? snapshotChainManager.getLatestGlobalSnapshotId() : + SnapshotUtils.getPreviousSnapshotId(snapshotInfo, snapshotChainManager); + if (!Objects.equals(expectedPreviousSnapshotId, previousSnapshotId)) { + throw new OMException("Snapshot validation failed. Expected previous snapshotId : " + + expectedPreviousSnapshotId + " but was " + previousSnapshotId, + OMException.ResultCodes.INVALID_REQUEST); + } + } catch (IOException e) { + LOG.error("Error while validating previous snapshot for snapshot: {}", + snapshotInfo == null ? null : snapshotInfo.getName(), e); + throw e; + } + } } From 29ab212a96b58b765a765a920dc55be5aa8c429e Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 12 Sep 2024 16:57:50 -0700 Subject: [PATCH 17/22] Merge Change-Id: I04ce37e93e1b072a23f01b28f92a6c71f5d27d07 --- .../hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java deleted file mode 100644 index e69de29bb2d1..000000000000 From d06414e445ab628f13b9248200ac3491cb722aac Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 12 Sep 2024 17:23:35 -0700 Subject: [PATCH 18/22] Merge Change-Id: Id645dcb042055ef53094ddcc8d42e60c037cfadd --- .../om/request/key/OMKeyPurgeRequest.java | 26 -------- .../OMSnapshotMoveTableKeysRequest.java | 5 +- .../service/AbstractKeyDeletingService.java | 59 ++++++------------- .../om/service/DirectoryDeletingService.java | 12 +--- .../ozone/om/service/KeyDeletingService.java | 48 ++------------- .../om/service/SnapshotDeletingService.java | 1 + 6 files changed, 28 insertions(+), 123 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java index 94436c64e152..219c237b960c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java @@ -21,17 +21,11 @@ import java.io.IOException; import java.util.ArrayList; -import com.google.common.collect.Maps; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.hdds.utils.TransactionInfo; -import org.apache.hadoop.hdds.utils.db.cache.CacheKey; -import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OzoneManager; @@ -51,11 +45,6 @@ import java.util.List; import java.util.UUID; -import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; -import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.validatePreviousSnapshotId; -import java.util.Map; -import java.util.UUID; - import static org.apache.hadoop.hdds.HddsUtils.fromProtobuf; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.validatePreviousSnapshotId; @@ -104,21 +93,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn List keysToBePurgedList = new ArrayList<>(); - // Validating previous snapshot since while purging rename keys, a snapshot create request could make this purge - // rename entry invalid on AOS since the key could be now present on the newly created snapshot since a rename - // request could have come through after creation of a snapshot. The purged deletedKey would not be even - // present. - UUID expectedPreviousSnapshotId = purgeKeysRequest.hasExpectedPreviousSnapshotID() - ? fromProtobuf(purgeKeysRequest.getExpectedPreviousSnapshotID()) : null; - try { - validatePreviousSnapshotId(fromSnapshotInfo, omMetadataManager.getSnapshotChainManager(), - expectedPreviousSnapshotId); - } catch (IOException e) { - LOG.error("Previous snapshot validation failed.", e); - return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, e)); - } - - List renamedKeysToBePurged = new ArrayList<>(purgeKeysRequest.getRenamedKeysList()); for (DeletedKeys bucketWithDeleteKeys : bucketDeletedKeysList) { keysToBePurgedList.addAll(bucketWithDeleteKeys.getKeysList()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java index 2cd7fe76a036..505d71b04dbd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.om.request.snapshot; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -98,11 +99,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn .collect(Collectors.toList()); // Update lastTransactionInfo for fromSnapshot and the nextSnapshot. - SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshot, termIndex); + fromSnapshot.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), CacheValue.get(termIndex.getIndex(), fromSnapshot)); if (nextSnapshot != null) { - SnapshotUtils.setTransactionInfoInSnapshot(nextSnapshot, termIndex); + nextSnapshot.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextSnapshot.getTableKey()), CacheValue.get(termIndex.getIndex(), nextSnapshot)); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index b2fb6da5600f..77c8e6033924 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -352,47 +352,6 @@ private Pair submitPurgeKeysRequest(List setSnapshotPropertyRequests) { + OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) + .addAllSetSnapshotPropertyRequests(setSnapshotPropertyRequests) + .setClientId(clientId.toString()) + .build(); + try { + OzoneManagerRatisUtils.submitRequest(getOzoneManager(), omRequest, clientId, getRunCount().get()); + } catch (ServiceException e) { + LOG.error("Failed to submit set snapshot property request", e); + } + } + /** * Class to take multiple locks on a resource. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index 44da80ffdb2f..330104326693 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.service; import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ServiceException; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; @@ -40,6 +41,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -187,16 +189,6 @@ private OzoneManagerProtocolProtos.SetSnapshotPropertyRequest getSetSnapshotRequ .build(); } - private void submitSetSnapshotRequest( - List setSnapshotPropertyRequests) { - OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() - .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) - .addAllSetSnapshotPropertyRequests(setSnapshotPropertyRequests) - .setClientId(clientId.toString()) - .build(); - submitRequest(omRequest, clientId); - } - /** * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index e6cb4ffd36ac..e6719435ff9f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -24,9 +24,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -35,36 +32,20 @@ import java.util.stream.Stream; import com.google.common.base.Preconditions; -import com.google.protobuf.ServiceException; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; -import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.KeyManager; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; -import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; -import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; @@ -77,11 +58,7 @@ import org.apache.hadoop.ozone.om.PendingKeysDeletion; import org.apache.hadoop.ozone.om.SnapshotChainManager; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.ratis.protocol.ClientId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -207,25 +184,6 @@ private OzoneManagerProtocolProtos.SetSnapshotPropertyRequest getSetSnapshotRequ .build(); } - private OzoneManagerProtocolProtos.SetSnapshotPropertyRequest - getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir(String snapshotKeyTable) { - return OzoneManagerProtocolProtos.SetSnapshotPropertyRequest.newBuilder() - .setSnapshotKey(snapshotKeyTable) - .setDeepCleanedDeletedKey(true) - .build(); - } - - private void submitSetSnapshotRequest( - List setSnapshotPropertyRequests) { - OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() - .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) - .addAllSetSnapshotPropertyRequests(setSnapshotPropertyRequests) - .setClientId(clientId.toString()) - .build(); - submitRequest(omRequest, clientId); - } - - /** * * @param currentSnapshotInfo if null, deleted directories in AOS should be processed. @@ -303,8 +261,10 @@ private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyMana //Updating directory deep clean flag of snapshot. if (currentSnapshotInfo != null) { - setSnapshotPropertyRequests.add(getSetSnapshotPropertyRequestupdatingDeepCleanSnapshotDir( - snapshotTableKey)); + setSnapshotPropertyRequests.add(OzoneManagerProtocolProtos.SetSnapshotPropertyRequest.newBuilder() + .setSnapshotKey(snapshotTableKey) + .setDeepCleanedDeletedKey(true) + .build()); } submitSetSnapshotRequest(setSnapshotPropertyRequests); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 8605f59e55b7..0569582a23f6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; From 4520004fae94a0d6b8e24cd89af6b18109b7f9c5 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 13 Sep 2024 06:35:50 -0700 Subject: [PATCH 19/22] Fix checkstyle Change-Id: I63fc93366d5e1d173348f8dc56ec84ab83634fce --- .../ozone/util/CheckExceptionOperation.java | 9 ----- .../ozone/util/CheckedExceptionOperation.java | 17 ++++++++ .../apache/hadoop/ozone/om/KeyManager.java | 20 +++++----- .../hadoop/ozone/om/KeyManagerImpl.java | 28 ++++++------- .../ozone/om/OmMetadataManagerImpl.java | 2 - .../hadoop/ozone/om/PendingKeysDeletion.java | 1 - .../key/OMDirectoriesPurgeRequestWithFSO.java | 9 +---- .../om/request/key/OMKeyPurgeRequest.java | 2 +- .../snapshot/OMSnapshotCreateRequest.java | 1 - .../OMSnapshotMoveDeletedKeysRequest.java | 2 - .../OMSnapshotMoveTableKeysRequest.java | 5 +-- .../snapshot/OMSnapshotPurgeRequest.java | 1 - .../OMSnapshotSetPropertyRequest.java | 1 - .../OMSnapshotMoveTableKeysResponse.java | 4 +- .../OMSnapshotSetPropertyResponse.java | 2 - .../service/AbstractKeyDeletingService.java | 39 ++++++++----------- .../om/service/DirectoryDeletingService.java | 27 +++---------- .../ozone/om/service/KeyDeletingService.java | 16 +++----- .../om/service/SnapshotDeletingService.java | 18 ++++----- .../ozone/om/snapshot/SnapshotUtils.java | 6 --- .../ozone/om/request/OMRequestTestUtils.java | 2 +- .../key/TestOMKeyPurgeRequestAndResponse.java | 6 --- 22 files changed, 84 insertions(+), 134 deletions(-) delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckExceptionOperation.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckedExceptionOperation.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckExceptionOperation.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckExceptionOperation.java deleted file mode 100644 index 73e939014af2..000000000000 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckExceptionOperation.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.apache.hadoop.ozone.util; - -public interface CheckExceptionOperation { - R apply(T t) throws E; - - default CheckExceptionOperation andThen(CheckExceptionOperation operation) throws E { - return (T t) -> operation.apply(this.apply(t)); - } -} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckedExceptionOperation.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckedExceptionOperation.java new file mode 100644 index 000000000000..c6a962c107c9 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckedExceptionOperation.java @@ -0,0 +1,17 @@ +package org.apache.hadoop.ozone.util; + +/** + * + * Represents a function that accepts one argument and produces a result. + * This is a functional interface whose functional method is apply(Object). + * Type parameters: + * – the type of the input to the function – the type of the result of the function + * - the type of exception thrown. + */ +public interface CheckedExceptionOperation { + R apply(T t) throws E; + + default CheckedExceptionOperation andThen(CheckedExceptionOperation operation) throws E { + return (T t) -> operation.apply(this.apply(t)); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 05df15425109..ae3f642bc999 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -34,7 +34,7 @@ import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket; -import org.apache.hadoop.ozone.util.CheckExceptionOperation; +import org.apache.hadoop.ozone.util.CheckedExceptionOperation; import java.io.IOException; import java.time.Duration; @@ -122,7 +122,7 @@ ListKeysResult listKeys(String volumeName, String bucketName, String startKey, * @throws IOException */ PendingKeysDeletion getPendingDeletionKeys( - CheckExceptionOperation, Boolean, IOException> filter, int count) + CheckedExceptionOperation, Boolean, IOException> filter, int count) throws IOException; /** * Returns a PendingKeysDeletion. It has a list of pending deletion key info @@ -137,7 +137,7 @@ PendingKeysDeletion getPendingDeletionKeys( */ PendingKeysDeletion getPendingDeletionKeys( String volume, String bucket, String startKey, - CheckExceptionOperation, Boolean, IOException> filter, int count) + CheckedExceptionOperation, Boolean, IOException> filter, int count) throws IOException; /** @@ -151,7 +151,7 @@ PendingKeysDeletion getPendingDeletionKeys( */ List> getRenamesKeyEntries( String volume, String bucket, String startKey, - CheckExceptionOperation, Boolean, IOException> filter, int count) + CheckedExceptionOperation, Boolean, IOException> filter, int count) throws IOException; /** @@ -165,7 +165,7 @@ List> getRenamesKeyEntries( */ List>> getDeletedKeyEntries( String volume, String bucket, String startKey, - CheckExceptionOperation, Boolean, IOException> filter, int count) + CheckedExceptionOperation, Boolean, IOException> filter, int count) throws IOException; /** @@ -271,8 +271,8 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, */ TableIterator> getPendingDeletionDirs() throws IOException; - TableIterator> getPendingDeletionDirs(String volume, String bucket) - throws IOException; + TableIterator> getPendingDeletionDirs( + String volume, String bucket) throws IOException; default List> getDeletedDirEntries(String volume, String bucket, int count) throws IOException { @@ -297,7 +297,7 @@ default List> getDeletedDirEntries(String volu */ List getPendingDeletionSubDirs( long volumeId, long bucketId, OmKeyInfo parentInfo, - CheckExceptionOperation, Boolean, IOException> filter, + CheckedExceptionOperation, Boolean, IOException> filter, long numEntries) throws IOException; @@ -311,8 +311,8 @@ List getPendingDeletionSubDirs( * @throws IOException */ List getPendingDeletionSubFiles(long volumeId, long bucketId, OmKeyInfo parentInfo, - CheckExceptionOperation, Boolean, - IOException> filter, + CheckedExceptionOperation, Boolean, + IOException> filter, long numEntries) throws IOException; /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 56b1a4ca19e7..79947db86c13 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -69,7 +69,7 @@ import org.apache.hadoop.net.TableMapping; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.common.BlockGroup; -import org.apache.hadoop.ozone.util.CheckExceptionOperation; +import org.apache.hadoop.ozone.util.CheckedExceptionOperation; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; @@ -637,15 +637,16 @@ public ListKeysResult listKeys(String volumeName, String bucketName, } @Override - public PendingKeysDeletion getPendingDeletionKeys(CheckExceptionOperation, Boolean, - IOException> filter, int count) throws IOException { + public PendingKeysDeletion getPendingDeletionKeys( + CheckedExceptionOperation, Boolean, IOException> filter, int count) + throws IOException { return getPendingDeletionKeys(null, null, null, filter, count); } @Override public PendingKeysDeletion getPendingDeletionKeys( String volume, String bucket, String startKey, - CheckExceptionOperation, Boolean, IOException> filter, + CheckedExceptionOperation, Boolean, IOException> filter, int count) throws IOException { List keyBlocksList = Lists.newArrayList(); Map keysToModify = new HashMap<>(); @@ -704,7 +705,7 @@ public PendingKeysDeletion getPendingDeletionKeys( @Override public List> getRenamesKeyEntries( String volume, String bucket, String startKey, - CheckExceptionOperation, Boolean, IOException> filter, + CheckedExceptionOperation, Boolean, IOException> filter, int count) throws IOException { // Bucket prefix would be empty if volume is empty i.e. either null or "". Optional bucketPrefix = Optional.ofNullable(volume).map(vol -> vol.isEmpty() ? null : vol) @@ -739,7 +740,7 @@ public List> getRenamesKeyEntries( @Override public List>> getDeletedKeyEntries( String volume, String bucket, String startKey, - CheckExceptionOperation, Boolean, IOException> filter, + CheckedExceptionOperation, Boolean, IOException> filter, int count) throws IOException { // Bucket prefix would be empty if volume is empty i.e. either null or "". Optional bucketPrefix = Optional.ofNullable(volume).map(vol -> vol.isEmpty() ? null : vol) @@ -2076,7 +2077,8 @@ public Table.KeyValue getPendingDeletionDir() } @Override - public TableIterator> getPendingDeletionDirs() throws IOException { + public TableIterator> getPendingDeletionDirs() + throws IOException { return this.getPendingDeletionDirs(null, null); } @@ -2098,7 +2100,7 @@ public Table.KeyValue getPendingDeletionDir() @Override public List getPendingDeletionSubDirs( long volumeId, long bucketId, OmKeyInfo parentInfo, - CheckExceptionOperation, Boolean, IOException> filter, + CheckedExceptionOperation, Boolean, IOException> filter, long numEntries) throws IOException { String seekDirInDB = metadataManager.getOzonePathKey(volumeId, bucketId, parentInfo.getObjectID(), ""); @@ -2114,7 +2116,7 @@ public List getPendingDeletionSubDirs( private List gatherSubDirsWithIterator(OmKeyInfo parentInfo, long numEntries, String seekDirInDB, - CheckExceptionOperation, Boolean, IOException> filter, + CheckedExceptionOperation, Boolean, IOException> filter, long countEntries, TableIterator> iterator) throws IOException { List directories = new ArrayList<>(); @@ -2134,7 +2136,7 @@ private List gatherSubDirsWithIterator(OmKeyInfo parentInfo, if (filter.apply(Table.newKeyValue(metadataManager.getOzoneDeletePathKey(omKeyInfo.getObjectID(), entry.getKey()), omKeyInfo))) { directories.add(omKeyInfo); - countEntries++;; + countEntries++; } } @@ -2145,7 +2147,7 @@ private List gatherSubDirsWithIterator(OmKeyInfo parentInfo, @Override public List getPendingDeletionSubFiles( long volumeId, long bucketId, OmKeyInfo parentInfo, - CheckExceptionOperation, Boolean, IOException> filter, long numEntries) + CheckedExceptionOperation, Boolean, IOException> filter, long numEntries) throws IOException { List files = new ArrayList<>(); String seekFileInDB = metadataManager.getOzonePathKey(volumeId, bucketId, @@ -2168,8 +2170,8 @@ public List getPendingDeletionSubFiles( continue; } OMFileRequest.setKeyNameAndFileName(parentInfo, fileInfo); - if (filter.apply(Table.newKeyValue(metadataManager.getOzoneDeletePathKey(fileInfo.getObjectID(), entry.getKey()) - , fileInfo))) { + if (filter.apply(Table.newKeyValue(metadataManager.getOzoneDeletePathKey(fileInfo.getObjectID(), + entry.getKey()), fileInfo))) { files.add(fileInfo); numEntries--; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 615e6306b8ae..0eb8fc96e659 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -87,7 +87,6 @@ import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; -import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket; @@ -112,7 +111,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_CHECKPOINT_DIR_CREATION_POLL_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotDirExist; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PendingKeysDeletion.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PendingKeysDeletion.java index bba44aee7193..9e4acd1d7466 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PendingKeysDeletion.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/PendingKeysDeletion.java @@ -21,7 +21,6 @@ import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; -import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index 23c4ae4b3f22..f47a40a3891a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -23,25 +23,18 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.UUID; -import com.google.common.collect.Maps; -import org.apache.commons.compress.utils.Lists; import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.SnapshotChainManager; -import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; -import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -86,7 +79,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn getOmRequest()); final SnapshotInfo fromSnapshotInfo; try { - fromSnapshotInfo = fromSnapshot != null ? SnapshotUtils.getSnapshotInfo(ozoneManager, + fromSnapshotInfo = fromSnapshot != null ? SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot) : null; // Checking if this request is an old request or new one. if (purgeDirsRequest.hasExpectedPreviousSnapshotID()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java index 219c237b960c..f095d1ee9f05 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java @@ -95,7 +95,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn List renamedKeysToBePurged = new ArrayList<>(purgeKeysRequest.getRenamedKeysList()); for (DeletedKeys bucketWithDeleteKeys : bucketDeletedKeysList) { - keysToBePurgedList.addAll(bucketWithDeleteKeys.getKeysList()); + keysToBePurgedList.addAll(bucketWithDeleteKeys.getKeysList()); } if (keysToBePurgedList.isEmpty() && renamedKeysToBePurged.isEmpty()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index cfb49d5de430..138046c27a94 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.ratis.server.protocol.TermIndex; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index 8d1e679c992e..7cc23f8e1afd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -23,8 +23,6 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.hdds.utils.TransactionInfo; -import org.apache.hadoop.hdds.utils.db.cache.CacheKey; -import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java index 505d71b04dbd..0472900384cf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveTableKeysRequest.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -32,13 +31,11 @@ import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveDeletedKeysResponse; import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveTableKeysResponse; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveTableKeysRequest; import org.apache.ratis.server.protocol.TermIndex; @@ -79,7 +76,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn snapshotChainManager, fromProtobuf(moveTableKeysRequest.getFromSnapshotID())); // If there is no Non-Deleted Snapshot move the // keys to Active Object Store. - SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager,fromSnapshot); + SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); // If next snapshot is not active then ignore move. Since this could be a redundant operations. if (nextSnapshot != null && nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index 12fee763a8f4..ce82e15959a2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -46,7 +46,6 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import java.util.Optional; import java.util.UUID; import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java index 9bcb050ae8fe..1d4671cbdea8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java index 221cea5ce769..54d759fae112 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java @@ -180,7 +180,7 @@ private RepeatedOmKeyInfo createRepeatedOmKeyInfo( RepeatedOmKeyInfo result = metadataManager.getDeletedTable().get(dbKey); if (result == null) { result = new RepeatedOmKeyInfo(keyInfoList); - } else if (!isSameAsLatestOmKeyInfo(keyInfoList, result)){ + } else if (!isSameAsLatestOmKeyInfo(keyInfoList, result)) { keyInfoList.forEach(result::addOmKeyInfo); } return result; @@ -190,7 +190,7 @@ private boolean isSameAsLatestOmKeyInfo(List omKeyInfos, RepeatedOmKeyInfo result) { int size = result.getOmKeyInfoList().size(); if (size >= omKeyInfos.size()) { - return omKeyInfos.equals(result.getOmKeyInfoList().subList(size - omKeyInfos.size(), size)); + return omKeyInfos.equals(result.getOmKeyInfoList().subList(size - omKeyInfos.size(), size)); } return false; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java index e0a179889bec..7b0d4e8545b7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java @@ -28,8 +28,6 @@ import jakarta.annotation.Nonnull; import java.io.IOException; import java.util.Collection; -import java.util.List; -import java.util.Set; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index 77c8e6033924..e70c0a5720d1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -37,7 +37,6 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -49,7 +48,6 @@ import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; import org.apache.hadoop.ozone.om.lock.OMLockDetails; import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; -import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -59,11 +57,9 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; -import org.apache.hadoop.ozone.util.CheckExceptionOperation; +import org.apache.hadoop.ozone.util.CheckedExceptionOperation; import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; -import org.apache.ratis.protocol.Message; -import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.util.Preconditions; import java.io.Closeable; @@ -91,10 +87,9 @@ */ public abstract class AbstractKeyDeletingService extends BackgroundService implements BootstrapStateHandler { - + private static final ClientId CLIENT_ID = ClientId.randomId(); private final OzoneManager ozoneManager; private final ScmBlockLocationProtocol scmClient; - protected static ClientId clientId = ClientId.randomId(); private final AtomicLong deletedDirsCount; private final AtomicLong movedDirsCount; private final AtomicLong movedFilesCount; @@ -334,13 +329,13 @@ private Pair submitPurgeKeysRequest(List prepareDeleteDirRequest( long remainNum, OmKeyInfo pendingDeletedDirInfo, String delDirName, List> subDirList, KeyManager keyManager, boolean deleteDir, - CheckExceptionOperation, Boolean, IOException> fileDeletionChecker) + CheckedExceptionOperation, Boolean, IOException> fileDeletionChecker) throws IOException { // step-0: Get one pending deleted directory if (LOG.isDebugEnabled()) { @@ -503,8 +498,8 @@ public Pair> optimizeDirD long remainNum, long dirNum, long subDirNum, long subFileNum, List> allSubDirList, List purgePathRequestList, String snapTableKey, long startTime, int remainingBufLimit, KeyManager keyManager, - CheckExceptionOperation, Boolean, IOException> subDirPurgeChecker, - CheckExceptionOperation, Boolean, IOException> fileDeletionChecker, + CheckedExceptionOperation, Boolean, IOException> subDirPurgeChecker, + CheckedExceptionOperation, Boolean, IOException> fileDeletionChecker, UUID expectedPreviousSnapshotId) { // Optimization to handle delete sub-dir and keys to remove quickly @@ -516,7 +511,7 @@ public Pair> optimizeDirD try { Pair stringOmKeyInfoPair = allSubDirList.get(subDirRecursiveCnt); Boolean result = subDirPurgeChecker.apply(Table.newKeyValue(stringOmKeyInfoPair.getKey(), - stringOmKeyInfoPair.getValue()) ); + stringOmKeyInfoPair.getValue())); Optional request = prepareDeleteDirRequest( remainNum, stringOmKeyInfoPair.getValue(), stringOmKeyInfoPair.getKey(), allSubDirList, @@ -873,10 +868,10 @@ protected void submitSetSnapshotRequest( OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) .addAllSetSnapshotPropertyRequests(setSnapshotPropertyRequests) - .setClientId(clientId.toString()) + .setClientId(CLIENT_ID.toString()) .build(); try { - OzoneManagerRatisUtils.submitRequest(getOzoneManager(), omRequest, clientId, getRunCount().get()); + OzoneManagerRatisUtils.submitRequest(getOzoneManager(), omRequest, CLIENT_ID, getRunCount().get()); } catch (ServiceException e) { LOG.error("Failed to submit set snapshot property request", e); } @@ -931,8 +926,8 @@ public void releaseLock() { /** * This class is responsible for opening last N snapshot given snapshot or AOS metadata manager by acquiring a lock. */ - private abstract class ReclaimableFilter implements CheckExceptionOperation, - Boolean, IOException>, Closeable { + private abstract class ReclaimableFilter implements CheckedExceptionOperation, + Boolean, IOException>, Closeable { private final SnapshotInfo currentSnapshotInfo; private final OmSnapshotManager omSnapshotManager; @@ -956,7 +951,7 @@ private abstract class ReclaimableFilter implements CheckExceptionOperation setSnapshotPropertyRequests = new ArrayList<>(); Map exclusiveReplicatedSizeMap = reclaimableSubFileFilter.getExclusiveReplicatedSizeMap(); Map exclusiveSizeMap = reclaimableSubFileFilter.getExclusiveSizeMap(); - for (String snapshot : Stream.of(exclusiveSizeMap.keySet(), - exclusiveReplicatedSizeMap.keySet()).flatMap(Collection::stream).distinct().collect(Collectors.toList())) { + for (String snapshot : Stream.of(exclusiveSizeMap.keySet(), exclusiveReplicatedSizeMap.keySet()) + .flatMap(Collection::stream).distinct().collect(Collectors.toList())) { setSnapshotPropertyRequests.add(getSetSnapshotRequestUpdatingExclusiveSize(exclusiveSizeMap, exclusiveReplicatedSizeMap, snapshot)); } @@ -350,7 +335,7 @@ public BackgroundTaskResult call() { continue; } try (ReferenceCounted omSnapshot = omSnapshotManager.getSnapshot(snapInfo.getVolumeName(), - snapInfo.getBucketName(), snapInfo.getName())) { + snapInfo.getBucketName(), snapInfo.getName())) { remainNum = processDeletedDirsForStore(snapInfo, omSnapshot.get().getKeyManager(), remainNum); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index e6719435ff9f..2ec06379725f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -20,7 +20,6 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -82,9 +81,6 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private int keyLimitPerTask; private final AtomicLong deletedKeyCount; private final AtomicBoolean suspended; - private final Map exclusiveSizeMap; - private final Map exclusiveReplicatedSizeMap; - private final Map snapshotSeekMap; private final boolean deepCleanSnapshots; public KeyDeletingService(OzoneManager ozoneManager, @@ -102,9 +98,6 @@ public KeyDeletingService(OzoneManager ozoneManager, OZONE_KEY_DELETING_LIMIT_PER_TASK + " cannot be negative."); this.deletedKeyCount = new AtomicLong(0); this.suspended = new AtomicBoolean(false); - this.exclusiveSizeMap = new HashMap<>(); - this.exclusiveReplicatedSizeMap = new HashMap<>(); - this.snapshotSeekMap = new HashMap<>(); this.deepCleanSnapshots = deepCleanSnapshots; } @@ -218,8 +211,8 @@ private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyMana // Purge deleted Keys in the deletedTable && rename entries in the snapshotRenamedTable which doesn't have a // reference in the previous snapshot. - try (ReclaimableKeyFilter reclaimableKeyFilter = new ReclaimableKeyFilter(omSnapshotManager, snapshotChainManager, - currentSnapshotInfo, keyManager.getMetadataManager(), lock); + try (ReclaimableKeyFilter reclaimableKeyFilter = new ReclaimableKeyFilter(omSnapshotManager, + snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); ReclaimableRenameEntryFilter renameEntryFilter = new ReclaimableRenameEntryFilter( omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock)) { List renamedTableEntries = @@ -239,8 +232,9 @@ private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyMana List keyBlocksList = pendingKeysDeletion.getKeyBlocksList(); //submit purge requests if there are renamed entries to be purged or keys to be purged. if (!renamedTableEntries.isEmpty() || keyBlocksList != null && !keyBlocksList.isEmpty()) { - Pair purgeResult = processKeyDeletes(keyBlocksList, getOzoneManager().getKeyManager(), - pendingKeysDeletion.getKeysToModify(), renamedTableEntries, snapshotTableKey, expectedPreviousSnapshotId); + Pair purgeResult = processKeyDeletes(keyBlocksList, getOzoneManager().getKeyManager(), + pendingKeysDeletion.getKeysToModify(), renamedTableEntries, snapshotTableKey, + expectedPreviousSnapshotId); remainNum -= purgeResult.getKey(); successStatus = purgeResult.getValue(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 0569582a23f6..cc02059b68e7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -37,17 +37,13 @@ import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; -import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; -import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveTableKeysRequest; @@ -159,7 +155,8 @@ public BackgroundTaskResult call() throws InterruptedException { snapshotChainManager, snapInfo); // Continue if the next snapshot is not active. This is to avoid unnecessary copies from one snapshot to // another. - if (nextSnapshot != null && nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { + if (nextSnapshot != null && + nextSnapshot.getSnapshotStatus() != SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) { continue; } @@ -175,9 +172,8 @@ public BackgroundTaskResult call() throws InterruptedException { .isLockAcquired()) { continue; } - try(ReferenceCounted snapshot = omSnapshotManager.getSnapshot(snapInfo.getVolumeName(), - snapInfo.getBucketName(), - snapInfo.getName())) { + try (ReferenceCounted snapshot = omSnapshotManager.getSnapshot(snapInfo.getVolumeName(), + snapInfo.getBucketName(), snapInfo.getName())) { KeyManager snapshotKeyManager = snapshot.get().getKeyManager(); int moveCount = 0; // Get all entries from deletedKeyTable. @@ -191,11 +187,13 @@ public BackgroundTaskResult call() throws InterruptedException { moveCount += deletedDirEntries.size(); // Get all entries from snapshotRenamedTable. List> renameEntries = snapshotKeyManager.getRenamesKeyEntries( - snapInfo.getVolumeName(), snapInfo.getBucketName(), null , (kv) -> true, remaining - moveCount); + snapInfo.getVolumeName(), snapInfo.getBucketName(), null, (kv) -> true, remaining - moveCount); moveCount += renameEntries.size(); if (moveCount > 0) { try { - submitSnapshotMoveDeletedKeys(snapInfo, deletedKeyEntries.stream().map(kv -> { + submitSnapshotMoveDeletedKeys( + snapInfo, + deletedKeyEntries.stream().map(kv -> { try { return SnapshotMoveKeyInfos.newBuilder().setKey(kv.getKey()).addAllKeyInfos(kv.getValue() .stream().map(val -> val.getProtobuf(ClientVersion.CURRENT_VERSION)) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index a8482309ff87..7312fdc9b256 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -18,20 +18,14 @@ package org.apache.hadoop.ozone.om.snapshot; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OmSnapshot; -import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; -import org.apache.hadoop.ozone.om.response.key.OMKeyPurgeResponse; -import org.apache.ratis.server.protocol.TermIndex; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.slf4j.Logger; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java index ab3d810473e2..39e3305c2e69 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java @@ -286,7 +286,7 @@ public static void addKeyToTable(boolean openKeyTable, boolean addToCache, } /** - * Add key entry to SnapshotRenamedTable + * Add key entry to SnapshotRenamedTable. */ public static String addRenamedEntryToTable(long trxnLogIndex, String volumeName, String bucketName, String key, OMMetadataManager omMetadataManager) throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java index a58e29cdf690..928a14efde78 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java @@ -28,9 +28,6 @@ import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; -import org.apache.hadoop.ozone.om.request.snapshot.TestOMSnapshotCreateRequest; -import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotCreateResponse; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.junit.jupiter.api.Test; @@ -47,10 +44,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.when; /** * Tests {@link OMKeyPurgeRequest} and {@link OMKeyPurgeResponse}. From ac74e51beddd33b7888bd516f509e87fc60121a4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 20 Sep 2024 06:19:25 -0700 Subject: [PATCH 20/22] HDDS-11244. Fix race condition b/w create snapshot & AOS key deleting service Change-Id: Ib21775038034459f3d2aeab19890ed3e94882ed0 --- .../hadoop/ozone/om/service/AbstractKeyDeletingService.java | 2 +- .../apache/hadoop/ozone/om/service/KeyDeletingService.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index e70c0a5720d1..584c484879ec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -494,7 +494,7 @@ protected Optional prepareDeleteDirRequest( } @SuppressWarnings("checkstyle:ParameterNumber") - public Pair> optimizeDirDeletesAndSubmitRequest( + public Pair> optimizeDirDeletesAndSubmitRequest( long remainNum, long dirNum, long subDirNum, long subFileNum, List> allSubDirList, List purgePathRequestList, String snapTableKey, long startTime, int remainingBufLimit, KeyManager keyManager, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 2ec06379725f..0e2fbb11c79c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -200,7 +200,7 @@ private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyMana OmSnapshotManager omSnapshotManager = getOzoneManager().getOmSnapshotManager(); SnapshotChainManager snapshotChainManager = ((OmMetadataManagerImpl)getOzoneManager().getMetadataManager()) .getSnapshotChainManager(); - // This is to avoid race condition b/w purge request and snapshot chain updation. For AOS taking the global + // This is to avoid race condition b/w purge request and snapshot chain update. For AOS taking the global // snapshotId since AOS could process multiple buckets in one iteration. While using path previous snapshot // Id for a snapshot since it would process only one bucket. UUID expectedPreviousSnapshotId = currentSnapshotInfo == null ? @@ -232,6 +232,9 @@ private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyMana List keyBlocksList = pendingKeysDeletion.getKeyBlocksList(); //submit purge requests if there are renamed entries to be purged or keys to be purged. if (!renamedTableEntries.isEmpty() || keyBlocksList != null && !keyBlocksList.isEmpty()) { + // Validating if the previous snapshot is still the same before purging the blocks. + SnapshotUtils.validatePreviousSnapshotId(currentSnapshotInfo, snapshotChainManager, + expectedPreviousSnapshotId); Pair purgeResult = processKeyDeletes(keyBlocksList, getOzoneManager().getKeyManager(), pendingKeysDeletion.getKeysToModify(), renamedTableEntries, snapshotTableKey, expectedPreviousSnapshotId); From 200c9a76f7904a4a9ca1091face149fde6719b8d Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 21 Oct 2024 13:44:55 -0700 Subject: [PATCH 21/22] HDDS-11244. Merge Change-Id: I4bb0bfcfa498114426c939aec5d7cb13ced0ff2c --- ...napshotDeletingServiceIntegrationTest.java | 12 +- .../TestSnapshotDirectoryCleaningService.java | 0 .../src/main/proto/OmClientProtocol.proto | 1 - .../apache/hadoop/ozone/om/KeyManager.java | 4 +- .../hadoop/ozone/om/KeyManagerImpl.java | 97 +-- .../hadoop/ozone/om/lock/MultiLocks.java | 54 ++ .../OMSnapshotSetPropertyRequest.java | 18 +- .../service/AbstractKeyDeletingService.java | 679 ------------------ .../om/service/DirectoryDeletingService.java | 19 +- .../ozone/om/service/KeyDeletingService.java | 17 +- .../om/service/SnapshotDeletingService.java | 19 +- .../SnapshotDirectoryCleaningService.java | 0 .../om/snapshot/SnapshotDiffManager.java | 5 +- .../ozone/om/snapshot/SnapshotUtils.java | 57 +- .../snapshot/filter/ReclaimableDirFilter.java | 95 +++ .../om/snapshot/filter/ReclaimableFilter.java | 204 ++++++ .../snapshot/filter/ReclaimableKeyFilter.java | 258 +++++++ .../filter/ReclaimableRenameEntryFilter.java | 83 +++ .../om/service/TestKeyDeletingService.java | 4 +- .../om/snapshot/TestSnapshotDiffManager.java | 3 +- 20 files changed, 829 insertions(+), 800 deletions(-) delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/MultiLocks.java delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableDirFilter.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableFilter.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableRenameEntryFilter.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java index 254de072e05b..b4a3676820f0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java @@ -632,9 +632,9 @@ public void testParallelExcecutionOfKeyDeletionAndSnapshotDeletion() throws Exce try (ReferenceCounted snapshot = om.getOmSnapshotManager().getSnapshot(testBucket.getVolumeName(), testBucket.getName(), testBucket.getName() + "snap2")) { renamesKeyEntries = snapshot.get().getKeyManager().getRenamesKeyEntries(testBucket.getVolumeName(), - testBucket.getName(), "", 1000); + testBucket.getName(), "", (kv) -> true, 1000); deletedKeyEntries = snapshot.get().getKeyManager().getDeletedKeyEntries(testBucket.getVolumeName(), - testBucket.getName(), "", 1000); + testBucket.getName(), "", (kv) -> true, 1000); deletedDirEntries = snapshot.get().getKeyManager().getDeletedDirEntries(testBucket.getVolumeName(), testBucket.getName(), 1000); } @@ -669,20 +669,20 @@ public void testParallelExcecutionOfKeyDeletionAndSnapshotDeletion() throws Exce testBucket.getName(), testBucket.getName() + "snap2")) { Assertions.assertEquals(Collections.emptyList(), snapshot.get().getKeyManager().getRenamesKeyEntries(testBucket.getVolumeName(), - testBucket.getName(), "", 1000)); + testBucket.getName(), "", (kv) -> true, 1000)); Assertions.assertEquals(Collections.emptyList(), snapshot.get().getKeyManager().getDeletedKeyEntries(testBucket.getVolumeName(), - testBucket.getName(), "", 1000)); + testBucket.getName(), "", (kv) -> true, 1000)); Assertions.assertEquals(Collections.emptyList(), snapshot.get().getKeyManager().getDeletedDirEntries(testBucket.getVolumeName(), testBucket.getName(), 1000)); } List> aosRenamesKeyEntries = om.getKeyManager().getRenamesKeyEntries(testBucket.getVolumeName(), - testBucket.getName(), "", 1000); + testBucket.getName(), "", (kv) -> true, 1000); List>> aosDeletedKeyEntries = om.getKeyManager().getDeletedKeyEntries(testBucket.getVolumeName(), - testBucket.getName(), "", 1000); + testBucket.getName(), "", (kv) -> true, 1000); List> aosDeletedDirEntries = om.getKeyManager().getDeletedDirEntries(testBucket.getVolumeName(), testBucket.getName(), 1000); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 0b08ce38c851..a0def0f27d98 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -2027,7 +2027,6 @@ message SnapshotProperty { message SnapshotSize { optional uint64 exclusiveSize = 1; optional uint64 exclusiveReplicatedSize = 2; - optional bool addSize = 3; } message DeleteTenantRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 56264496e382..9cfddcbeebe5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -150,7 +150,7 @@ PendingKeysDeletion getPendingDeletionKeys( */ List> getRenamesKeyEntries( String volume, String bucket, String startKey, - CheckedExceptionOperation, Boolean, IOException> filter, int count) + CheckedExceptionOperation, Boolean, IOException> filter, int size) throws IOException; /** @@ -264,6 +264,8 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, */ Table.KeyValue getPendingDeletionDir() throws IOException; + TableIterator> getPendingDeletionDirs( + String volume, String bucket) throws IOException; /** * Returns an iterator for pending deleted directories. * @throws IOException diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 9040d5e72de9..db0cdf875e28 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -652,8 +652,7 @@ public PendingKeysDeletion getPendingDeletionKeys( Map keysToModify = new HashMap<>(); final String nextPageStartKey; // Bucket prefix would be empty if volume is empty i.e. either null or "". - Optional bucketPrefix = Optional.ofNullable(volume).map(vol -> vol.isEmpty() ? null : vol) - .map(vol -> metadataManager.getBucketKeyPrefix(vol, bucket)); + Optional bucketPrefix = getBucketPrefix(volume, bucket, false); try (TableIterator> delKeyIter = metadataManager.getDeletedTable().iterator(bucketPrefix.orElse(""))) { @@ -702,81 +701,19 @@ public PendingKeysDeletion getPendingDeletionKeys( return new PendingKeysDeletion(keyBlocksList, keysToModify, nextPageStartKey); } - @Override - public List> getRenamesKeyEntries( - String volume, String bucket, String startKey, - CheckedExceptionOperation, Boolean, IOException> filter, - int count) throws IOException { - // Bucket prefix would be empty if volume is empty i.e. either null or "". - Optional bucketPrefix = Optional.ofNullable(volume).map(vol -> vol.isEmpty() ? null : vol) - .map(vol -> metadataManager.getBucketKeyPrefix(vol, bucket)); - List> renamedEntries = new ArrayList<>(); - try (TableIterator> - renamedKeyIter = metadataManager.getSnapshotRenamedTable().iterator(bucketPrefix.orElse(""))) { - - /* Seeking to the start key if it not null. The next key picked up would be ensured to start with the bucket - prefix, {@link org.apache.hadoop.hdds.utils.db.Table#iterator(bucketPrefix)} would ensure this. - */ - if (startKey != null) { - renamedKeyIter.seek(startKey); - } - int currentCount = 0; - while (renamedKeyIter.hasNext() && currentCount < count) { - RepeatedOmKeyInfo notReclaimableKeyInfo = new RepeatedOmKeyInfo(); - Table.KeyValue kv = renamedKeyIter.next(); - if (kv != null) { - - // Multiple keys with the same path can be queued in one DB entry - if (filter.apply(kv)) { - renamedEntries.add(Table.newKeyValue(kv.getKey(), kv.getValue())); - currentCount++; - } - } - } - } - return renamedEntries; - } - - @Override - public List>> getDeletedKeyEntries( - String volume, String bucket, String startKey, - CheckedExceptionOperation, Boolean, IOException> filter, - int count) throws IOException { - // Bucket prefix would be empty if volume is empty i.e. either null or "". - Optional bucketPrefix = Optional.ofNullable(volume).map(vol -> vol.isEmpty() ? null : vol) - .map(vol -> metadataManager.getBucketKeyPrefix(vol, bucket)); - List>> deletedKeyEntries = new ArrayList<>(count); - try (TableIterator> - delKeyIter = metadataManager.getDeletedTable().iterator(bucketPrefix.orElse(""))) { - - /* Seeking to the start key if it not null. The next key picked up would be ensured to start with the bucket - prefix, {@link org.apache.hadoop.hdds.utils.db.Table#iterator(bucketPrefix)} would ensure this. - */ - if (startKey != null) { - delKeyIter.seek(startKey); - } - int currentCount = 0; - while (delKeyIter.hasNext() && currentCount < count) { - Table.KeyValue kv = delKeyIter.next(); - if (kv != null && filter.apply(kv)) { - deletedKeyEntries.add(Table.newKeyValue(kv.getKey(), kv.getValue().cloneOmKeyInfoList())); - } - } - } - return deletedKeyEntries; - } - private List> getTableEntries(String startKey, TableIterator> tableIterator, - Function valueFunction, int size) throws IOException { + Function valueFunction, + CheckedExceptionOperation, Boolean, IOException> filter, + int size) throws IOException { List> entries = new ArrayList<>(); /* Seek to the start key if it not null. The next key in queue is ensured to start with the bucket prefix, {@link org.apache.hadoop.hdds.utils.db.Table#iterator(bucketPrefix)} would ensure this. */ if (startKey != null) { tableIterator.seek(startKey); - tableIterator.seekToFirst(); } + int currentCount = 0; while (tableIterator.hasNext() && currentCount < size) { Table.KeyValue kv = tableIterator.next(); @@ -802,21 +739,25 @@ private Optional getBucketPrefix(String volumeName, String bucketName, b @Override public List> getRenamesKeyEntries( - String volume, String bucket, String startKey, int size) throws IOException { + String volume, String bucket, String startKey, + CheckedExceptionOperation, Boolean, IOException> filter, + int size) throws IOException { Optional bucketPrefix = getBucketPrefix(volume, bucket, false); try (TableIterator> renamedKeyIter = metadataManager.getSnapshotRenamedTable().iterator(bucketPrefix.orElse(""))) { - return getTableEntries(startKey, renamedKeyIter, Function.identity(), size); + return getTableEntries(startKey, renamedKeyIter, Function.identity(), filter, size); } } @Override public List>> getDeletedKeyEntries( - String volume, String bucket, String startKey, int size) throws IOException { + String volume, String bucket, String startKey, + CheckedExceptionOperation, Boolean, IOException> filter, + int size) throws IOException { Optional bucketPrefix = getBucketPrefix(volume, bucket, false); try (TableIterator> delKeyIter = metadataManager.getDeletedTable().iterator(bucketPrefix.orElse(""))) { - return getTableEntries(startKey, delKeyIter, RepeatedOmKeyInfo::cloneOmKeyInfoList, size); + return getTableEntries(startKey, delKeyIter, RepeatedOmKeyInfo::cloneOmKeyInfoList, filter, size); } } @@ -2132,16 +2073,8 @@ public Table.KeyValue getPendingDeletionDir() } @Override - public TableIterator> getPendingDeletionDirs() - throws IOException { - return this.getPendingDeletionDirs(null, null); - } - - @Override - public TableIterator> getPendingDeletionDirs(String volume, - String bucket) - throws IOException { - + public TableIterator> getPendingDeletionDirs( + String volume, String bucket) throws IOException { // Either both volume & bucket should be null or none of them should be null. if (!StringUtils.isBlank(volume) && StringUtils.isBlank(bucket) || StringUtils.isBlank(volume) && !StringUtils.isBlank(bucket)) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/MultiLocks.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/MultiLocks.java new file mode 100644 index 000000000000..646a827439d7 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/MultiLocks.java @@ -0,0 +1,54 @@ +package org.apache.hadoop.ozone.om.lock; + +import org.apache.hadoop.ozone.om.exceptions.OMException; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.Queue; + +/** + * Class to take multiple locks on a resource. + */ +public class MultiLocks { + private final Queue objectLocks; + private final IOzoneManagerLock lock; + private final OzoneManagerLock.Resource resource; + private final boolean writeLock; + + public MultiLocks(IOzoneManagerLock lock, OzoneManagerLock.Resource resource, boolean writeLock) { + this.writeLock = writeLock; + this.resource = resource; + this.lock = lock; + this.objectLocks = new LinkedList<>(); + } + + public OMLockDetails acquireLock(Collection objects) throws OMException { + if (!objectLocks.isEmpty()) { + throw new OMException("More locks cannot be acquired when locks have been already acquired. Locks acquired : " + + objectLocks, OMException.ResultCodes.INTERNAL_ERROR); + } + OMLockDetails omLockDetails = OMLockDetails.EMPTY_DETAILS_LOCK_ACQUIRED; + for (T object : objects) { + if (object != null) { + omLockDetails = this.writeLock ? lock.acquireWriteLock(resource, object.toString()) + : lock.acquireReadLock(resource, object.toString()); + objectLocks.add(object); + if (!omLockDetails.isLockAcquired()) { + break; + } + } + } + if (!omLockDetails.isLockAcquired()) { + releaseLock(); + } + return omLockDetails; + } + + public void releaseLock() { + while (!objectLocks.isEmpty()) { + T object = objectLocks.poll(); + OMLockDetails lockDetails = this.writeLock ? lock.releaseWriteLock(resource, object.toString()) + : lock.releaseReadLock(resource, object.toString()); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java index 1d4671cbdea8..0e93a1b6229e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java @@ -71,16 +71,16 @@ private void updateSnapshotProperty( } if (setSnapshotPropertyRequest.hasSnapshotSize()) { - SnapshotSize snapshotSize = setSnapshotPropertyRequest - .getSnapshotSize(); - boolean addToSnapshotSize = snapshotSize.hasAddSize() && snapshotSize.getAddSize(); - long exclusiveSize = (addToSnapshotSize ? snapInfo.getExclusiveSize() : 0) + - snapshotSize.getExclusiveSize(); - long exclusiveReplicatedSize = (addToSnapshotSize ? snapInfo.getExclusiveReplicatedSize() : 0) + - snapshotSize.getExclusiveReplicatedSize(); + SnapshotSize snapshotSize = setSnapshotPropertyRequest.getSnapshotSize(); // Set Exclusive size. - snapInfo.setExclusiveSize(exclusiveSize); - snapInfo.setExclusiveReplicatedSize(exclusiveReplicatedSize); + snapInfo.setExclusiveSize(snapshotSize.getExclusiveSize()); + snapInfo.setExclusiveReplicatedSize(snapshotSize.getExclusiveReplicatedSize()); + } + if (setSnapshotPropertyRequest.hasSnapshotDirSize()) { + SnapshotSize snapshotSize = setSnapshotPropertyRequest.getSnapshotDirSize(); + // Set Exclusive size. + snapInfo.setExclusiveSize(snapshotSize.getExclusiveSize()); + snapInfo.setExclusiveReplicatedSize(snapshotSize.getExclusiveReplicatedSize()); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index c77f9807e579..1b7e427e504d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -20,7 +20,6 @@ import com.google.protobuf.ServiceException; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.HddsUtils; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.hdds.utils.db.BatchOperation; @@ -32,24 +31,10 @@ import org.apache.hadoop.ozone.common.DeleteBlockGroupResult; import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmSnapshot; -import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.SnapshotChainManager; -import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; -import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; -import org.apache.hadoop.ozone.om.lock.OMLockDetails; -import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; -import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; -import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeletedKeys; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -62,23 +47,17 @@ import org.apache.ratis.protocol.ClientId; import org.apache.ratis.util.Preconditions; -import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Queue; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; -import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; /** @@ -111,60 +90,6 @@ public AbstractKeyDeletingService(String serviceName, long interval, this.runCount = new AtomicLong(0); } - // TODO: Move this util class. - public static boolean isBlockLocationInfoSame(OmKeyInfo prevKeyInfo, - OmKeyInfo deletedKeyInfo) { - - if (prevKeyInfo == null && deletedKeyInfo == null) { - LOG.debug("Both prevKeyInfo and deletedKeyInfo are null."); - return true; - } - if (prevKeyInfo == null || deletedKeyInfo == null) { - LOG.debug("prevKeyInfo: '{}' or deletedKeyInfo: '{}' is null.", - prevKeyInfo, deletedKeyInfo); - return false; - } - // For hsync, Though the blockLocationInfo of a key may not be same - // at the time of snapshot and key deletion as blocks can be appended. - // If the objectId is same then the key is same. - if (prevKeyInfo.isHsync() && deletedKeyInfo.isHsync()) { - return true; - } - - if (prevKeyInfo.getKeyLocationVersions().size() != - deletedKeyInfo.getKeyLocationVersions().size()) { - return false; - } - - OmKeyLocationInfoGroup deletedOmKeyLocation = - deletedKeyInfo.getLatestVersionLocations(); - OmKeyLocationInfoGroup prevOmKeyLocation = - prevKeyInfo.getLatestVersionLocations(); - - if (deletedOmKeyLocation == null || prevOmKeyLocation == null) { - return false; - } - - List deletedLocationList = - deletedOmKeyLocation.getLocationList(); - List prevLocationList = - prevOmKeyLocation.getLocationList(); - - if (deletedLocationList.size() != prevLocationList.size()) { - return false; - } - - for (int idx = 0; idx < deletedLocationList.size(); idx++) { - OmKeyLocationInfo deletedLocationInfo = deletedLocationList.get(idx); - OmKeyLocationInfo prevLocationInfo = prevLocationList.get(idx); - if (!deletedLocationInfo.hasSameBlockAs(prevLocationInfo)) { - return false; - } - } - - return true; - } - protected Pair processKeyDeletes(List keyBlocksList, KeyManager manager, Map keysToModify, @@ -564,242 +489,11 @@ public Pair> optimizeDirDe return Pair.of(remainNum, Optional.ofNullable(response)); } - /** - * To calculate Exclusive Size for current snapshot, Check - * the next snapshot deletedTable if the deleted key is - * referenced in current snapshot and not referenced in the - * previous snapshot then that key is exclusive to the current - * snapshot. Here since we are only iterating through - * deletedTable we can check the previous and previous to - * previous snapshot to achieve the same. - * previousSnapshot - Snapshot for which exclusive size is - * getting calculating. - * currSnapshot - Snapshot's deletedTable is used to calculate - * previousSnapshot snapshot's exclusive size. - * previousToPrevSnapshot - Snapshot which is used to check - * if key is exclusive to previousSnapshot. - */ - @SuppressWarnings("checkstyle:ParameterNumber") - public void calculateExclusiveSize( - SnapshotInfo previousSnapshot, - SnapshotInfo previousToPrevSnapshot, - OmKeyInfo keyInfo, - OmBucketInfo bucketInfo, long volumeId, - Table snapRenamedTable, - Table previousKeyTable, - Table prevRenamedTable, - Table previousToPrevKeyTable, - Map exclusiveSizeMap, - Map exclusiveReplicatedSizeMap) throws IOException { - String prevSnapKey = previousSnapshot.getTableKey(); - long exclusiveReplicatedSize = - exclusiveReplicatedSizeMap.getOrDefault( - prevSnapKey, 0L) + keyInfo.getReplicatedSize(); - long exclusiveSize = exclusiveSizeMap.getOrDefault( - prevSnapKey, 0L) + keyInfo.getDataSize(); - - // If there is no previous to previous snapshot, then - // the previous snapshot is the first snapshot. - if (previousToPrevSnapshot == null) { - exclusiveSizeMap.put(prevSnapKey, exclusiveSize); - exclusiveReplicatedSizeMap.put(prevSnapKey, - exclusiveReplicatedSize); - } else { - OmKeyInfo keyInfoPrevSnapshot = getPreviousSnapshotKeyName( - keyInfo, bucketInfo, volumeId, - snapRenamedTable, previousKeyTable); - OmKeyInfo keyInfoPrevToPrevSnapshot = getPreviousSnapshotKeyName( - keyInfoPrevSnapshot, bucketInfo, volumeId, - prevRenamedTable, previousToPrevKeyTable); - // If the previous to previous snapshot doesn't - // have the key, then it is exclusive size for the - // previous snapshot. - if (keyInfoPrevToPrevSnapshot == null) { - exclusiveSizeMap.put(prevSnapKey, exclusiveSize); - exclusiveReplicatedSizeMap.put(prevSnapKey, - exclusiveReplicatedSize); - } - } - } - - private OmKeyInfo getPreviousSnapshotKeyName( - OmKeyInfo keyInfo, OmBucketInfo bucketInfo, long volumeId, - Table snapRenamedTable, - Table previousKeyTable) throws IOException { - - if (keyInfo == null) { - return null; - } - - String dbKeyPrevSnap; - if (bucketInfo.getBucketLayout().isFileSystemOptimized()) { - dbKeyPrevSnap = getOzoneManager().getMetadataManager().getOzonePathKey( - volumeId, - bucketInfo.getObjectID(), - keyInfo.getParentObjectID(), - keyInfo.getFileName()); - } else { - dbKeyPrevSnap = getOzoneManager().getMetadataManager().getOzoneKey( - keyInfo.getVolumeName(), - keyInfo.getBucketName(), - keyInfo.getKeyName()); - } - - String dbRenameKey = getOzoneManager().getMetadataManager().getRenameKey( - keyInfo.getVolumeName(), - keyInfo.getBucketName(), - keyInfo.getObjectID()); - - String renamedKey = snapRenamedTable.getIfExist(dbRenameKey); - OmKeyInfo prevKeyInfo = renamedKey != null ? - previousKeyTable.get(renamedKey) : - previousKeyTable.get(dbKeyPrevSnap); - - if (prevKeyInfo == null || - prevKeyInfo.getObjectID() != keyInfo.getObjectID()) { - return null; - } - - return isBlockLocationInfoSame(prevKeyInfo, keyInfo) ? - prevKeyInfo : null; - } - protected boolean isBufferLimitCrossed( int maxLimit, int cLimit, int increment) { return cLimit + increment >= maxLimit; } - protected boolean isKeyReclaimable( - Table previousKeyTable, - Table renamedTable, - OmKeyInfo deletedKeyInfo, OmBucketInfo bucketInfo, - long volumeId, HddsProtos.KeyValue.Builder renamedKeyBuilder) - throws IOException { - - String dbKey; - // Handle case when the deleted snapshot is the first snapshot. - if (previousKeyTable == null) { - return true; - } - - // These are uncommitted blocks wrapped into a pseudo KeyInfo - if (deletedKeyInfo.getObjectID() == OBJECT_ID_RECLAIM_BLOCKS) { - return true; - } - - // Construct keyTable or fileTable DB key depending on the bucket type - if (bucketInfo.getBucketLayout().isFileSystemOptimized()) { - dbKey = ozoneManager.getMetadataManager().getOzonePathKey( - volumeId, - bucketInfo.getObjectID(), - deletedKeyInfo.getParentObjectID(), - deletedKeyInfo.getFileName()); - } else { - dbKey = ozoneManager.getMetadataManager().getOzoneKey( - deletedKeyInfo.getVolumeName(), - deletedKeyInfo.getBucketName(), - deletedKeyInfo.getKeyName()); - } - - /* - snapshotRenamedTable: - 1) /volumeName/bucketName/objectID -> - /volumeId/bucketId/parentId/fileName (FSO) - 2) /volumeName/bucketName/objectID -> - /volumeName/bucketName/keyName (non-FSO) - */ - String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey( - deletedKeyInfo.getVolumeName(), deletedKeyInfo.getBucketName(), - deletedKeyInfo.getObjectID()); - - // Condition: key should not exist in snapshotRenamedTable - // of the current snapshot and keyTable of the previous snapshot. - // Check key exists in renamedTable of the Snapshot - String renamedKey = renamedTable.getIfExist(dbRenameKey); - - if (renamedKey != null && renamedKeyBuilder != null) { - renamedKeyBuilder.setKey(dbRenameKey).setValue(renamedKey); - } - // previousKeyTable is fileTable if the bucket is FSO, - // otherwise it is the keyTable. - OmKeyInfo prevKeyInfo = renamedKey != null ? previousKeyTable - .get(renamedKey) : previousKeyTable.get(dbKey); - - if (prevKeyInfo == null || - prevKeyInfo.getObjectID() != deletedKeyInfo.getObjectID()) { - return true; - } - - // For key overwrite the objectID will remain the same, In this - // case we need to check if OmKeyLocationInfo is also same. - return !isBlockLocationInfoSame(prevKeyInfo, deletedKeyInfo); - } - - protected boolean isDirReclaimable( - Table.KeyValue deletedDir, - Table previousDirTable, - Table renamedTable) throws IOException { - - if (previousDirTable == null) { - return true; - } - - String deletedDirDbKey = deletedDir.getKey(); - OmKeyInfo deletedDirInfo = deletedDir.getValue(); - String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey( - deletedDirInfo.getVolumeName(), deletedDirInfo.getBucketName(), - deletedDirInfo.getObjectID()); - - /* - snapshotRenamedTable: /volumeName/bucketName/objectID -> - /volumeId/bucketId/parentId/dirName - */ - String dbKeyBeforeRename = renamedTable.getIfExist(dbRenameKey); - String prevDbKey = null; - - if (dbKeyBeforeRename != null) { - prevDbKey = dbKeyBeforeRename; - } else { - // In OMKeyDeleteResponseWithFSO OzonePathKey is converted to - // OzoneDeletePathKey. Changing it back to check the previous DirTable. - prevDbKey = ozoneManager.getMetadataManager() - .getOzoneDeletePathDirKey(deletedDirDbKey); - } - - OmDirectoryInfo prevDirectoryInfo = previousDirTable.get(prevDbKey); - if (prevDirectoryInfo == null) { - return true; - } - - return prevDirectoryInfo.getObjectID() != deletedDirInfo.getObjectID(); - } - - protected boolean isRenameEntryReclaimable(Table.KeyValue renameEntry, - Table previousDirTable, - Table prevKeyInfoTable) throws IOException { - - if (previousDirTable == null && prevKeyInfoTable == null) { - return true; - } - String prevDbKey = renameEntry.getValue(); - - - if (previousDirTable != null) { - OmDirectoryInfo prevDirectoryInfo = previousDirTable.getIfExist(prevDbKey); - if (prevDirectoryInfo != null) { - return false; - } - } - - if (prevKeyInfoTable != null) { - OmKeyInfo omKeyInfo = prevKeyInfoTable.getIfExist(prevDbKey); - return omKeyInfo == null; - } - return true; - } - - - public boolean isRatisEnabled() { if (ozoneManager == null) { return false; @@ -877,377 +571,4 @@ protected void submitSetSnapshotRequest( LOG.error("Failed to submit set snapshot property request", e); } } - - /** - * Class to take multiple locks on a resource. - */ - protected static class MultiLocks { - private final Queue objectLocks; - private final IOzoneManagerLock lock; - private final OzoneManagerLock.Resource resource; - private final boolean writeLock; - public MultiLocks(IOzoneManagerLock lock, OzoneManagerLock.Resource resource, boolean writeLock) { - this.writeLock = writeLock; - this.resource = resource; - this.lock = lock; - this.objectLocks = new LinkedList<>(); - } - - public OMLockDetails acquireLock(Collection objects) throws OMException { - if (!objectLocks.isEmpty()) { - throw new OMException("More locks cannot be acquired when locks have been already acquired. Locks acquired : " - + objectLocks, OMException.ResultCodes.INTERNAL_ERROR); - } - OMLockDetails omLockDetails = OMLockDetails.EMPTY_DETAILS_LOCK_ACQUIRED; - for (T object : objects) { - if (object != null) { - omLockDetails = this.writeLock ? lock.acquireWriteLock(resource, object.toString()) - : lock.acquireReadLock(resource, object.toString()); - objectLocks.add(object); - if (!omLockDetails.isLockAcquired()) { - break; - } - } - } - if (!omLockDetails.isLockAcquired()) { - releaseLock(); - } - return omLockDetails; - } - - public void releaseLock() { - while (!objectLocks.isEmpty()) { - T object = objectLocks.poll(); - OMLockDetails lockDetails = this.writeLock ? lock.releaseWriteLock(resource, object.toString()) - : lock.releaseReadLock(resource, object.toString()); - } - } - } - - /** - * This class is responsible for opening last N snapshot given snapshot or AOS metadata manager by acquiring a lock. - */ - private abstract class ReclaimableFilter implements CheckedExceptionOperation, - Boolean, IOException>, Closeable { - - private final SnapshotInfo currentSnapshotInfo; - private final OmSnapshotManager omSnapshotManager; - private final SnapshotChainManager snapshotChainManager; - - private final List previousSnapshotInfos; - private final List> previousOmSnapshots; - private final MultiLocks snapshotIdLocks; - private Long volumeId; - private OmBucketInfo bucketInfo; - private final OMMetadataManager metadataManager; - private final int numberOfPreviousSnapshotsFromChain; - - /** - * Filter to return deleted keys/directories which are reclaimable based on their presence in previous snapshot in - * the snapshot chain. - * @param omSnapshotManager - * @param snapshotChainManager - * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot - * in the snapshot chain corresponding to bucket key needs to be processed. - * @param metadataManager : MetadataManager corresponding to snapshot or AOS. - * @param lock : Lock for Active OM. - */ - private ReclaimableFilter(OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, - SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, - IOzoneManagerLock lock, - int numberOfPreviousSnapshotsFromChain) { - this.omSnapshotManager = omSnapshotManager; - this.currentSnapshotInfo = currentSnapshotInfo; - this.snapshotChainManager = snapshotChainManager; - this.snapshotIdLocks = new MultiLocks<>(lock, OzoneManagerLock.Resource.SNAPSHOT_GC_LOCK, false); - this.metadataManager = metadataManager; - this.numberOfPreviousSnapshotsFromChain = numberOfPreviousSnapshotsFromChain; - this.previousOmSnapshots = new ArrayList<>(numberOfPreviousSnapshotsFromChain); - this.previousSnapshotInfos = new ArrayList<>(numberOfPreviousSnapshotsFromChain); - } - - private List getLastNSnapshotInChain(String volume, String bucket) throws IOException { - if (currentSnapshotInfo != null && (!currentSnapshotInfo.getVolumeName().equals(volume) || - !currentSnapshotInfo.getBucketName().equals(bucket))) { - throw new IOException("Volume & Bucket name for snapshot : " + currentSnapshotInfo + " not matching for " + - "key in volume: " + volume + " bucket: " + bucket); - } - SnapshotInfo expectedPreviousSnapshotInfo = currentSnapshotInfo == null - ? SnapshotUtils.getLatestSnapshotInfo(volume, bucket, ozoneManager, snapshotChainManager) - : SnapshotUtils.getPreviousSnapshot(ozoneManager, snapshotChainManager, currentSnapshotInfo); - List snapshotInfos = new ArrayList<>(); - snapshotInfos.add(expectedPreviousSnapshotInfo); - SnapshotInfo snapshotInfo = expectedPreviousSnapshotInfo; - while (snapshotInfos.size() < numberOfPreviousSnapshotsFromChain) { - snapshotInfo = snapshotInfo == null ? null - : SnapshotUtils.getPreviousSnapshot(ozoneManager, snapshotChainManager, expectedPreviousSnapshotInfo); - snapshotInfos.add(snapshotInfo); - // If changes made to the snapshot have not been flushed to disk, throw exception immediately, next run of - // garbage collection would process the snapshot. - if (!OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapshotInfo)) { - throw new IOException("Changes made to the snapshot " + snapshotInfo + " have not been flushed to the disk "); - } - } - - // Reversing list to get the correct order in chain. To ensure locking order is as per the chain ordering. - Collections.reverse(snapshotInfos); - return snapshotInfos; - } - - private boolean validateExistingLastNSnapshotsInChain(String volume, String bucket) throws IOException { - List expectedLastNSnapshotsInChain = getLastNSnapshotInChain(volume, bucket); - List expectedSnapshotIds = expectedLastNSnapshotsInChain.stream() - .map(snapshotInfo -> snapshotInfo == null ? null : snapshotInfo.getSnapshotId()) - .collect(Collectors.toList()); - List existingSnapshotIds = previousOmSnapshots.stream() - .map(omSnapshotReferenceCounted -> omSnapshotReferenceCounted == null ? null : - omSnapshotReferenceCounted.get().getSnapshotID()).collect(Collectors.toList()); - return expectedSnapshotIds.equals(existingSnapshotIds); - } - - // Initialize the last N snapshots in the chain by acquiring locks. Throw IOException if it fails. - private void initializePreviousSnapshotsFromChain(String volume, String bucket) throws IOException { - // If existing snapshotIds don't match then close all snapshots and reopen the previous N snapshots. - if (!validateExistingLastNSnapshotsInChain(volume, bucket)) { - close(); - try { - // Acquire lock only on last N-1 snapshot & current snapshot(AOS if it is null). - List expectedLastNSnapshotsInChain = getLastNSnapshotInChain(volume, bucket); - List expectedSnapshotIds = expectedLastNSnapshotsInChain.stream() - .map(snapshotInfo -> snapshotInfo == null ? null : snapshotInfo.getSnapshotId()) - .collect(Collectors.toList()); - List lockIds = new ArrayList<>(expectedSnapshotIds.subList(1, expectedSnapshotIds.size())); - lockIds.add(currentSnapshotInfo == null ? null : currentSnapshotInfo.getSnapshotId()); - - if (snapshotIdLocks.acquireLock(lockIds).isLockAcquired()) { - for (SnapshotInfo snapshotInfo : expectedLastNSnapshotsInChain) { - if (snapshotInfo != null) { - // For AOS fail operation if any of the previous snapshots are not active. currentSnapshotInfo for - // AOS will be null. - previousOmSnapshots.add(currentSnapshotInfo == null - ? omSnapshotManager.getActiveSnapshot(snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), - snapshotInfo.getName()) - : omSnapshotManager.getSnapshot(snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), - snapshotInfo.getName())); - previousSnapshotInfos.add(snapshotInfo); - } else { - previousOmSnapshots.add(null); - previousSnapshotInfos.add(null); - } - - // TODO: Getting volumeId and bucket from active OM. This would be wrong on volume & bucket renames - // support. - volumeId = getOzoneManager().getMetadataManager().getVolumeId(volume); - String dbBucketKey = getOzoneManager().getMetadataManager().getBucketKey(volume, bucket); - bucketInfo = getOzoneManager().getMetadataManager().getBucketTable().get(dbBucketKey); - } - } else { - throw new IOException("Lock acquisition failed for last N snapshots : " + - expectedLastNSnapshotsInChain + " " + currentSnapshotInfo); - } - } catch (IOException e) { - this.close(); - throw e; - } - } - } - - @Override - public Boolean apply(Table.KeyValue keyValue) throws IOException { - String volume = getVolumeName(keyValue); - String bucket = getBucketName(keyValue); - initializePreviousSnapshotsFromChain(volume, bucket); - boolean isReclaimable = isReclaimable(keyValue); - // This is to ensure the reclamation ran on the same previous snapshot and no change occurred in the chain - // while processing the entry. - return isReclaimable && validateExistingLastNSnapshotsInChain(volume, bucket); - } - - protected abstract String getVolumeName(Table.KeyValue keyValue) throws IOException; - protected abstract String getBucketName(Table.KeyValue keyValue) throws IOException; - - protected abstract Boolean isReclaimable(Table.KeyValue omKeyInfo) throws IOException; - - @Override - public void close() throws IOException { - this.snapshotIdLocks.releaseLock(); - for (ReferenceCounted previousOmSnapshot : previousOmSnapshots) { - previousOmSnapshot.close(); - } - previousOmSnapshots.clear(); - previousSnapshotInfos.clear(); - } - - public ReferenceCounted getPreviousOmSnapshot(int index) { - return previousOmSnapshots.get(index); - } - - public OMMetadataManager getMetadataManager() { - return metadataManager; - } - - public Long getVolumeId() { - return volumeId; - } - - public OmBucketInfo getBucketInfo() { - return bucketInfo; - } - - public SnapshotInfo getPreviousSnapshotInfo(int index) { - return previousSnapshotInfos.get(index); - } - } - - protected class ReclaimableKeyFilter extends ReclaimableFilter { - private final Map exclusiveSizeMap; - private final Map exclusiveReplicatedSizeMap; - - /** - * Filter to return deleted keys which are reclaimable based on their presence in previous snapshot in - * the snapshot chain. - * @param omSnapshotManager - * @param snapshotChainManager - * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot - * in the snapshot chain corresponding to bucket key needs to be processed. - * @param metadataManager : MetadataManager corresponding to snapshot or AOS. - * @param lock : Lock for Active OM. - */ - public ReclaimableKeyFilter(OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, - SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, - IOzoneManagerLock lock) { - super(omSnapshotManager, snapshotChainManager, currentSnapshotInfo, metadataManager, lock, 2); - this.exclusiveSizeMap = new HashMap<>(); - this.exclusiveReplicatedSizeMap = new HashMap<>(); - } - - @Override - protected String getVolumeName(Table.KeyValue keyValue) throws IOException { - return keyValue.getValue().getVolumeName(); - } - - @Override - protected String getBucketName(Table.KeyValue keyValue) throws IOException { - return keyValue.getValue().getBucketName(); - } - - @Override - protected Boolean isReclaimable(Table.KeyValue deletedKeyInfo) throws IOException { - ReferenceCounted previousSnapshot = getPreviousOmSnapshot(1); - ReferenceCounted previousToPreviousSnapshot = getPreviousOmSnapshot(0); - - Table previousKeyTable = null; - Table previousPrevKeyTable = null; - - Table renamedTable = getMetadataManager().getSnapshotRenamedTable(); - Table prevRenamedTable = null; - - SnapshotInfo previousSnapshotInfo = getPreviousSnapshotInfo(1); - SnapshotInfo prevPrevSnapshotInfo = getPreviousSnapshotInfo(0); - - if (previousSnapshot != null) { - previousKeyTable = previousSnapshot.get().getMetadataManager().getKeyTable(getBucketInfo().getBucketLayout()); - prevRenamedTable = previousSnapshot.get().getMetadataManager().getSnapshotRenamedTable(); - } - if (previousToPreviousSnapshot != null) { - previousPrevKeyTable = previousToPreviousSnapshot.get().getMetadataManager() - .getKeyTable(getBucketInfo().getBucketLayout()); - } - if (isKeyReclaimable(previousKeyTable, renamedTable, deletedKeyInfo.getValue(), getBucketInfo(), getVolumeId(), - null)) { - return true; - } - calculateExclusiveSize(previousSnapshotInfo, prevPrevSnapshotInfo, deletedKeyInfo.getValue(), getBucketInfo(), - getVolumeId(), renamedTable, previousKeyTable, prevRenamedTable, previousPrevKeyTable, exclusiveSizeMap, - exclusiveReplicatedSizeMap); - return false; - } - - - public Map getExclusiveSizeMap() { - return exclusiveSizeMap; - } - - public Map getExclusiveReplicatedSizeMap() { - return exclusiveReplicatedSizeMap; - } - } - - protected class ReclaimableDirFilter extends ReclaimableFilter { - - /** - * Filter to return deleted directories which are reclaimable based on their presence in previous snapshot in - * the snapshot chain. - * @param omSnapshotManager - * @param snapshotChainManager - * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot - * in the snapshot chain corresponding to bucket key needs to be processed. - * @param metadataManager : MetadataManager corresponding to snapshot or AOS. - * @param lock : Lock for Active OM. - */ - public ReclaimableDirFilter(OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, - SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, - IOzoneManagerLock lock) { - super(omSnapshotManager, snapshotChainManager, currentSnapshotInfo, metadataManager, lock, 1); - } - - @Override - protected String getVolumeName(Table.KeyValue keyValue) throws IOException { - return keyValue.getValue().getVolumeName(); - } - - @Override - protected String getBucketName(Table.KeyValue keyValue) throws IOException { - return keyValue.getValue().getBucketName(); - } - - @Override - protected Boolean isReclaimable(Table.KeyValue deletedDirInfo) throws IOException { - ReferenceCounted previousSnapshot = getPreviousOmSnapshot(0); - Table prevDirTable = previousSnapshot == null ? null : - previousSnapshot.get().getMetadataManager().getDirectoryTable(); - return isDirReclaimable(deletedDirInfo, prevDirTable, getMetadataManager().getSnapshotRenamedTable()); - } - } - - protected class ReclaimableRenameEntryFilter extends ReclaimableFilter { - - /** - * Filter to return rename table entries which are reclaimable based on the key presence in previous snapshot's - * keyTable/DirectoryTable in the snapshot chain. - * @param omSnapshotManager - * @param snapshotChainManager - * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot - * in the snapshot chain corresponding to bucket key needs to be processed. - * @param metadataManager : MetadataManager corresponding to snapshot or AOS. - * @param lock : Lock for Active OM. - */ - public ReclaimableRenameEntryFilter(OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, - SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, - IOzoneManagerLock lock) { - super(omSnapshotManager, snapshotChainManager, currentSnapshotInfo, metadataManager, lock, 1); - } - - @Override - protected Boolean isReclaimable(Table.KeyValue renameEntry) throws IOException { - ReferenceCounted previousSnapshot = getPreviousOmSnapshot(0); - Table previousKeyTable = null; - Table prevDirTable = null; - if (previousSnapshot != null) { - previousKeyTable = previousSnapshot.get().getMetadataManager().getKeyTable(getBucketInfo().getBucketLayout()); - prevDirTable = previousSnapshot.get().getMetadataManager().getDirectoryTable(); - } - return isRenameEntryReclaimable(renameEntry, prevDirTable, previousKeyTable); - } - - @Override - protected String getVolumeName(Table.KeyValue keyValue) throws IOException { - return getMetadataManager().splitRenameKey(keyValue.getKey())[0]; - } - - @Override - protected String getBucketName(Table.KeyValue keyValue) throws IOException { - return getMetadataManager().splitRenameKey(keyValue.getKey())[1]; - } - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index f58f91eb46ab..4300e6269d1e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -38,6 +38,8 @@ import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.om.snapshot.filter.ReclaimableDirFilter; +import org.apache.hadoop.ozone.om.snapshot.filter.ReclaimableKeyFilter; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PurgePathRequest; import org.apache.hadoop.util.Time; @@ -91,6 +93,7 @@ public class DirectoryDeletingService extends AbstractKeyDeletingService { private final long pathLimitPerTask; private final int ratisByteLimit; private final AtomicBoolean suspended; + private AtomicBoolean isRunningOnAOS; public DirectoryDeletingService(long interval, TimeUnit unit, long serviceTimeout, OzoneManager ozoneManager, @@ -107,6 +110,7 @@ public DirectoryDeletingService(long interval, TimeUnit unit, // always go to 90% of max limit for request as other header will be added this.ratisByteLimit = (int) (limit * 0.9); this.suspended = new AtomicBoolean(false); + this.isRunningOnAOS = new AtomicBoolean(false); } private boolean shouldRun() { @@ -117,6 +121,10 @@ private boolean shouldRun() { return getOzoneManager().isLeaderReady() && !suspended.get(); } + public boolean isRunningOnAOS() { + return isRunningOnAOS.get(); + } + /** * Suspend the service. */ @@ -209,10 +217,10 @@ private long processDeletedDirsForStore(SnapshotInfo currentSnapshotInfo, KeyMan SnapshotUtils.getPreviousSnapshotId(currentSnapshotInfo, snapshotChainManager); IOzoneManagerLock lock = getOzoneManager().getMetadataManager().getLock(); - try (ReclaimableDirFilter reclaimableDirFilter = new ReclaimableDirFilter(omSnapshotManager, - snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); - ReclaimableKeyFilter reclaimableSubFileFilter = new ReclaimableKeyFilter(omSnapshotManager, - snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock)) { + try (ReclaimableDirFilter reclaimableDirFilter = new ReclaimableDirFilter(getOzoneManager(), + omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); + ReclaimableKeyFilter reclaimableSubFileFilter = new ReclaimableKeyFilter(getOzoneManager(), + omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock)) { long startTime = Time.monotonicNow(); while (remainNum > 0 && deletedDirsIterator.hasNext()) { pendingDeletedDirInfo = deletedDirsIterator.next(); @@ -297,6 +305,7 @@ public BackgroundTaskResult call() { if (LOG.isDebugEnabled()) { LOG.debug("Running DirectoryDeletingService"); } + isRunningOnAOS.set(true); getRunCount().incrementAndGet(); long remainNum = pathLimitPerTask; try { @@ -305,6 +314,8 @@ public BackgroundTaskResult call() { } catch (IOException e) { LOG.error("Error while running delete directories and files " + "background task. Will retry at next run. on active object store", e); + } finally { + isRunningOnAOS.set(false); } if (remainNum > 0) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 013f1ce1c6c0..45e97378de68 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -43,6 +43,8 @@ import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.om.snapshot.filter.ReclaimableKeyFilter; +import org.apache.hadoop.ozone.om.snapshot.filter.ReclaimableRenameEntryFilter; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; import org.apache.hadoop.hdds.utils.BackgroundTask; @@ -82,6 +84,7 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private final AtomicLong deletedKeyCount; private final AtomicBoolean suspended; private final boolean deepCleanSnapshots; + private AtomicBoolean isRunningOnAOS; public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, @@ -99,6 +102,7 @@ public KeyDeletingService(OzoneManager ozoneManager, this.deletedKeyCount = new AtomicLong(0); this.suspended = new AtomicBoolean(false); this.deepCleanSnapshots = deepCleanSnapshots; + this.isRunningOnAOS = new AtomicBoolean(false); } /** @@ -111,6 +115,10 @@ public AtomicLong getDeletedKeyCount() { return deletedKeyCount; } + public boolean isRunningOnAOS() { + return isRunningOnAOS.get(); + } + @Override public BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); @@ -211,10 +219,11 @@ private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyMana // Purge deleted Keys in the deletedTable && rename entries in the snapshotRenamedTable which doesn't have a // reference in the previous snapshot. - try (ReclaimableKeyFilter reclaimableKeyFilter = new ReclaimableKeyFilter(omSnapshotManager, + try (ReclaimableKeyFilter reclaimableKeyFilter = new ReclaimableKeyFilter(getOzoneManager(), + omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); ReclaimableRenameEntryFilter renameEntryFilter = new ReclaimableRenameEntryFilter( - omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock)) { + getOzoneManager(), omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock)) { List renamedTableEntries = keyManager.getRenamesKeyEntries(volume, bucket, startKey, renameEntryFilter, remainNum).stream() .map(entry -> { @@ -287,7 +296,7 @@ public BackgroundTaskResult call() { if (shouldRun()) { final long run = getRunCount().incrementAndGet(); LOG.debug("Running KeyDeletingService {}", run); - + isRunningOnAOS.set(true); int remainNum = keyLimitPerTask; try { remainNum = processDeletedKeysForStore(null, getOzoneManager().getKeyManager(), @@ -295,6 +304,8 @@ public BackgroundTaskResult call() { } catch (IOException e) { LOG.error("Error while running delete directories and files " + "background task. Will retry at next run. on active object store", e); + } finally { + isRunningOnAOS.set(false); } if (deepCleanSnapshots && remainNum > 0) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 13f9a33abfcc..4a971b8d23e1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; +import org.apache.hadoop.ozone.om.lock.MultiLocks; import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; @@ -86,7 +87,7 @@ public class SnapshotDeletingService extends AbstractKeyDeletingService { private final OzoneManager ozoneManager; private final OmSnapshotManager omSnapshotManager; - private final SnapshotChainManager snapshotChainManager; + private final SnapshotChainManager chainManager; private final AtomicBoolean suspended; private final OzoneConfiguration conf; private final AtomicLong successRunCount; @@ -154,12 +155,11 @@ public BackgroundTaskResult call() throws InterruptedException { continue; } SnapshotInfo nextToNextSnapshot = nextSnapshot == null ? null : SnapshotUtils.getNextSnapshot(ozoneManager, - snapshotChainManager, snapInfo); + chainManager, snapInfo); // Wait for the next iteration if the next snapshot or next to next snapshot is still not deep cleaned // since purge transaction will add entries and it could be processed by mistake. - if ((nextSnapshot != null && isSnapshotDeepCleaned(nextSnapshot)) || - (nextToNextSnapshot != null && isSnapshotDeepCleaned(nextToNextSnapshot))) { + if (isSnapshotNotDeepCleaned(nextSnapshot) || isSnapshotNotDeepCleaned(nextToNextSnapshot)) { continue; } // Acquire write lock on current snapshot and next snapshot in chain. @@ -183,7 +183,7 @@ public BackgroundTaskResult call() throws InterruptedException { moveCount += deletedDirEntries.size(); // Get all entries from snapshotRenamedTable. List> renameEntries = snapshotKeyManager.getRenamesKeyEntries( - snapInfo.getVolumeName(), snapInfo.getBucketName(), null, remaining - moveCount); + snapInfo.getVolumeName(), snapInfo.getBucketName(), null, (kv) -> true,remaining - moveCount); moveCount += renameEntries.size(); if (moveCount > 0) { List deletedKeys = new ArrayList<>(deletedKeyEntries.size()); @@ -304,10 +304,13 @@ boolean shouldIgnoreSnapshot(SnapshotInfo snapInfo) throws IOException { SnapshotInfo.SnapshotStatus snapshotStatus = snapInfo.getSnapshotStatus(); return snapshotStatus != SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED || !OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapInfo) - || !isSnapshotDeepCleaned(snapInfo); + || isSnapshotNotDeepCleaned(snapInfo); } - private boolean isSnapshotDeepCleaned(SnapshotInfo snapInfo) { - return (!snapInfo.getDeepClean() || !snapInfo.getDeepCleanedDeletedDir()); + + @VisibleForTesting + boolean isSnapshotNotDeepCleaned(SnapshotInfo snapInfo) { + // if snapshot is null it means snapshot doesn't exist, so it means deep cleaned. + return snapInfo != null && !(snapInfo.getDeepCleanedDeletedDir() && snapInfo.getDeepClean()); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 906af5e21c38..c6ee5709409c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -46,7 +46,6 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithObjectID; import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; -import org.apache.hadoop.ozone.om.service.AbstractKeyDeletingService; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; @@ -1417,7 +1416,7 @@ long generateDiffReport( private boolean isKeyModified(OmKeyInfo fromKey, OmKeyInfo toKey) { return !fromKey.isKeyInfoSame(toKey, false, false, false, false, true) - || !AbstractKeyDeletingService.isBlockLocationInfoSame( + || !SnapshotUtils.isBlockLocationInfoSame( fromKey, toKey); } @@ -1466,7 +1465,7 @@ private boolean isBlockLocationSame( "OmKeyInfo"); } - return AbstractKeyDeletingService.isBlockLocationInfoSame( + return SnapshotUtils.isBlockLocationInfoSame( (OmKeyInfo) fromObject, (OmKeyInfo) toObject); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index 53e1ca96f294..c9c104db1613 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.snapshot; +import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -25,6 +26,8 @@ import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; @@ -194,7 +197,7 @@ public static SnapshotInfo getPreviousSnapshot(OzoneManager ozoneManager, /** * Get the previous snapshot in the snapshot chain. */ - private static UUID getPreviousSnapshotId(SnapshotInfo snapInfo, SnapshotChainManager chainManager) + public static UUID getPreviousSnapshotId(SnapshotInfo snapInfo, SnapshotChainManager chainManager) throws IOException { // If the snapshot is deleted in the previous run, then the in-memory // SnapshotChainManager might throw NoSuchElementException as the snapshot @@ -356,4 +359,56 @@ public static void validatePreviousSnapshotId(SnapshotInfo snapshotInfo, OMException.ResultCodes.INVALID_REQUEST); } } + + public static boolean isBlockLocationInfoSame(OmKeyInfo prevKeyInfo, + OmKeyInfo deletedKeyInfo) { + + if (prevKeyInfo == null && deletedKeyInfo == null) { + BackgroundService.LOG.debug("Both prevKeyInfo and deletedKeyInfo are null."); + return true; + } + if (prevKeyInfo == null || deletedKeyInfo == null) { + BackgroundService.LOG.debug("prevKeyInfo: '{}' or deletedKeyInfo: '{}' is null.", + prevKeyInfo, deletedKeyInfo); + return false; + } + // For hsync, Though the blockLocationInfo of a key may not be same + // at the time of snapshot and key deletion as blocks can be appended. + // If the objectId is same then the key is same. + if (prevKeyInfo.isHsync() && deletedKeyInfo.isHsync()) { + return true; + } + + if (prevKeyInfo.getKeyLocationVersions().size() != + deletedKeyInfo.getKeyLocationVersions().size()) { + return false; + } + + OmKeyLocationInfoGroup deletedOmKeyLocation = + deletedKeyInfo.getLatestVersionLocations(); + OmKeyLocationInfoGroup prevOmKeyLocation = + prevKeyInfo.getLatestVersionLocations(); + + if (deletedOmKeyLocation == null || prevOmKeyLocation == null) { + return false; + } + + List deletedLocationList = + deletedOmKeyLocation.getLocationList(); + List prevLocationList = + prevOmKeyLocation.getLocationList(); + + if (deletedLocationList.size() != prevLocationList.size()) { + return false; + } + + for (int idx = 0; idx < deletedLocationList.size(); idx++) { + OmKeyLocationInfo deletedLocationInfo = deletedLocationList.get(idx); + OmKeyLocationInfo prevLocationInfo = prevLocationList.get(idx); + if (!deletedLocationInfo.hasSameBlockAs(prevLocationInfo)) { + return false; + } + } + return true; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableDirFilter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableDirFilter.java new file mode 100644 index 000000000000..ab561ac56d60 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableDirFilter.java @@ -0,0 +1,95 @@ +package org.apache.hadoop.ozone.om.snapshot.filter; + +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; +import org.apache.hadoop.ozone.om.service.AbstractKeyDeletingService; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; + +import java.io.IOException; + +public class ReclaimableDirFilter extends ReclaimableFilter { + + private final OzoneManager ozoneManager; + + /** + * Filter to return deleted directories which are reclaimable based on their presence in previous snapshot in + * the snapshot chain. + * + * @param omSnapshotManager + * @param snapshotChainManager + * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot + * in the snapshot chain corresponding to bucket key needs to be processed. + * @param metadataManager : MetadataManager corresponding to snapshot or AOS. + * @param lock : Lock for Active OM. + */ + public ReclaimableDirFilter(OzoneManager ozoneManager, + OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, + SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, + IOzoneManagerLock lock) { + super(ozoneManager, omSnapshotManager, snapshotChainManager, currentSnapshotInfo, metadataManager, lock, 1); + this.ozoneManager = ozoneManager; + } + + @Override + protected String getVolumeName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getVolumeName(); + } + + @Override + protected String getBucketName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getBucketName(); + } + + @Override + protected Boolean isReclaimable(Table.KeyValue deletedDirInfo) throws IOException { + ReferenceCounted previousSnapshot = getPreviousOmSnapshot(0); + Table prevDirTable = previousSnapshot == null ? null : + previousSnapshot.get().getMetadataManager().getDirectoryTable(); + return isDirReclaimable(deletedDirInfo, prevDirTable, + getMetadataManager().getSnapshotRenamedTable()); + } + + private boolean isDirReclaimable(Table.KeyValue deletedDir, + Table previousDirTable, + Table renamedTable) throws IOException { + if (previousDirTable == null) { + return true; + } + + String deletedDirDbKey = deletedDir.getKey(); + OmKeyInfo deletedDirInfo = deletedDir.getValue(); + String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey( + deletedDirInfo.getVolumeName(), deletedDirInfo.getBucketName(), + deletedDirInfo.getObjectID()); + + /* + snapshotRenamedTable: /volumeName/bucketName/objectID -> + /volumeId/bucketId/parentId/dirName + */ + String dbKeyBeforeRename = renamedTable.getIfExist(dbRenameKey); + String prevDbKey = null; + + if (dbKeyBeforeRename != null) { + prevDbKey = dbKeyBeforeRename; + } else { + // In OMKeyDeleteResponseWithFSO OzonePathKey is converted to + // OzoneDeletePathKey. Changing it back to check the previous DirTable. + prevDbKey = ozoneManager.getMetadataManager() + .getOzoneDeletePathDirKey(deletedDirDbKey); + } + + OmDirectoryInfo prevDirectoryInfo = previousDirTable.get(prevDbKey); + if (prevDirectoryInfo == null) { + return true; + } + return prevDirectoryInfo.getObjectID() != deletedDirInfo.getObjectID(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableFilter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableFilter.java new file mode 100644 index 000000000000..3c2bfa8cb18f --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableFilter.java @@ -0,0 +1,204 @@ +package org.apache.hadoop.ozone.om.snapshot.filter; + +import com.google.common.collect.Lists; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; +import org.apache.hadoop.ozone.om.lock.MultiLocks; +import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.util.CheckedExceptionOperation; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * This class is responsible for opening last N snapshot given snapshot or AOS metadata manager by acquiring a lock. + */ +public abstract class ReclaimableFilter implements CheckedExceptionOperation, + Boolean, IOException>, Closeable { + + private final OzoneManager ozoneManager; + private final SnapshotInfo currentSnapshotInfo; + private final OmSnapshotManager omSnapshotManager; + private final SnapshotChainManager snapshotChainManager; + + private final List previousSnapshotInfos; + private final List> previousOmSnapshots; + private final MultiLocks snapshotIdLocks; + private Long volumeId; + private OmBucketInfo bucketInfo; + private final OMMetadataManager metadataManager; + private final int numberOfPreviousSnapshotsFromChain; + + /** + * Filter to return deleted keys/directories which are reclaimable based on their presence in previous snapshot in + * the snapshot chain. + * + * @param omSnapshotManager + * @param snapshotChainManager + * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot + * in the snapshot chain corresponding to bucket key needs to be processed. + * @param metadataManager : MetadataManager corresponding to snapshot or AOS. + * @param lock : Lock for Active OM. + */ + public ReclaimableFilter(OzoneManager ozoneManager, OmSnapshotManager omSnapshotManager, + SnapshotChainManager snapshotChainManager, + SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, + IOzoneManagerLock lock, + int numberOfPreviousSnapshotsFromChain) { + this.ozoneManager = ozoneManager; + this.omSnapshotManager = omSnapshotManager; + this.currentSnapshotInfo = currentSnapshotInfo; + this.snapshotChainManager = snapshotChainManager; + this.snapshotIdLocks = new MultiLocks<>(lock, OzoneManagerLock.Resource.SNAPSHOT_GC_LOCK, false); + this.metadataManager = metadataManager; + this.numberOfPreviousSnapshotsFromChain = numberOfPreviousSnapshotsFromChain; + this.previousOmSnapshots = new ArrayList<>(numberOfPreviousSnapshotsFromChain); + this.previousSnapshotInfos = new ArrayList<>(numberOfPreviousSnapshotsFromChain); + } + + private List getLastNSnapshotInChain(String volume, String bucket) throws IOException { + if (currentSnapshotInfo != null && + (!currentSnapshotInfo.getVolumeName().equals(volume) || !currentSnapshotInfo.getBucketName().equals(bucket))) { + throw new IOException("Volume & Bucket name for snapshot : " + currentSnapshotInfo + " not matching for " + + "key in volume: " + volume + " bucket: " + bucket); + } + SnapshotInfo expectedPreviousSnapshotInfo = currentSnapshotInfo == null + ? SnapshotUtils.getLatestSnapshotInfo(volume, bucket, ozoneManager, snapshotChainManager) + : SnapshotUtils.getPreviousSnapshot(ozoneManager, snapshotChainManager, currentSnapshotInfo); + List snapshotInfos = Lists.newArrayList(expectedPreviousSnapshotInfo); + SnapshotInfo snapshotInfo = expectedPreviousSnapshotInfo; + while (snapshotInfos.size() < numberOfPreviousSnapshotsFromChain) { + snapshotInfo = snapshotInfo == null ? null + : SnapshotUtils.getPreviousSnapshot(ozoneManager, snapshotChainManager, snapshotInfo); + snapshotInfos.add(snapshotInfo); + // If changes made to the snapshot have not been flushed to disk, throw exception immediately, next run of + // garbage collection would process the snapshot. + if (!OmSnapshotManager.areSnapshotChangesFlushedToDB(ozoneManager.getMetadataManager(), snapshotInfo)) { + throw new IOException("Changes made to the snapshot " + snapshotInfo + " have not been flushed to the disk "); + } + } + + // Reversing list to get the correct order in chain. To ensure locking order is as per the chain ordering. + Collections.reverse(snapshotInfos); + return snapshotInfos; + } + + private boolean validateExistingLastNSnapshotsInChain(String volume, String bucket) throws IOException { + List expectedLastNSnapshotsInChain = getLastNSnapshotInChain(volume, bucket); + List expectedSnapshotIds = expectedLastNSnapshotsInChain.stream() + .map(snapshotInfo -> snapshotInfo == null ? null : snapshotInfo.getSnapshotId()) + .collect(Collectors.toList()); + List existingSnapshotIds = previousOmSnapshots.stream() + .map(omSnapshotReferenceCounted -> omSnapshotReferenceCounted == null ? null : + omSnapshotReferenceCounted.get().getSnapshotID()).collect(Collectors.toList()); + return expectedSnapshotIds.equals(existingSnapshotIds); + } + + // Initialize the last N snapshots in the chain by acquiring locks. Throw IOException if it fails. + private void initializePreviousSnapshotsFromChain(String volume, String bucket) throws IOException { + // If existing snapshotIds don't match then close all snapshots and reopen the previous N snapshots. + if (!validateExistingLastNSnapshotsInChain(volume, bucket)) { + close(); + try { + // Acquire lock only on last N-1 snapshot & current snapshot(AOS if it is null). + List expectedLastNSnapshotsInChain = getLastNSnapshotInChain(volume, bucket); + List expectedSnapshotIds = expectedLastNSnapshotsInChain.stream() + .map(snapshotInfo -> snapshotInfo == null ? null : snapshotInfo.getSnapshotId()) + .collect(Collectors.toList()); + List lockIds = new ArrayList<>(expectedSnapshotIds.subList(1, expectedSnapshotIds.size())); + lockIds.add(currentSnapshotInfo == null ? null : currentSnapshotInfo.getSnapshotId()); + + if (snapshotIdLocks.acquireLock(lockIds).isLockAcquired()) { + for (SnapshotInfo snapshotInfo : expectedLastNSnapshotsInChain) { + if (snapshotInfo != null) { + // For AOS fail operation if any of the previous snapshots are not active. currentSnapshotInfo for + // AOS will be null. + previousOmSnapshots.add(currentSnapshotInfo == null + ? omSnapshotManager.getActiveSnapshot(snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), + snapshotInfo.getName()) + : omSnapshotManager.getSnapshot(snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), + snapshotInfo.getName())); + previousSnapshotInfos.add(snapshotInfo); + } else { + previousOmSnapshots.add(null); + previousSnapshotInfos.add(null); + } + + // TODO: Getting volumeId and bucket from active OM. This would be wrong on volume & bucket renames + // support. + volumeId = ozoneManager.getMetadataManager().getVolumeId(volume); + String dbBucketKey = ozoneManager.getMetadataManager().getBucketKey(volume, bucket); + bucketInfo = ozoneManager.getMetadataManager().getBucketTable().get(dbBucketKey); + } + } else { + throw new IOException("Lock acquisition failed for last N snapshots : " + + expectedLastNSnapshotsInChain + " " + currentSnapshotInfo); + } + } catch (IOException e) { + this.close(); + throw e; + } + } + } + + @Override + public Boolean apply(Table.KeyValue keyValue) throws IOException { + String volume = getVolumeName(keyValue); + String bucket = getBucketName(keyValue); + initializePreviousSnapshotsFromChain(volume, bucket); + boolean isReclaimable = isReclaimable(keyValue); + // This is to ensure the reclamation ran on the same previous snapshot and no change occurred in the chain + // while processing the entry. + return isReclaimable && validateExistingLastNSnapshotsInChain(volume, bucket); + } + + protected abstract String getVolumeName(Table.KeyValue keyValue) throws IOException; + + protected abstract String getBucketName(Table.KeyValue keyValue) throws IOException; + + protected abstract Boolean isReclaimable(Table.KeyValue omKeyInfo) throws IOException; + + @Override + public void close() throws IOException { + this.snapshotIdLocks.releaseLock(); + for (ReferenceCounted previousOmSnapshot : previousOmSnapshots) { + previousOmSnapshot.close(); + } + previousOmSnapshots.clear(); + previousSnapshotInfos.clear(); + } + + public ReferenceCounted getPreviousOmSnapshot(int index) { + return previousOmSnapshots.get(index); + } + + public OMMetadataManager getMetadataManager() { + return metadataManager; + } + + public Long getVolumeId() { + return volumeId; + } + + public OmBucketInfo getBucketInfo() { + return bucketInfo; + } + + public SnapshotInfo getPreviousSnapshotInfo(int index) { + return previousSnapshotInfos.get(index); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java new file mode 100644 index 000000000000..970a768fb0bd --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java @@ -0,0 +1,258 @@ +package org.apache.hadoop.ozone.om.snapshot.filter; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.isBlockLocationInfoSame; + +public class ReclaimableKeyFilter extends ReclaimableFilter { + private final OzoneManager ozoneManager; + private final Map exclusiveSizeMap; + private final Map exclusiveReplicatedSizeMap; + + /** + * Filter to return deleted keys which are reclaimable based on their presence in previous snapshot in + * the snapshot chain. + * + * @param omSnapshotManager + * @param snapshotChainManager + * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot + * in the snapshot chain corresponding to bucket key needs to be processed. + * @param metadataManager : MetadataManager corresponding to snapshot or AOS. + * @param lock : Lock for Active OM. + */ + public ReclaimableKeyFilter(OzoneManager ozoneManager, + OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, + SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, + IOzoneManagerLock lock) { + super(ozoneManager, omSnapshotManager, snapshotChainManager, currentSnapshotInfo, metadataManager, lock, 2); + this.ozoneManager = ozoneManager; + this.exclusiveSizeMap = new HashMap<>(); + this.exclusiveReplicatedSizeMap = new HashMap<>(); + } + + @Override + protected String getVolumeName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getVolumeName(); + } + + @Override + protected String getBucketName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getBucketName(); + } + + @Override + protected Boolean isReclaimable(Table.KeyValue deletedKeyInfo) throws IOException { + ReferenceCounted previousSnapshot = getPreviousOmSnapshot(1); + ReferenceCounted previousToPreviousSnapshot = getPreviousOmSnapshot(0); + + Table previousKeyTable = null; + Table previousPrevKeyTable = null; + + Table renamedTable = getMetadataManager().getSnapshotRenamedTable(); + Table prevRenamedTable = null; + + SnapshotInfo previousSnapshotInfo = getPreviousSnapshotInfo(1); + SnapshotInfo prevPrevSnapshotInfo = getPreviousSnapshotInfo(0); + + if (previousSnapshot != null) { + previousKeyTable = previousSnapshot.get().getMetadataManager().getKeyTable(getBucketInfo().getBucketLayout()); + prevRenamedTable = previousSnapshot.get().getMetadataManager().getSnapshotRenamedTable(); + } + if (previousToPreviousSnapshot != null) { + previousPrevKeyTable = previousToPreviousSnapshot.get().getMetadataManager() + .getKeyTable(getBucketInfo().getBucketLayout()); + } + if (isKeyReclaimable(previousKeyTable, renamedTable, deletedKeyInfo.getValue(), + getBucketInfo(), getVolumeId(), + null)) { + return true; + } + calculateExclusiveSize(previousSnapshotInfo, prevPrevSnapshotInfo, deletedKeyInfo.getValue(), getBucketInfo(), + getVolumeId(), renamedTable, previousKeyTable, prevRenamedTable, previousPrevKeyTable, exclusiveSizeMap, + exclusiveReplicatedSizeMap); + return false; + } + + + public Map getExclusiveSizeMap() { + return exclusiveSizeMap; + } + + public Map getExclusiveReplicatedSizeMap() { + return exclusiveReplicatedSizeMap; + } + + private boolean isKeyReclaimable( + Table previousKeyTable, + Table renamedTable, + OmKeyInfo deletedKeyInfo, OmBucketInfo bucketInfo, + long volumeId, HddsProtos.KeyValue.Builder renamedKeyBuilder) + throws IOException { + + String dbKey; + // Handle case when the deleted snapshot is the first snapshot. + if (previousKeyTable == null) { + return true; + } + + // These are uncommitted blocks wrapped into a pseudo KeyInfo + if (deletedKeyInfo.getObjectID() == OBJECT_ID_RECLAIM_BLOCKS) { + return true; + } + + // Construct keyTable or fileTable DB key depending on the bucket type + if (bucketInfo.getBucketLayout().isFileSystemOptimized()) { + dbKey = ozoneManager.getMetadataManager().getOzonePathKey( + volumeId, + bucketInfo.getObjectID(), + deletedKeyInfo.getParentObjectID(), + deletedKeyInfo.getFileName()); + } else { + dbKey = ozoneManager.getMetadataManager().getOzoneKey( + deletedKeyInfo.getVolumeName(), + deletedKeyInfo.getBucketName(), + deletedKeyInfo.getKeyName()); + } + + /* + snapshotRenamedTable: + 1) /volumeName/bucketName/objectID -> + /volumeId/bucketId/parentId/fileName (FSO) + 2) /volumeName/bucketName/objectID -> + /volumeName/bucketName/keyName (non-FSO) + */ + String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey( + deletedKeyInfo.getVolumeName(), deletedKeyInfo.getBucketName(), + deletedKeyInfo.getObjectID()); + + // Condition: key should not exist in snapshotRenamedTable + // of the current snapshot and keyTable of the previous snapshot. + // Check key exists in renamedTable of the Snapshot + String renamedKey = renamedTable.getIfExist(dbRenameKey); + + if (renamedKey != null && renamedKeyBuilder != null) { + renamedKeyBuilder.setKey(dbRenameKey).setValue(renamedKey); + } + // previousKeyTable is fileTable if the bucket is FSO, + // otherwise it is the keyTable. + OmKeyInfo prevKeyInfo = renamedKey != null ? previousKeyTable + .get(renamedKey) : previousKeyTable.get(dbKey); + + if (prevKeyInfo == null || + prevKeyInfo.getObjectID() != deletedKeyInfo.getObjectID()) { + return true; + } + + // For key overwrite the objectID will remain the same, In this + // case we need to check if OmKeyLocationInfo is also same. + return !isBlockLocationInfoSame(prevKeyInfo, deletedKeyInfo); + } + + /** + * To calculate Exclusive Size for current snapshot, Check + * the next snapshot deletedTable if the deleted key is + * referenced in current snapshot and not referenced in the + * previous snapshot then that key is exclusive to the current + * snapshot. Here since we are only iterating through + * deletedTable we can check the previous and previous to + * previous snapshot to achieve the same. + * previousSnapshot - Snapshot for which exclusive size is + * getting calculating. + * currSnapshot - Snapshot's deletedTable is used to calculate + * previousSnapshot snapshot's exclusive size. + * previousToPrevSnapshot - Snapshot which is used to check + * if key is exclusive to previousSnapshot. + */ + @SuppressWarnings("checkstyle:ParameterNumber") + public void calculateExclusiveSize( + SnapshotInfo previousSnapshot, + SnapshotInfo previousToPrevSnapshot, + OmKeyInfo keyInfo, + OmBucketInfo bucketInfo, long volumeId, + Table snapRenamedTable, + Table previousKeyTable, + Table prevRenamedTable, + Table previousToPrevKeyTable, + Map exclusiveSizeMap, + Map exclusiveReplicatedSizeMap) throws IOException { + String prevSnapKey = previousSnapshot.getTableKey(); + long exclusiveReplicatedSize = exclusiveReplicatedSizeMap.getOrDefault( + prevSnapKey, 0L) + keyInfo.getReplicatedSize(); + long exclusiveSize = exclusiveSizeMap.getOrDefault(prevSnapKey, 0L) + keyInfo.getDataSize(); + + // If there is no previous to previous snapshot, then + // the previous snapshot is the first snapshot. + if (previousToPrevSnapshot == null) { + exclusiveSizeMap.put(prevSnapKey, exclusiveSize); + exclusiveReplicatedSizeMap.put(prevSnapKey, + exclusiveReplicatedSize); + } else { + OmKeyInfo keyInfoPrevSnapshot = getPreviousSnapshotKeyName( + keyInfo, bucketInfo, volumeId, + snapRenamedTable, previousKeyTable); + OmKeyInfo keyInfoPrevToPrevSnapshot = getPreviousSnapshotKeyName( + keyInfoPrevSnapshot, bucketInfo, volumeId, + prevRenamedTable, previousToPrevKeyTable); + // If the previous to previous snapshot doesn't + // have the key, then it is exclusive size for the + // previous snapshot. + if (keyInfoPrevToPrevSnapshot == null) { + exclusiveSizeMap.put(prevSnapKey, exclusiveSize); + exclusiveReplicatedSizeMap.put(prevSnapKey, + exclusiveReplicatedSize); + } + } + } + + private OmKeyInfo getPreviousSnapshotKeyName(OmKeyInfo keyInfo, OmBucketInfo bucketInfo, long volumeId, + Table snapRenamedTable, Table previousKeyTable) throws IOException { + + if (keyInfo == null) { + return null; + } + + String dbKeyPrevSnap; + if (bucketInfo.getBucketLayout().isFileSystemOptimized()) { + dbKeyPrevSnap = ozoneManager.getMetadataManager().getOzonePathKey( + volumeId, + bucketInfo.getObjectID(), + keyInfo.getParentObjectID(), + keyInfo.getFileName()); + } else { + dbKeyPrevSnap = ozoneManager.getMetadataManager().getOzoneKey( + keyInfo.getVolumeName(), + keyInfo.getBucketName(), + keyInfo.getKeyName()); + } + + String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey( + keyInfo.getVolumeName(), + keyInfo.getBucketName(), + keyInfo.getObjectID()); + + String renamedKey = snapRenamedTable.getIfExist(dbRenameKey); + OmKeyInfo prevKeyInfo = renamedKey != null ? previousKeyTable.get(renamedKey) : previousKeyTable.get(dbKeyPrevSnap); + + if (prevKeyInfo == null || prevKeyInfo.getObjectID() != keyInfo.getObjectID()) { + return null; + } + + return isBlockLocationInfoSame(prevKeyInfo, keyInfo) ? prevKeyInfo : null; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableRenameEntryFilter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableRenameEntryFilter.java new file mode 100644 index 000000000000..60371d70a9a8 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableRenameEntryFilter.java @@ -0,0 +1,83 @@ +package org.apache.hadoop.ozone.om.snapshot.filter; + +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; +import org.apache.hadoop.ozone.om.service.AbstractKeyDeletingService; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; + +import java.io.IOException; + +public class ReclaimableRenameEntryFilter extends ReclaimableFilter { + + /** + * Filter to return rename table entries which are reclaimable based on the key presence in previous snapshot's + * keyTable/DirectoryTable in the snapshot chain. + * + * @param omSnapshotManager + * @param snapshotChainManager + * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot + * in the snapshot chain corresponding to bucket key needs to be processed. + * @param metadataManager : MetadataManager corresponding to snapshot or AOS. + * @param lock : Lock for Active OM. + */ + public ReclaimableRenameEntryFilter(OzoneManager ozoneManager, + OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, + SnapshotInfo currentSnapshotInfo, OMMetadataManager metadataManager, + IOzoneManagerLock lock) { + super(ozoneManager, omSnapshotManager, snapshotChainManager, currentSnapshotInfo, metadataManager, lock, 1); + } + + @Override + protected Boolean isReclaimable(Table.KeyValue renameEntry) throws IOException { + ReferenceCounted previousSnapshot = getPreviousOmSnapshot(0); + Table previousKeyTable = null; + Table prevDirTable = null; + if (previousSnapshot != null) { + previousKeyTable = previousSnapshot.get().getMetadataManager().getKeyTable(getBucketInfo().getBucketLayout()); + prevDirTable = previousSnapshot.get().getMetadataManager().getDirectoryTable(); + } + return isRenameEntryReclaimable(renameEntry, prevDirTable, previousKeyTable); + } + + @Override + protected String getVolumeName(Table.KeyValue keyValue) throws IOException { + return getMetadataManager().splitRenameKey(keyValue.getKey())[0]; + } + + @Override + protected String getBucketName(Table.KeyValue keyValue) throws IOException { + return getMetadataManager().splitRenameKey(keyValue.getKey())[1]; + } + + private boolean isRenameEntryReclaimable(Table.KeyValue renameEntry, + Table previousDirTable, + Table prevKeyInfoTable) throws IOException { + + if (previousDirTable == null && prevKeyInfoTable == null) { + return true; + } + String prevDbKey = renameEntry.getValue(); + + + if (previousDirTable != null) { + OmDirectoryInfo prevDirectoryInfo = previousDirTable.getIfExist(prevDbKey); + if (prevDirectoryInfo != null) { + return false; + } + } + + if (prevKeyInfoTable != null) { + OmKeyInfo omKeyInfo = prevKeyInfoTable.getIfExist(prevDbKey); + return omKeyInfo == null; + } + return true; + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java index 61c913b9bdd1..9fdcece46661 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java @@ -57,6 +57,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.util.CheckedExceptionOperation; import org.apache.ozone.test.OzoneTestBase; import org.apache.ratis.util.ExitUtils; import org.junit.jupiter.api.AfterAll; @@ -381,7 +382,8 @@ public void testAOSKeyDeletingWithSnapshotCreateParallelExecution() Assertions.assertNotEquals(deletePathKey[0], group.getGroupID()); } return pendingKeysDeletion; - }).when(km).getPendingDeletionKeys(anyInt()); + }).when(km).getPendingDeletionKeys(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), + Mockito.any(CheckedExceptionOperation.class), anyInt()); service.runPeriodicalTaskNow(); service.runPeriodicalTaskNow(); assertTableRowCount(snapshotInfoTable, initialSnapshotCount + 2, metadataManager); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 9483ac0c3747..7c342a87b9d6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -49,7 +49,6 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; -import org.apache.hadoop.ozone.om.service.AbstractKeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.om.snapshot.SnapshotTestUtils.StubbedPersistentMap; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; @@ -800,7 +799,7 @@ public void testGenerateDiffReport() throws IOException { mockedSnapshotDeletingService = mockStatic( SnapshotDeletingService.class)) { mockedSnapshotDeletingService.when(() -> - AbstractKeyDeletingService.isBlockLocationInfoSame(any(OmKeyInfo.class), + SnapshotUtils.isBlockLocationInfoSame(any(OmKeyInfo.class), any(OmKeyInfo.class))) .thenAnswer(i -> { int keyVal = Integer.parseInt(((OmKeyInfo)i.getArgument(0)) From 91a4c426285668c343d424ef012e1f1295814d59 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 21 Oct 2024 14:03:46 -0700 Subject: [PATCH 22/22] HDDS-11244. Fix Checkstyle Change-Id: I7b0cfb02c6db9237a08853b0659be362e54dc729 --- .../hadoop/ozone/om/KeyManagerImpl.java | 1 - .../ozone/om/service/KeyDeletingService.java | 3 +- .../om/service/SnapshotDeletingService.java | 2 +- .../snapshot/filter/ReclaimableDirFilter.java | 23 ++++++++++- .../om/snapshot/filter/ReclaimableFilter.java | 18 ++++++++ .../snapshot/filter/ReclaimableKeyFilter.java | 41 ++++++++++++++----- .../filter/ReclaimableRenameEntryFilter.java | 26 ++++++++++-- .../om/snapshot/filter/package-info.java | 22 ++++++++++ 8 files changed, 118 insertions(+), 18 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/package-info.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index db0cdf875e28..570392da7985 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -98,7 +98,6 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 45e97378de68..14c7c7c9d21b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -223,7 +223,8 @@ private int processDeletedKeysForStore(SnapshotInfo currentSnapshotInfo, KeyMana omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock); ReclaimableRenameEntryFilter renameEntryFilter = new ReclaimableRenameEntryFilter( - getOzoneManager(), omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager.getMetadataManager(), lock)) { + getOzoneManager(), omSnapshotManager, snapshotChainManager, currentSnapshotInfo, + keyManager.getMetadataManager(), lock)) { List renamedTableEntries = keyManager.getRenamesKeyEntries(volume, bucket, startKey, renameEntryFilter, remainNum).stream() .map(entry -> { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 4a971b8d23e1..85e3c8b9da85 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -183,7 +183,7 @@ public BackgroundTaskResult call() throws InterruptedException { moveCount += deletedDirEntries.size(); // Get all entries from snapshotRenamedTable. List> renameEntries = snapshotKeyManager.getRenamesKeyEntries( - snapInfo.getVolumeName(), snapInfo.getBucketName(), null, (kv) -> true,remaining - moveCount); + snapInfo.getVolumeName(), snapInfo.getBucketName(), null, (kv) -> true, remaining - moveCount); moveCount += renameEntries.size(); if (moveCount > 0) { List deletedKeys = new ArrayList<>(deletedKeyEntries.size()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableDirFilter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableDirFilter.java index ab561ac56d60..3410d73c8148 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableDirFilter.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableDirFilter.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package org.apache.hadoop.ozone.om.snapshot.filter; import org.apache.hadoop.hdds.utils.db.Table; @@ -10,11 +28,14 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; -import org.apache.hadoop.ozone.om.service.AbstractKeyDeletingService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import java.io.IOException; +/** + * Filter to return deleted directories which are reclaimable based on their presence in previous snapshot in + * the snapshot chain. + */ public class ReclaimableDirFilter extends ReclaimableFilter { private final OzoneManager ozoneManager; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableFilter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableFilter.java index 3c2bfa8cb18f..200a35c47724 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableFilter.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableFilter.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package org.apache.hadoop.ozone.om.snapshot.filter; import com.google.common.collect.Lists; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java index 970a768fb0bd..00c7d30de882 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package org.apache.hadoop.ozone.om.snapshot.filter; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -20,15 +38,16 @@ import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.isBlockLocationInfoSame; +/** + * Filter to return deleted keys which are reclaimable based on their presence in previous snapshot in + * the snapshot chain. + */ public class ReclaimableKeyFilter extends ReclaimableFilter { private final OzoneManager ozoneManager; private final Map exclusiveSizeMap; private final Map exclusiveReplicatedSizeMap; /** - * Filter to return deleted keys which are reclaimable based on their presence in previous snapshot in - * the snapshot chain. - * * @param omSnapshotManager * @param snapshotChainManager * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot @@ -189,18 +208,18 @@ public void calculateExclusiveSize( Table previousKeyTable, Table prevRenamedTable, Table previousToPrevKeyTable, - Map exclusiveSizeMap, - Map exclusiveReplicatedSizeMap) throws IOException { + Map exclusiveSizes, + Map exclusiveReplicatedSizes) throws IOException { String prevSnapKey = previousSnapshot.getTableKey(); - long exclusiveReplicatedSize = exclusiveReplicatedSizeMap.getOrDefault( + long exclusiveReplicatedSize = exclusiveReplicatedSizes.getOrDefault( prevSnapKey, 0L) + keyInfo.getReplicatedSize(); - long exclusiveSize = exclusiveSizeMap.getOrDefault(prevSnapKey, 0L) + keyInfo.getDataSize(); + long exclusiveSize = exclusiveSizes.getOrDefault(prevSnapKey, 0L) + keyInfo.getDataSize(); // If there is no previous to previous snapshot, then // the previous snapshot is the first snapshot. if (previousToPrevSnapshot == null) { - exclusiveSizeMap.put(prevSnapKey, exclusiveSize); - exclusiveReplicatedSizeMap.put(prevSnapKey, + exclusiveSizes.put(prevSnapKey, exclusiveSize); + exclusiveReplicatedSizes.put(prevSnapKey, exclusiveReplicatedSize); } else { OmKeyInfo keyInfoPrevSnapshot = getPreviousSnapshotKeyName( @@ -213,8 +232,8 @@ public void calculateExclusiveSize( // have the key, then it is exclusive size for the // previous snapshot. if (keyInfoPrevToPrevSnapshot == null) { - exclusiveSizeMap.put(prevSnapKey, exclusiveSize); - exclusiveReplicatedSizeMap.put(prevSnapKey, + exclusiveSizes.put(prevSnapKey, exclusiveSize); + exclusiveReplicatedSizes.put(prevSnapKey, exclusiveReplicatedSize); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableRenameEntryFilter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableRenameEntryFilter.java index 60371d70a9a8..48b08ed5bf42 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableRenameEntryFilter.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableRenameEntryFilter.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package org.apache.hadoop.ozone.om.snapshot.filter; import org.apache.hadoop.hdds.utils.db.Table; @@ -10,16 +28,18 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; -import org.apache.hadoop.ozone.om.service.AbstractKeyDeletingService; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import java.io.IOException; +/** + * Filter to return rename table entries which are reclaimable based on the key presence in previous snapshot's + * keyTable/DirectoryTable in the snapshot chain. + */ public class ReclaimableRenameEntryFilter extends ReclaimableFilter { /** - * Filter to return rename table entries which are reclaimable based on the key presence in previous snapshot's - * keyTable/DirectoryTable in the snapshot chain. + * * * @param omSnapshotManager * @param snapshotChainManager diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/package-info.java new file mode 100644 index 000000000000..700f7b9c6d05 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +/** + * Package containing filter to perform reclaimable check on snapshots. + */ +package org.apache.hadoop.ozone.om.snapshot.filter;