From dcf1bd966cad270726cef12739f1d9f42698892d Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 14 May 2025 07:16:31 -0400 Subject: [PATCH 1/2] HDDS-13030. Snapshot Purge should unset deep cleaning flag for next 2 snapshots in the chain Change-Id: I9cc49ea98bbce752e4987c56a15c2bccdf731a7b --- .../ozone/om/request/snapshot/OMSnapshotPurgeRequest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index dcbf3ad65bd5..025bd4546596 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -101,9 +101,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut continue; } SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); - - // Step 1: Update the deep clean flag for the next active snapshot + SnapshotInfo nextToNextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, + nextSnapshot); + // Step 1: Update the deep clean flag for the next snapshot updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, trxnLogIndex); + updateSnapshotInfoAndCache(nextToNextSnapshot, omMetadataManager, trxnLogIndex); // Step 2: Update the snapshot chain. updateSnapshotChainAndCache(omMetadataManager, fromSnapshot, trxnLogIndex); // Step 3: Purge the snapshot from SnapshotInfoTable cache and also remove from the map. @@ -140,6 +142,7 @@ private void updateSnapshotInfoAndCache(SnapshotInfo snapInfo, OmMetadataManager // current snapshot is deleted. We can potentially // reclaim more keys in the next snapshot. snapInfo.setDeepClean(false); + snapInfo.setDeepCleanedDeletedDir(false); // Update table cache first omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapInfo.getTableKey()), From 5de662fcffb71d6ec50c2d0dc5da312ae02d1865 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sat, 17 May 2025 11:23:26 -0400 Subject: [PATCH 2/2] HDDS-13030. Add tests Change-Id: Id4fc522b350635c516069d9e74d746c4c259c76f --- .../snapshot/OMSnapshotPurgeRequest.java | 4 +-- ...TestOMSnapshotPurgeRequestAndResponse.java | 30 +++++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index 025bd4546596..5f08c52c1cdc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -101,8 +101,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut continue; } SnapshotInfo nextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, fromSnapshot); - SnapshotInfo nextToNextSnapshot = SnapshotUtils.getNextSnapshot(ozoneManager, snapshotChainManager, - nextSnapshot); + SnapshotInfo nextToNextSnapshot = nextSnapshot == null ? null : SnapshotUtils.getNextSnapshot(ozoneManager, + snapshotChainManager, nextSnapshot); // Step 1: Update the deep clean flag for the next snapshot updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, trxnLogIndex); updateSnapshotInfoAndCache(nextToNextSnapshot, omMetadataManager, trxnLogIndex); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java index 108047f22b30..db870e8480e5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java @@ -394,7 +394,11 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( SnapshotInfo.getTableKey(getVolumeName(), bucket, snapshotName); SnapshotInfo snapshotInfo = getOmMetadataManager().getSnapshotInfoTable().get(snapshotTableKey); - snapshotInfoList.add(snapshotInfo); + snapshotInfo.setDeepClean(true); + snapshotInfo.setDeepCleanedDeletedDir(true); + getOmMetadataManager().getSnapshotInfoTable().addCacheEntry(snapshotTableKey, snapshotInfo, + System.currentTimeMillis()); + snapshotInfoList.add(getOmMetadataManager().getSnapshotInfoTable().get(snapshotTableKey)); } } @@ -403,13 +407,16 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( assertEquals(totalKeys, numberOfSnapshotBeforePurge); assertEquals(totalKeys, chainManager.getGlobalSnapshotChain().size()); Map expectedTransactionInfos = new HashMap<>(); + Map expectedDeepCleanFlags = new HashMap<>(); // Ratis transaction uses term index 1 while creating snapshot. ByteString expectedLastTransactionVal = TransactionInfo.valueOf(TransactionInfo.getTermIndex(1L)) .toByteString(); for (SnapshotInfo snapshotInfo : snapshotInfoList) { expectedTransactionInfos.put(snapshotInfo.getSnapshotId(), expectedLastTransactionVal); + expectedDeepCleanFlags.put(snapshotInfo.getSnapshotId(), true); } - validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(snapshotInfoList, expectedTransactionInfos); + validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(snapshotInfoList, expectedTransactionInfos, + expectedDeepCleanFlags); // Ratis transaction uses term index 200 while purging snapshot. expectedLastTransactionVal = TransactionInfo.valueOf(TransactionInfo.getTermIndex(200L)) .toByteString(); @@ -422,8 +429,15 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( expectedTransactionInfos.put(chainManager.nextGlobalSnapshot(snapId), expectedLastTransactionVal); } if (chainManager.hasNextPathSnapshot(purgeSnapshotInfo.getSnapshotPath(), snapId)) { - expectedTransactionInfos.put(chainManager.nextPathSnapshot(purgeSnapshotInfo.getSnapshotPath(), snapId), - expectedLastTransactionVal); + UUID nextPathSnapshot = chainManager.nextPathSnapshot(purgeSnapshotInfo.getSnapshotPath(), snapId); + expectedTransactionInfos.put(nextPathSnapshot, expectedLastTransactionVal); + expectedDeepCleanFlags.put(nextPathSnapshot, false); + if (chainManager.hasNextPathSnapshot(purgeSnapshotInfo.getSnapshotPath(), nextPathSnapshot)) { + UUID nextToNextPathSnapshot = chainManager.nextPathSnapshot(purgeSnapshotInfo.getSnapshotPath(), + nextPathSnapshot); + expectedTransactionInfos.put(nextToNextPathSnapshot, expectedLastTransactionVal); + expectedDeepCleanFlags.put(nextToNextPathSnapshot, false); + } } String purgeSnapshotKey = SnapshotInfo.getTableKey(getVolumeName(), purgeSnapshotInfo.getBucketName(), @@ -453,16 +467,20 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( actualNumberOfSnapshotAfterPurge); assertEquals(expectNumberOfSnapshotAfterPurge, chainManager .getGlobalSnapshotChain().size()); - validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(snapshotInfoListAfterPurge, expectedTransactionInfos); + validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(snapshotInfoListAfterPurge, expectedTransactionInfos, + expectedDeepCleanFlags); } private void validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain( - List snapshotInfoList, Map expectedTransactionInfos) throws IOException { + List snapshotInfoList, Map expectedTransactionInfos, + Map expectedDeepCleanFlags) throws IOException { if (snapshotInfoList.isEmpty()) { return; } for (SnapshotInfo snapshotInfo : snapshotInfoList) { assertEquals(snapshotInfo.getLastTransactionInfo(), expectedTransactionInfos.get(snapshotInfo.getSnapshotId())); + assertEquals(snapshotInfo.getDeepClean(), expectedDeepCleanFlags.get(snapshotInfo.getSnapshotId())); + assertEquals(snapshotInfo.getDeepCleanedDeletedDir(), expectedDeepCleanFlags.get(snapshotInfo.getSnapshotId())); } OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) getOmMetadataManager();