From 62a2c1c13cd434a45501757af776f5af583b8152 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 09:39:42 -0800 Subject: [PATCH 01/20] HDDS-11705. Snapshot diff fails on linked buckets Change-Id: I821e19b67e405d9bdffc49ad5fe8f483f224a6dd --- .../hadoop/ozone/om/snapshot/SnapshotDiffManager.java | 8 +++++++- .../apache/hadoop/ozone/om/snapshot/SnapshotUtils.java | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 6393f12066c4..50503ca3ac08 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -40,6 +40,7 @@ import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; @@ -1567,7 +1568,12 @@ protected BucketLayout getBucketLayout(final String volume, final OMMetadataManager mManager) throws IOException { final String bucketTableKey = mManager.getBucketKey(volume, bucket); - return mManager.getBucketTable().get(bucketTableKey).getBucketLayout(); + OmBucketInfo bucketInfo = mManager.getBucketTable().get(bucketTableKey); + if (bucketInfo.isLink()) { + return mManager.getBucketTable().get(mManager.getBucketKey(bucketInfo.getSourceVolume(), + bucketInfo.getSourceBucket())).getBucketLayout(); + } + return bucketInfo.getBucketLayout(); } private boolean areKeysEqual(WithObjectID oldKey, WithObjectID newKey) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index 201a9fe0c9c9..f1a966b0bbdc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -225,6 +226,12 @@ public static Map getColumnFamilyToKeyPrefixMap( String volumeName, String bucketName ) throws IOException { + final String bucketTableKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(bucketTableKey); + if (bucketInfo.isLink()) { + volumeName = bucketInfo.getSourceVolume(); + bucketName = bucketInfo.getSourceBucket(); + } String keyPrefix = getOzonePathKey(volumeName, bucketName); String keyPrefixFso = getOzonePathKeyForFso(omMetadataManager, volumeName, bucketName); From 2a3c55aaa4804e0c033d1a35b90630e4f2f1471b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 16:01:39 -0800 Subject: [PATCH 02/20] HDDS-11705. Snapshot operations should on linked buckets should work on actual bucket Change-Id: I23b167deca04f5d6d8156c011300f005351892fc --- .../hadoop/ozone/om/OmSnapshotManager.java | 8 +++++++- .../apache/hadoop/ozone/om/OzoneManager.java | 19 ++++++++++++++++++- .../snapshot/OMSnapshotCreateRequest.java | 14 +++++++++++--- .../snapshot/OMSnapshotDeleteRequest.java | 7 +++++++ .../snapshot/OMSnapshotRenameRequest.java | 8 ++++++++ 5 files changed, 51 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index dde5b22e7937..4ca7da27f3ae 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -38,6 +38,7 @@ import java.util.UUID; import com.google.common.cache.RemovalListener; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; @@ -656,7 +657,12 @@ private ReferenceCounted getSnapshot( // don't allow snapshot indicator without snapshot name throw new OMException(INVALID_KEY_NAME); } - + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket resolvedBucket = ozoneManager.resolveBucketLink(Pair.of(volumeName, + bucketName), false); + volumeName = resolvedBucket.realVolume(); + bucketName = resolvedBucket.realBucket(); String snapshotTableKey = SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 2bb8c915b3c2..ace65e29b638 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2968,6 +2968,11 @@ public ListKeysLightResult listKeysLight(String volumeName, public SnapshotInfo getSnapshotInfo(String volumeName, String bucketName, String snapshotName) throws IOException { metrics.incNumSnapshotInfos(); + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); + volumeName = resolvedBucket.realVolume(); + bucketName = resolvedBucket.realBucket(); Map auditMap = buildAuditMap(volumeName); auditMap.put(OzoneConsts.BUCKET, bucketName); try { @@ -2994,6 +2999,11 @@ public ListSnapshotResponse listSnapshot( String volumeName, String bucketName, String snapshotPrefix, String prevSnapshot, int maxListResult) throws IOException { metrics.incNumSnapshotLists(); + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); + volumeName = resolvedBucket.realVolume(); + bucketName = resolvedBucket.realBucket(); Map auditMap = buildAuditMap(volumeName); auditMap.put(OzoneConsts.BUCKET, bucketName); try { @@ -4898,6 +4908,11 @@ public SnapshotDiffResponse snapshotDiff(String volume, boolean forceFullDiff, boolean disableNativeDiff) throws IOException { + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volume, bucket), false); + volume = resolvedBucket.realVolume(); + bucket = resolvedBucket.realBucket(); if (isAclEnabled) { omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.READ, volume, bucket, null); @@ -4925,7 +4940,9 @@ public List listSnapshotDiffJobs(String volume, String jobStatus, boolean listAll) throws IOException { - + ResolvedBucket resolvedBucket = this.resolveBucketLink(Pair.of(volume, bucket), false); + volume = resolvedBucket.realBucket(); + bucket = resolvedBucket.realBucket(); if (isAclEnabled) { omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST, volume, bucket, null); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 2ded4f6a83e9..1598f228d36b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -18,10 +18,12 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -75,8 +77,8 @@ public class OMSnapshotCreateRequest extends OMClientRequest { LoggerFactory.getLogger(OMSnapshotCreateRequest.class); private final String snapshotPath; - private final String volumeName; - private final String bucketName; + private String volumeName; + private String bucketName; private final String snapshotName; private final SnapshotInfo snapshotInfo; @@ -106,7 +108,11 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { final OMRequest omRequest = super.preExecute(ozoneManager); // Verify name OmUtils.validateSnapshotName(snapshotName); - + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket bucket = ozoneManager.resolveBucketLink(Pair.of(volumeName, bucketName), this); + this.volumeName = bucket.realVolume(); + this.bucketName = bucket.realBucket(); UserGroupInformation ugi = createUGIForApi(); String bucketOwner = ozoneManager.getBucketOwner(volumeName, bucketName, IAccessAuthorizer.ACLType.READ, OzoneObj.ResourceType.BUCKET); @@ -120,6 +126,8 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { return omRequest.toBuilder().setCreateSnapshotRequest( omRequest.getCreateSnapshotRequest().toBuilder() .setSnapshotId(toProtobuf(UUID.randomUUID())) + .setVolumeName(volumeName) + .setBucketName(this.bucketName) .setCreationTime(Time.now()) .build()).build(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java index a2b00138cf32..95f99c627c46 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; @@ -82,6 +84,11 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { String volumeName = deleteSnapshotRequest.getVolumeName(); String bucketName = deleteSnapshotRequest.getBucketName(); + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket resolvedBucket = ozoneManager.resolveBucketLink(Pair.of(volumeName, bucketName), this); + volumeName = resolvedBucket.realVolume(); + bucketName = resolvedBucket.realBucket(); // Permission check UserGroupInformation ugi = createUGIForApi(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotRenameRequest.java index 8341f875504a..8cf0579647c7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotRenameRequest.java @@ -25,6 +25,8 @@ import java.io.IOException; import java.nio.file.InvalidPathException; + +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OmUtils; @@ -32,6 +34,7 @@ import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMClientRequest; @@ -75,6 +78,11 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { String volumeName = renameSnapshotRequest.getVolumeName(); String bucketName = renameSnapshotRequest.getBucketName(); + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket resolvedBucket = ozoneManager.resolveBucketLink(Pair.of(volumeName, bucketName), this); + volumeName = resolvedBucket.realVolume(); + bucketName = resolvedBucket.realBucket(); // Permission check UserGroupInformation ugi = createUGIForApi(); From 77f69a2377015ca70d3219d403f224ad1d947990 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 16:15:30 -0800 Subject: [PATCH 03/20] HDDS-11705. Revert fixes made on snapshot flows Change-Id: I2138306c1a3077ed4b77aeb1d2366c7acb8d2d66 --- .../hadoop/ozone/om/snapshot/SnapshotDiffManager.java | 8 +------- .../apache/hadoop/ozone/om/snapshot/SnapshotUtils.java | 7 ------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 50503ca3ac08..6393f12066c4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -40,7 +40,6 @@ import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; @@ -1568,12 +1567,7 @@ protected BucketLayout getBucketLayout(final String volume, final OMMetadataManager mManager) throws IOException { final String bucketTableKey = mManager.getBucketKey(volume, bucket); - OmBucketInfo bucketInfo = mManager.getBucketTable().get(bucketTableKey); - if (bucketInfo.isLink()) { - return mManager.getBucketTable().get(mManager.getBucketKey(bucketInfo.getSourceVolume(), - bucketInfo.getSourceBucket())).getBucketLayout(); - } - return bucketInfo.getBucketLayout(); + return mManager.getBucketTable().get(bucketTableKey).getBucketLayout(); } private boolean areKeysEqual(WithObjectID oldKey, WithObjectID newKey) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index f1a966b0bbdc..201a9fe0c9c9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -24,7 +24,6 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -226,12 +225,6 @@ public static Map getColumnFamilyToKeyPrefixMap( String volumeName, String bucketName ) throws IOException { - final String bucketTableKey = omMetadataManager.getBucketKey(volumeName, bucketName); - OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(bucketTableKey); - if (bucketInfo.isLink()) { - volumeName = bucketInfo.getSourceVolume(); - bucketName = bucketInfo.getSourceBucket(); - } String keyPrefix = getOzonePathKey(volumeName, bucketName); String keyPrefixFso = getOzonePathKeyForFso(omMetadataManager, volumeName, bucketName); From df52e73bdcf5c256dcb5dbe8b35002fc77c954f5 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 16:24:37 -0800 Subject: [PATCH 04/20] HDDS-11705. Address review comments Change-Id: I12e98b5a4f5896f1871f8b019fd6929f37e39f4a --- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index ace65e29b638..a5192707e6bf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -4914,10 +4914,6 @@ public SnapshotDiffResponse snapshotDiff(String volume, volume = resolvedBucket.realVolume(); bucket = resolvedBucket.realBucket(); - if (isAclEnabled) { - omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.READ, volume, bucket, null); - } - return omSnapshotManager.getSnapshotDiffReport(volume, bucket, fromSnapshot, toSnapshot, token, pageSize, forceFullDiff, disableNativeDiff); } @@ -4927,10 +4923,9 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volume, String fromSnapshot, String toSnapshot) throws IOException { - - if (isAclEnabled) { - omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.READ, volume, bucket, null); - } + ResolvedBucket resolvedBucket = this.resolveBucketLink(Pair.of(volume, bucket), false); + volume = resolvedBucket.realBucket(); + bucket = resolvedBucket.realBucket(); return omSnapshotManager.cancelSnapshotDiff(volume, bucket, fromSnapshot, toSnapshot); } From 3115158830483c527fb978e1edc15473d81191d2 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 18:34:21 -0800 Subject: [PATCH 05/20] HDDS-11705. Add test cases Change-Id: Ie9fec98f741c875362e904ad48b00bf9c1f286b7 --- .../org/apache/hadoop/ozone/TestDataUtil.java | 43 ++++++- .../ozone/om/snapshot/TestOmSnapshot.java | 107 ++++++++++-------- .../om/snapshot/TestOmSnapshotFileSystem.java | 33 +++--- ...apshotFileSystemFsoWithLinkedBuckets.java} | 6 +- ...shotFileSystemFsoWithoutLinkedBuckets.java | 31 +++++ ...hotFileSystemLegacyWithLinkedBuckets.java} | 6 +- ...tFileSystemLegacyWithoutLinkedBuckets.java | 31 +++++ ...hotFsoWithNativeLibWithLinkedBuckets.java} | 6 +- ...tFsoWithNativeLibWithoutLinkedBuckets.java | 36 ++++++ ...FsoWithoutNativeLibWithLinkedBuckets.java} | 6 +- ...oWithoutNativeLibWithoutLinkedBuckets.java | 34 ++++++ ...SnapshotObjectStoreWithLinkedBuckets.java} | 6 +- ...apshotObjectStoreWithoutLinkedBuckets.java | 34 ++++++ ...estOmSnapshotWithBucketLinkingLegacy.java} | 6 +- ...tOmSnapshotWithoutBucketLinkingLegacy.java | 34 ++++++ 15 files changed, 335 insertions(+), 84 deletions(-) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotFileSystemFso.java => TestOmSnapshotFileSystemFsoWithLinkedBuckets.java} (83%) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithoutLinkedBuckets.java rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotFileSystemLegacy.java => TestOmSnapshotFileSystemLegacyWithLinkedBuckets.java} (82%) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets.java rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotFsoWithNativeLib.java => TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java} (85%) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets.java rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotFsoWithoutNativeLib.java => TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets.java} (82%) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets.java rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotObjectStore.java => TestOmSnapshotObjectStoreWithLinkedBuckets.java} (83%) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithoutLinkedBuckets.java rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotLegacy.java => TestOmSnapshotWithBucketLinkingLegacy.java} (84%) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithoutBucketLinkingLegacy.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index 4488e467c29a..01883c135482 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -164,21 +164,56 @@ private static BucketLayout getDefaultBucketLayout(OzoneClient client) { public static OzoneBucket createBucket(OzoneClient client, String vol, BucketArgs bucketArgs, String bukName) throws IOException { + return createBucket(client, vol, bucketArgs, bukName, false); + } + + public static OzoneBucket createBucket(OzoneClient client, + String vol, BucketArgs bucketArgs, String bukName, + boolean createLinkedBucket) + throws IOException { ObjectStore objectStore = client.getObjectStore(); OzoneVolume volume = objectStore.getVolume(vol); - volume.createBucket(bukName, bucketArgs); - return volume.getBucket(bukName); + String sourceBucket = bukName; + if (createLinkedBucket) { + sourceBucket = bukName + RandomStringUtils.randomNumeric(5); + } + volume.createBucket(sourceBucket, bucketArgs); + OzoneBucket ozoneBucket = volume.getBucket(sourceBucket); + if (createLinkedBucket) { + ozoneBucket = createLinkedBucket(client, vol, sourceBucket, bukName); + } + return ozoneBucket; + } + + public static OzoneBucket createLinkedBucket(OzoneClient client, String vol, String sourceBucketName, + String linkedBucketName) throws IOException { + BucketArgs.Builder bb = new BucketArgs.Builder() + .setStorageType(StorageType.DEFAULT) + .setVersioning(false) + .setSourceVolume(vol) + .setSourceBucket(sourceBucketName); + return createBucket(client, vol, bb.build(), linkedBucketName); + } + + public static OzoneBucket createVolumeAndBucket(OzoneClient client, + BucketLayout bucketLayout) + throws IOException { + return createVolumeAndBucket(client, bucketLayout, false); } public static OzoneBucket createVolumeAndBucket(OzoneClient client, - BucketLayout bucketLayout) throws IOException { + BucketLayout bucketLayout, boolean createLinkedBucket) throws IOException { final int attempts = 5; for (int i = 0; i < attempts; i++) { try { String volumeName = "volume" + RandomStringUtils.randomNumeric(5); String bucketName = "bucket" + RandomStringUtils.randomNumeric(5); - return createVolumeAndBucket(client, volumeName, bucketName, + OzoneBucket ozoneBucket = createVolumeAndBucket(client, volumeName, bucketName, bucketLayout); + if (createLinkedBucket) { + String targetBucketName = ozoneBucket.getName() + RandomStringUtils.randomNumeric(5); + createLinkedBucket(client, volumeName, bucketName, targetBucketName); + } } catch (OMException e) { if (e.getResult() != OMException.ResultCodes.VOLUME_ALREADY_EXISTS && e.getResult() != OMException.ResultCodes.BUCKET_ALREADY_EXISTS) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 9a6bca29b889..f641850d494f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -182,17 +182,20 @@ public abstract class TestOmSnapshot { private final boolean forceFullSnapshotDiff; private final boolean disableNativeDiff; private final AtomicInteger counter; + private final boolean createLinkedBucket; public TestOmSnapshot(BucketLayout newBucketLayout, boolean newEnableFileSystemPaths, boolean forceFullSnapDiff, - boolean disableNativeDiff) + boolean disableNativeDiff, + boolean createLinkedBucket) throws Exception { this.enabledFileSystemPaths = newEnableFileSystemPaths; this.bucketLayout = newBucketLayout; this.forceFullSnapshotDiff = forceFullSnapDiff; this.disableNativeDiff = disableNativeDiff; this.counter = new AtomicInteger(); + this.createLinkedBucket = createLinkedBucket; init(); } @@ -218,7 +221,7 @@ private void init() throws Exception { cluster.waitForClusterToBeReady(); client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem - ozoneBucket = TestDataUtil.createVolumeAndBucket(client, bucketLayout); + ozoneBucket = TestDataUtil.createVolumeAndBucket(client, bucketLayout, createLinkedBucket); volumeName = ozoneBucket.getVolumeName(); bucketName = ozoneBucket.getName(); ozoneManager = cluster.getOzoneManager(); @@ -232,6 +235,16 @@ private void init() throws Exception { finalizeOMUpgrade(); } + private void createBucket(OzoneVolume volume, String bucketVal) throws IOException { + if (createLinkedBucket) { + String sourceBucketName = bucketVal + counter.incrementAndGet(); + volume.createBucket(sourceBucketName); + TestDataUtil.createLinkedBucket(client, volume.getName(), sourceBucketName, bucketVal); + } else { + volume.createBucket(bucketVal); + } + } + private void stopKeyManager() throws IOException { KeyManagerImpl keyManager = (KeyManagerImpl) HddsWhiteboxTestUtils .getInternalState(ozoneManager, "keyManager"); @@ -319,10 +332,10 @@ public void testListKey() throws Exception { store.createVolume(volumeB); OzoneVolume volA = store.getVolume(volumeA); OzoneVolume volB = store.getVolume(volumeB); - volA.createBucket(bucketA); - volA.createBucket(bucketB); - volB.createBucket(bucketA); - volB.createBucket(bucketB); + createBucket(volA, bucketA); + createBucket(volA, bucketB); + createBucket(volB, bucketA); + createBucket(volB, bucketB); OzoneBucket volAbucketA = volA.getBucket(bucketA); OzoneBucket volAbucketB = volA.getBucket(bucketB); OzoneBucket volBbucketA = volB.getBucket(bucketA); @@ -401,7 +414,7 @@ public void testListKeyOnEmptyBucket() String bucket = "buc-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume vol = store.getVolume(volume); - vol.createBucket(bucket); + createBucket(vol, bucket); String snapshotKeyPrefix = createSnapshot(volume, bucket); OzoneBucket buc = vol.getBucket(bucket); Iterator keys = buc.listKeys(snapshotKeyPrefix); @@ -478,7 +491,7 @@ public void testListDeleteKey() throws Exception { String bucket = "buc-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume vol = store.getVolume(volume); - vol.createBucket(bucket); + createBucket(vol, bucket); OzoneBucket volBucket = vol.getBucket(bucket); String key = "key-"; @@ -503,7 +516,7 @@ public void testListAddNewKey() throws Exception { String bucket = "buc-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume vol = store.getVolume(volume); - vol.createBucket(bucket); + createBucket(vol, bucket); OzoneBucket bucket1 = vol.getBucket(bucket); String key1 = "key-1-"; @@ -553,7 +566,7 @@ public void testCreateSnapshotMissingMandatoryParams() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); // Create Key1 and take snapshot String key1 = "key-1-"; @@ -621,7 +634,7 @@ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String key1 = "k1"; key1 = createFileKeyWithPrefix(bucket, key1); @@ -659,7 +672,7 @@ public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String key1 = "k1"; key1 = createFileKeyWithPrefix(bucket, key1); @@ -706,7 +719,7 @@ public void testSnapDiffReclaimWithKeyRecreation() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String key1 = "k1"; key1 = createFileKeyWithPrefix(bucket, key1); @@ -760,7 +773,7 @@ public void testSnapDiffReclaimWithKeyRename() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String key1 = "k1"; key1 = createFileKeyWithPrefix(bucket, key1); @@ -805,7 +818,7 @@ public void testSnapDiffWith2RenamesAndDelete() throws Exception { String testBucketName = "bucket" + counter.incrementAndGet(); store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String key1 = "k1"; key1 = createFileKeyWithPrefix(bucket, key1); @@ -864,7 +877,7 @@ public void testSnapDiffWithKeyRenamesRecreationAndDelete() String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String key1 = "k1"; key1 = createFileKeyWithPrefix(bucket, key1); @@ -908,7 +921,7 @@ public void testSnapDiffReclaimWithDeferredKeyDeletion() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String snap1 = "snap1"; createSnapshot(testVolumeName, testBucketName, snap1); @@ -942,7 +955,7 @@ public void testSnapDiffWithNoEffectiveRename() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String snap1 = "snap1"; String key1 = "k1"; @@ -984,7 +997,7 @@ public void testSnapDiffWithDirectory() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String snap1 = "snap1"; String key1 = "k1"; @@ -1042,7 +1055,7 @@ public void testSnapDiffWithDirectoryDelete() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String snap1 = "snap1"; String key1 = "k1"; @@ -1078,7 +1091,7 @@ public void testSnapdiffWithObjectMetaModification() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String snap1 = "snap1"; String key1 = "k1"; @@ -1110,7 +1123,7 @@ public void testSnapdiffWithFilesystemCreate() String testBucketName = "bucket" + counter.incrementAndGet(); store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); String rootPath = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, testBucketName, testVolumeName); try (FileSystem fs = FileSystem.get(new URI(rootPath), cluster.getConf())) { @@ -1153,7 +1166,7 @@ public void testSnapDiffWithFilesystemDirectoryRenameOperation() String testBucketName = "bucket" + counter.incrementAndGet(); store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); String rootPath = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, testBucketName, testVolumeName); try (FileSystem fs = FileSystem.get(new URI(rootPath), cluster.getConf())) { @@ -1196,7 +1209,7 @@ public void testSnapDiffWithFilesystemDirectoryMoveOperation() String testBucketName = "bucket" + counter.incrementAndGet(); store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); String rootPath = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, testBucketName, testVolumeName); try (FileSystem fs = FileSystem.get(new URI(rootPath), cluster.getConf())) { @@ -1239,8 +1252,8 @@ public void testBucketDeleteIfSnapshotExists() throws Exception { String bucket2 = "buc-" + counter.incrementAndGet(); store.createVolume(volume1); OzoneVolume volume = store.getVolume(volume1); - volume.createBucket(bucket1); - volume.createBucket(bucket2); + createBucket(volume, bucket1); + createBucket(volume, bucket2); OzoneBucket bucketWithSnapshot = volume.getBucket(bucket1); OzoneBucket bucketWithoutSnapshot = volume.getBucket(bucket2); String key = "key-"; @@ -1263,7 +1276,7 @@ public void testGetSnapshotInfo() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); createFileKey(bucket1, "key-1"); @@ -1313,7 +1326,7 @@ public void testSnapDiffWithDirRename() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); bucket1.createDirectory("dir1"); String snap1 = "snap1"; @@ -1335,7 +1348,7 @@ public void testSnapDiff() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); // Create Key1 and take snapshot String key1 = "key-1-"; @@ -1567,7 +1580,7 @@ public void testSnapDiffNoSnapshot() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); // Create Key1 and take snapshot String key1 = "key-1-"; @@ -1599,7 +1612,7 @@ public void testSnapDiffNonExistentUrl() throws Exception { String bucketb = "buck-" + counter.incrementAndGet(); store.createVolume(volumea); OzoneVolume volume1 = store.getVolume(volumea); - volume1.createBucket(bucketa); + createBucket(volume1, bucketa); OzoneBucket bucket1 = volume1.getBucket(bucketa); // Create Key1 and take 2 snapshots String key1 = "key-1-"; @@ -1638,7 +1651,7 @@ public void testSnapDiffWithKeyOverwrite() throws Exception { String testBucketName = "bucket1"; store.createVolume(testVolumeName); OzoneVolume volume = store.getVolume(testVolumeName); - volume.createBucket(testBucketName); + createBucket(volume, testBucketName); OzoneBucket bucket = volume.getBucket(testBucketName); String key1 = "k1"; key1 = createFileKeyWithPrefix(bucket, key1); @@ -1663,7 +1676,7 @@ public void testSnapDiffMissingMandatoryParams() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); // Create Key1 and take snapshot String key1 = "key-1-"; @@ -1700,8 +1713,8 @@ public void testSnapDiffMultipleBuckets() throws Exception { String bucketName2 = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucketName1); - volume1.createBucket(bucketName2); + createBucket(volume1, bucketName1); + createBucket(volume1, bucketName2); OzoneBucket bucket1 = volume1.getBucket(bucketName1); OzoneBucket bucket2 = volume1.getBucket(bucketName2); // Create Key1 and take snapshot @@ -1738,7 +1751,7 @@ public void testListSnapshotDiffWithInvalidParameters() // Create the volume and the bucket. store.createVolume(volume); OzoneVolume ozVolume = store.getVolume(volume); - ozVolume.createBucket(bucket); + createBucket(ozVolume, bucket); assertDoesNotThrow(() -> store.listSnapshotDiffJobs(volume, bucket, "", true)); @@ -1785,8 +1798,8 @@ public void testSnapDiffWithMultipleSSTs() throws Exception { String bucketName2 = "buck2"; store.createVolume(volumeName1); OzoneVolume volume1 = store.getVolume(volumeName1); - volume1.createBucket(bucketName1); - volume1.createBucket(bucketName2); + createBucket(volume1, bucketName1); + createBucket(volume1, bucketName2); OzoneBucket bucket1 = volume1.getBucket(bucketName1); OzoneBucket bucket2 = volume1.getBucket(bucketName2); String keyPrefix = "key-"; @@ -1822,7 +1835,7 @@ public void testDeleteSnapshotTwice() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); // Create Key1 and take snapshot String key1 = "key-1-"; @@ -1843,7 +1856,7 @@ public void testDeleteSnapshotFailure() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); // Create Key1 and take snapshot String key1 = "key-1-"; @@ -1870,7 +1883,7 @@ public void testDeleteSnapshotMissingMandatoryParams() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); // Create Key1 and take snapshot String key1 = "key-1-"; @@ -1895,7 +1908,7 @@ public void testSnapshotQuotaHandling() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); bucket1.setQuota(OzoneQuota.parseQuota("102400000", "500")); volume1.setQuota(OzoneQuota.parseQuota("204800000", "1000")); @@ -2182,7 +2195,7 @@ public void testDayWeekMonthSnapshotCreationAndExpiration() throws Exception { String bucketA = "buc-a-" + RandomStringUtils.randomNumeric(5); store.createVolume(volumeA); OzoneVolume volA = store.getVolume(volumeA); - volA.createBucket(bucketA); + createBucket(volA, bucketA); OzoneBucket volAbucketA = volA.getBucket(bucketA); int latestDayIndex = 0; @@ -2391,7 +2404,7 @@ public void testSnapshotCompactionDag() throws Exception { store.createVolume(volume1); OzoneVolume ozoneVolume = store.getVolume(volume1); - ozoneVolume.createBucket(bucket1); + createBucket(ozoneVolume, bucket1); OzoneBucket ozoneBucket1 = ozoneVolume.getBucket(bucket1); DBStore activeDbStore = ozoneManager.getMetadataManager().getStore(); @@ -2404,7 +2417,7 @@ public void testSnapshotCompactionDag() throws Exception { createSnapshot(volume1, bucket1, "bucket1-snap1"); activeDbStore.compactDB(); - ozoneVolume.createBucket(bucket2); + createBucket(ozoneVolume, bucket2); OzoneBucket ozoneBucket2 = ozoneVolume.getBucket(bucket2); for (int i = 100; i < 200; i++) { @@ -2417,7 +2430,7 @@ public void testSnapshotCompactionDag() throws Exception { createSnapshot(volume1, bucket2, "bucket2-snap1"); activeDbStore.compactDB(); - ozoneVolume.createBucket(bucket3); + createBucket(ozoneVolume, bucket3); OzoneBucket ozoneBucket3 = ozoneVolume.getBucket(bucket3); for (int i = 200; i < 300; i++) { @@ -2496,7 +2509,7 @@ public void testSnapshotReuseSnapName() throws Exception { String bucket = "buck-" + counter.incrementAndGet(); store.createVolume(volume); OzoneVolume volume1 = store.getVolume(volume); - volume1.createBucket(bucket); + createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); // Create Key1 and take snapshot String key1 = "key-1-"; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java index 0849b9007810..4cf8ba9db957 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java @@ -54,9 +54,9 @@ import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,6 +99,7 @@ * Abstract class for OmSnapshot file system tests. */ @Timeout(120) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class TestOmSnapshotFileSystem { protected static final String VOLUME_NAME = "volume" + RandomStringUtils.randomNumeric(5); @@ -107,26 +108,28 @@ public abstract class TestOmSnapshotFileSystem { protected static final String BUCKET_NAME_LEGACY = "bucket-legacy-" + RandomStringUtils.randomNumeric(5); - private static MiniOzoneCluster cluster = null; - private static OzoneClient client; - private static ObjectStore objectStore; - private static OzoneConfiguration conf; - private static OzoneManagerProtocol writeClient; - private static OzoneManager ozoneManager; - private static String keyPrefix; + private MiniOzoneCluster cluster = null; + private OzoneClient client; + private ObjectStore objectStore; + private OzoneConfiguration conf; + private OzoneManagerProtocol writeClient; + private OzoneManager ozoneManager; + private String keyPrefix; private final String bucketName; + private final boolean createLinkedBuckets; private FileSystem fs; private OzoneFileSystem o3fs; private static final Logger LOG = LoggerFactory.getLogger(TestOmSnapshot.class); - public TestOmSnapshotFileSystem(String bucketName) { + public TestOmSnapshotFileSystem(String bucketName, boolean createLinkedBuckets) throws Exception { this.bucketName = bucketName; + this.createLinkedBuckets = createLinkedBuckets; + init(); } - @BeforeAll - public static void init() throws Exception { + private void init() throws Exception { conf = new OzoneConfiguration(); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true); cluster = MiniOzoneCluster.newBuilder(conf).build(); @@ -140,10 +143,10 @@ public static void init() throws Exception { TestDataUtil.createVolume(client, VOLUME_NAME); TestDataUtil.createBucket(client, VOLUME_NAME, new BucketArgs.Builder().setBucketLayout(FILE_SYSTEM_OPTIMIZED).build(), - BUCKET_NAME_FSO); + BUCKET_NAME_FSO, createLinkedBuckets); TestDataUtil.createBucket(client, VOLUME_NAME, new BucketArgs.Builder().setBucketLayout(LEGACY).build(), - BUCKET_NAME_LEGACY); + BUCKET_NAME_LEGACY, createLinkedBuckets); // stop the deletion services so that keys can still be read KeyManagerImpl keyManager = (KeyManagerImpl) ozoneManager.getKeyManager(); @@ -163,7 +166,7 @@ public void setupFsClient() throws IOException { } @AfterAll - public static void tearDown() throws Exception { + void tearDown() { IOUtils.closeQuietly(client); if (cluster != null) { cluster.shutdown(); @@ -376,7 +379,7 @@ private void createKey(OzoneBucket ozoneBucket, String key, int length, assertEquals(inputString, new String(read, StandardCharsets.UTF_8)); } - private static void setKeyPrefix(String s) { + private void setKeyPrefix(String s) { keyPrefix = s; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithLinkedBuckets.java similarity index 83% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithLinkedBuckets.java index 47bdd8f3bd52..e9d1017cddb4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithLinkedBuckets.java @@ -24,8 +24,8 @@ * OmSnapshot file system tests for FSO. */ @Timeout(120) -public class TestOmSnapshotFileSystemFso extends TestOmSnapshotFileSystem { - TestOmSnapshotFileSystemFso() { - super(BUCKET_NAME_FSO); +public class TestOmSnapshotFileSystemFsoWithLinkedBuckets extends TestOmSnapshotFileSystem { + TestOmSnapshotFileSystemFsoWithLinkedBuckets() throws Exception { + super(BUCKET_NAME_FSO, true); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithoutLinkedBuckets.java new file mode 100644 index 000000000000..c063ad548500 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithoutLinkedBuckets.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.junit.jupiter.api.Timeout; + +/** + * OmSnapshot file system tests for FSO. + */ +@Timeout(120) +public class TestOmSnapshotFileSystemFsoWithoutLinkedBuckets extends TestOmSnapshotFileSystem { + TestOmSnapshotFileSystemFsoWithoutLinkedBuckets() throws Exception { + super(BUCKET_NAME_FSO, false); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithLinkedBuckets.java similarity index 82% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacy.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithLinkedBuckets.java index b8d81c31cf5f..61f92cc7c0b9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacy.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithLinkedBuckets.java @@ -24,8 +24,8 @@ * OmSnapshot file system tests for Legacy. */ @Timeout(120) -public class TestOmSnapshotFileSystemLegacy extends TestOmSnapshotFileSystem { - TestOmSnapshotFileSystemLegacy() { - super(BUCKET_NAME_LEGACY); +public class TestOmSnapshotFileSystemLegacyWithLinkedBuckets extends TestOmSnapshotFileSystem { + TestOmSnapshotFileSystemLegacyWithLinkedBuckets() throws Exception { + super(BUCKET_NAME_LEGACY, true); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets.java new file mode 100644 index 000000000000..2dc2f5e1e145 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.junit.jupiter.api.Timeout; + +/** + * OmSnapshot file system tests for Legacy. + */ +@Timeout(120) +public class TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets extends TestOmSnapshotFileSystem { + TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets() throws Exception { + super(BUCKET_NAME_LEGACY, false); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java similarity index 85% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java index 06fbebb2efa2..c499a7056494 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java @@ -29,8 +29,8 @@ */ @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) @Timeout(300) -class TestOmSnapshotFsoWithNativeLib extends TestOmSnapshot { - TestOmSnapshotFsoWithNativeLib() throws Exception { - super(FILE_SYSTEM_OPTIMIZED, false, false, false); +class TestOmSnapshotFsoWithNativeLibWithLinkedBuckets extends TestOmSnapshot { + TestOmSnapshotFsoWithNativeLibWithLinkedBuckets() throws Exception { + super(FILE_SYSTEM_OPTIMIZED, false, false, false, true); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets.java new file mode 100644 index 000000000000..2fae92026a52 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.apache.ozone.test.tag.Native; +import org.junit.jupiter.api.Timeout; + +import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME; +import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; + +/** + * Test OmSnapshot for FSO bucket type when native lib is enabled. + */ +@Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) +@Timeout(300) +class TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets extends TestOmSnapshot { + TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets() throws Exception { + super(FILE_SYSTEM_OPTIMIZED, false, false, false, false); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets.java similarity index 82% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets.java index c1782b73d195..4387f77b3fc2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets.java @@ -26,9 +26,9 @@ * Test OmSnapshot for FSO bucket type when native lib is disabled. */ @Timeout(300) -public class TestOmSnapshotFsoWithoutNativeLib extends TestOmSnapshot { +public class TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets extends TestOmSnapshot { - public TestOmSnapshotFsoWithoutNativeLib() throws Exception { - super(FILE_SYSTEM_OPTIMIZED, false, false, true); + public TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets() throws Exception { + super(FILE_SYSTEM_OPTIMIZED, false, false, true, true); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets.java new file mode 100644 index 000000000000..772dc73ff2d5 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.junit.jupiter.api.Timeout; + +import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; + +/** + * Test OmSnapshot for FSO bucket type when native lib is disabled. + */ +@Timeout(300) +public class TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets extends TestOmSnapshot { + + public TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets() throws Exception { + super(FILE_SYSTEM_OPTIMIZED, false, false, true, false); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithLinkedBuckets.java similarity index 83% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithLinkedBuckets.java index 13c8cb5fca32..64765e717187 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithLinkedBuckets.java @@ -26,9 +26,9 @@ * Test OmSnapshot for Object Store bucket type. */ @Timeout(300) -public class TestOmSnapshotObjectStore extends TestOmSnapshot { +public class TestOmSnapshotObjectStoreWithLinkedBuckets extends TestOmSnapshot { - public TestOmSnapshotObjectStore() throws Exception { - super(OBJECT_STORE, false, false, false); + public TestOmSnapshotObjectStoreWithLinkedBuckets() throws Exception { + super(OBJECT_STORE, false, false, false, true); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithoutLinkedBuckets.java new file mode 100644 index 000000000000..755e7e1412b8 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithoutLinkedBuckets.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.junit.jupiter.api.Timeout; + +import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE; + +/** + * Test OmSnapshot for Object Store bucket type. + */ +@Timeout(300) +public class TestOmSnapshotObjectStoreWithoutLinkedBuckets extends TestOmSnapshot { + + public TestOmSnapshotObjectStoreWithoutLinkedBuckets() throws Exception { + super(OBJECT_STORE, false, false, false, true); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLegacy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithBucketLinkingLegacy.java similarity index 84% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLegacy.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithBucketLinkingLegacy.java index bf4a2fee0def..f1ced6c4a801 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLegacy.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithBucketLinkingLegacy.java @@ -26,9 +26,9 @@ * Test OmSnapshot for Legacy bucket type. */ @Timeout(300) -public class TestOmSnapshotLegacy extends TestOmSnapshot { +public class TestOmSnapshotWithBucketLinkingLegacy extends TestOmSnapshot { - public TestOmSnapshotLegacy() throws Exception { - super(LEGACY, false, false, false); + public TestOmSnapshotWithBucketLinkingLegacy() throws Exception { + super(LEGACY, false, false, false, true); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithoutBucketLinkingLegacy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithoutBucketLinkingLegacy.java new file mode 100644 index 000000000000..95549471e61b --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithoutBucketLinkingLegacy.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.junit.jupiter.api.Timeout; + +import static org.apache.hadoop.ozone.om.helpers.BucketLayout.LEGACY; + +/** + * Test OmSnapshot for Legacy bucket type. + */ +@Timeout(300) +public class TestOmSnapshotWithoutBucketLinkingLegacy extends TestOmSnapshot { + + public TestOmSnapshotWithoutBucketLinkingLegacy() throws Exception { + super(LEGACY, false, false, false, false); + } +} From 516f0a3747bb68f530053c3b6f1fbc854aaca7e9 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 19:28:46 -0800 Subject: [PATCH 06/20] HDDS-11705. Address review comments Change-Id: I73ca6a308e1aa4f2fa6d4de224881803b078ef25 --- .../apache/hadoop/ozone/om/OzoneManager.java | 40 ++++++------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index a5192707e6bf..77ae7aefb0c6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2971,17 +2971,11 @@ public SnapshotInfo getSnapshotInfo(String volumeName, String bucketName, // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a // permission check, since linked bucket permissions and source bucket permissions could be different. ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); - volumeName = resolvedBucket.realVolume(); - bucketName = resolvedBucket.realBucket(); - Map auditMap = buildAuditMap(volumeName); - auditMap.put(OzoneConsts.BUCKET, bucketName); + Map auditMap = buildAuditMap(resolvedBucket.realVolume()); + auditMap.put(OzoneConsts.BUCKET, resolvedBucket.realBucket()); try { - if (isAclEnabled) { - omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, - ACLType.READ, volumeName, bucketName, null); - } SnapshotInfo snapshotInfo = - metadataManager.getSnapshotInfo(volumeName, bucketName, snapshotName); + metadataManager.getSnapshotInfo(resolvedBucket.realVolume(), resolvedBucket.realBucket(), snapshotName); AUDIT.logReadSuccess(buildAuditMessageForSuccess( OMAction.SNAPSHOT_INFO, auditMap)); @@ -3002,17 +2996,15 @@ public ListSnapshotResponse listSnapshot( // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a // permission check, since linked bucket permissions and source bucket permissions could be different. ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); - volumeName = resolvedBucket.realVolume(); - bucketName = resolvedBucket.realBucket(); - Map auditMap = buildAuditMap(volumeName); - auditMap.put(OzoneConsts.BUCKET, bucketName); + Map auditMap = buildAuditMap(resolvedBucket.realVolume()); + auditMap.put(OzoneConsts.BUCKET, resolvedBucket.realBucket()); try { if (isAclEnabled) { omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, - ACLType.LIST, volumeName, bucketName, null); + ACLType.LIST, resolvedBucket.realVolume(), resolvedBucket.realBucket(), null); } ListSnapshotResponse listSnapshotResponse = - metadataManager.listSnapshot(volumeName, bucketName, + metadataManager.listSnapshot(resolvedBucket.realVolume(), resolvedBucket.realBucket(), snapshotPrefix, prevSnapshot, maxListResult); AUDIT.logReadSuccess(buildAuditMessageForSuccess( @@ -4911,11 +4903,8 @@ public SnapshotDiffResponse snapshotDiff(String volume, // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a // permission check, since linked bucket permissions and source bucket permissions could be different. ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volume, bucket), false); - volume = resolvedBucket.realVolume(); - bucket = resolvedBucket.realBucket(); - - return omSnapshotManager.getSnapshotDiffReport(volume, bucket, fromSnapshot, toSnapshot, - token, pageSize, forceFullDiff, disableNativeDiff); + return omSnapshotManager.getSnapshotDiffReport(resolvedBucket.realVolume(), resolvedBucket.realBucket(), + fromSnapshot, toSnapshot, token, pageSize, forceFullDiff, disableNativeDiff); } public CancelSnapshotDiffResponse cancelSnapshotDiff(String volume, @@ -4924,10 +4913,8 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volume, String toSnapshot) throws IOException { ResolvedBucket resolvedBucket = this.resolveBucketLink(Pair.of(volume, bucket), false); - volume = resolvedBucket.realBucket(); - bucket = resolvedBucket.realBucket(); - - return omSnapshotManager.cancelSnapshotDiff(volume, bucket, fromSnapshot, toSnapshot); + return omSnapshotManager.cancelSnapshotDiff(resolvedBucket.realVolume(), resolvedBucket.realBucket(), + fromSnapshot, toSnapshot); } public List listSnapshotDiffJobs(String volume, @@ -4936,13 +4923,12 @@ public List listSnapshotDiffJobs(String volume, boolean listAll) throws IOException { ResolvedBucket resolvedBucket = this.resolveBucketLink(Pair.of(volume, bucket), false); - volume = resolvedBucket.realBucket(); - bucket = resolvedBucket.realBucket(); if (isAclEnabled) { omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST, volume, bucket, null); } - return omSnapshotManager.getSnapshotDiffList(volume, bucket, jobStatus, listAll); + return omSnapshotManager.getSnapshotDiffList(resolvedBucket.realVolume(), resolvedBucket.realBucket(), + jobStatus, listAll); } public String printCompactionLogDag(String fileNamePrefix, From 7830c2baafd367e44f0c3dec308dca3873cdb5c7 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 21:20:38 -0800 Subject: [PATCH 07/20] HDDS-11705. Fix tests Change-Id: Ib85be9bdb30ec1ea7ad055ad6cbd696ff886bcff --- .../org/apache/hadoop/ozone/TestDataUtil.java | 3 +- .../ozone/om/snapshot/TestOmSnapshot.java | 49 +++++++++++-------- .../om/snapshot/TestOmSnapshotFileSystem.java | 37 +++++++++----- 3 files changed, 56 insertions(+), 33 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index 01883c135482..eb54e4a25190 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -212,8 +212,9 @@ public static OzoneBucket createVolumeAndBucket(OzoneClient client, bucketLayout); if (createLinkedBucket) { String targetBucketName = ozoneBucket.getName() + RandomStringUtils.randomNumeric(5); - createLinkedBucket(client, volumeName, bucketName, targetBucketName); + ozoneBucket = createLinkedBucket(client, volumeName, bucketName, targetBucketName); } + return ozoneBucket; } catch (OMException e) { if (e.getResult() != OMException.ResultCodes.VOLUME_ALREADY_EXISTS && e.getResult() != OMException.ResultCodes.BUCKET_ALREADY_EXISTS) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index f641850d494f..2f802a489873 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -20,6 +20,7 @@ import java.net.URI; import java.net.URISyntaxException; +import java.util.HashMap; import java.util.List; import com.google.common.collect.Lists; @@ -100,6 +101,7 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Iterator; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -127,6 +129,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.CONTAINS_SNAPSHOT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE; import static org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_ALREADY_CANCELLED_JOB; @@ -183,6 +186,7 @@ public abstract class TestOmSnapshot { private final boolean disableNativeDiff; private final AtomicInteger counter; private final boolean createLinkedBucket; + private final Map linkedBuckets = new HashMap<>(); public TestOmSnapshot(BucketLayout newBucketLayout, boolean newEnableFileSystemPaths, @@ -222,6 +226,9 @@ private void init() throws Exception { client = cluster.newClient(); // create a volume and a bucket to be used by OzoneFileSystem ozoneBucket = TestDataUtil.createVolumeAndBucket(client, bucketLayout, createLinkedBucket); + if (createLinkedBucket) { + this.linkedBuckets.put(ozoneBucket.getName(), ozoneBucket.getSourceBucket()); + } volumeName = ozoneBucket.getVolumeName(); bucketName = ozoneBucket.getName(); ozoneManager = cluster.getOzoneManager(); @@ -231,15 +238,16 @@ private void init() throws Exception { // stop the deletion services so that keys can still be read stopKeyManager(); - preFinalizationChecks(); +// preFinalizationChecks(); finalizeOMUpgrade(); } private void createBucket(OzoneVolume volume, String bucketVal) throws IOException { if (createLinkedBucket) { - String sourceBucketName = bucketVal + counter.incrementAndGet(); + String sourceBucketName = linkedBuckets.computeIfAbsent(bucketVal, (k) -> bucketVal + counter.incrementAndGet()); volume.createBucket(sourceBucketName); TestDataUtil.createLinkedBucket(client, volume.getName(), sourceBucketName, bucketVal); + this.linkedBuckets.put(bucketVal, sourceBucketName); } else { volume.createBucket(bucketVal); } @@ -610,11 +618,11 @@ private Set getDeletedKeysFromRocksDb( private void getOmKeyInfo(String volume, String bucket, String key) throws IOException { ResolvedBucket resolvedBucket = new ResolvedBucket(volume, bucket, - volume, bucket, "", bucketLayout); + volume, this.linkedBuckets.getOrDefault(bucket, bucket), "", bucketLayout); cluster.getOzoneManager().getKeyManager() .getKeyInfo(new OmKeyArgs.Builder() .setVolumeName(volume) - .setBucketName(bucket) + .setBucketName(this.linkedBuckets.getOrDefault(bucket, bucket)) .setKeyName(key).build(), resolvedBucket, null); } @@ -1263,7 +1271,7 @@ public void testBucketDeleteIfSnapshotExists() throws Exception { deleteKeys(bucketWithSnapshot); deleteKeys(bucketWithoutSnapshot); OMException omException = assertThrows(OMException.class, - () -> volume.deleteBucket(bucket1)); + () -> volume.deleteBucket(linkedBuckets.getOrDefault(bucket1, bucket1))); assertEquals(CONTAINS_SNAPSHOT, omException.getResult()); // TODO: Delete snapshot then delete bucket1 when deletion is implemented // no exception for bucket without snapshot @@ -1291,12 +1299,12 @@ public void testGetSnapshotInfo() throws Exception { assertEquals(snap1, snapshot1.getName()); assertEquals(volume, snapshot1.getVolumeName()); - assertEquals(bucket, snapshot1.getBucketName()); + assertEquals(linkedBuckets.getOrDefault(bucket, bucket), snapshot1.getBucketName()); OzoneSnapshot snapshot2 = store.getSnapshotInfo(volume, bucket, snap2); assertEquals(snap2, snapshot2.getName()); assertEquals(volume, snapshot2.getVolumeName()); - assertEquals(bucket, snapshot2.getBucketName()); + assertEquals(linkedBuckets.getOrDefault(bucket, bucket), snapshot2.getBucketName()); testGetSnapshotInfoFailure(null, bucket, "snapshotName", "volume can't be null or empty."); @@ -1305,9 +1313,10 @@ public void testGetSnapshotInfo() throws Exception { testGetSnapshotInfoFailure(volume, bucket, null, "snapshot name can't be null or empty."); testGetSnapshotInfoFailure(volume, bucket, "snapshotName", - "Snapshot '/" + volume + "/" + bucket + "/snapshotName' is not found."); + "Snapshot '/" + volume + "/" + linkedBuckets.getOrDefault(bucket, bucket) + + "/snapshotName' is not found."); testGetSnapshotInfoFailure(volume, "bucketName", "snapshotName", - "Snapshot '/" + volume + "/bucketName/snapshotName' is not found."); + "Bucket not found: " + volume + "/bucketName"); } public void testGetSnapshotInfoFailure(String volName, @@ -1486,9 +1495,9 @@ public void testSnapDiffCancel() throws Exception { assertEquals(CANCELLED, response.getJobStatus()); String fromSnapshotTableKey = - SnapshotInfo.getTableKey(volumeName, bucketName, fromSnapName); + SnapshotInfo.getTableKey(volumeName, linkedBuckets.getOrDefault(bucketName, bucketName), fromSnapName); String toSnapshotTableKey = - SnapshotInfo.getTableKey(volumeName, bucketName, toSnapName); + SnapshotInfo.getTableKey(volumeName, linkedBuckets.getOrDefault(bucketName, bucketName), toSnapName); UUID fromSnapshotID = SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshotTableKey).getSnapshotId(); UUID toSnapshotID = SnapshotUtils.getSnapshotInfo(ozoneManager, toSnapshotTableKey).getSnapshotId(); @@ -1625,16 +1634,16 @@ public void testSnapDiffNonExistentUrl() throws Exception { OMException omException = assertThrows(OMException.class, () -> store.snapshotDiff(volumea, bucketb, snap1, snap2, null, 0, forceFullSnapshotDiff, disableNativeDiff)); - assertEquals(KEY_NOT_FOUND, omException.getResult()); + assertEquals(BUCKET_NOT_FOUND, omException.getResult()); // Volume is nonexistent omException = assertThrows(OMException.class, () -> store.snapshotDiff(volumeb, bucketa, snap2, snap1, null, 0, forceFullSnapshotDiff, disableNativeDiff)); - assertEquals(KEY_NOT_FOUND, omException.getResult()); + assertEquals(VOLUME_NOT_FOUND, omException.getResult()); omException = assertThrows(OMException.class, () -> store.snapshotDiff(volumeb, bucketb, snap2, snap1, null, 0, forceFullSnapshotDiff, disableNativeDiff)); - assertEquals(KEY_NOT_FOUND, omException.getResult()); + assertEquals(VOLUME_NOT_FOUND, omException.getResult()); } /** @@ -1739,13 +1748,12 @@ public void testListSnapshotDiffWithInvalidParameters() String volume = "vol-" + RandomStringUtils.randomNumeric(5); String bucket = "buck-" + RandomStringUtils.randomNumeric(5); - String volBucketErrorMessage = "Provided volume name " + volume + - " or bucket name " + bucket + " doesn't exist"; + String volErrorMessage = "Volume not found: " + volume; Exception volBucketEx = assertThrows(OMException.class, () -> store.listSnapshotDiffJobs(volume, bucket, "", true)); - assertEquals(volBucketErrorMessage, + assertEquals(volErrorMessage, volBucketEx.getMessage()); // Create the volume and the bucket. @@ -1910,7 +1918,8 @@ public void testSnapshotQuotaHandling() throws Exception { OzoneVolume volume1 = store.getVolume(volume); createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); - bucket1.setQuota(OzoneQuota.parseQuota("102400000", "500")); + OzoneBucket originalBucket1 = volume1.getBucket(linkedBuckets.get(bucket)); + originalBucket1.setQuota(OzoneQuota.parseQuota("102400000", "500")); volume1.setQuota(OzoneQuota.parseQuota("204800000", "1000")); long volUsedNamespaceInitial = volume1.getUsedNamespace(); @@ -1986,7 +1995,7 @@ private String createSnapshot(String volName, String buckName, OmSnapshotManager.getSnapshotPrefix(snapshotName); SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager() .getSnapshotInfoTable() - .get(SnapshotInfo.getTableKey(volName, buckName, snapshotName)); + .get(SnapshotInfo.getTableKey(volName, linkedBuckets.getOrDefault(buckName, buckName), snapshotName)); String snapshotDirName = OmSnapshotManager.getSnapshotPath(ozoneManager.getConfiguration(), snapshotInfo) + OM_KEY_PREFIX + "CURRENT"; @@ -2322,7 +2331,7 @@ private void checkDayWeekMonthSnapshotData(OzoneBucket ozoneBucketClient, // Validate keys metadata in active Ozone namespace OzoneKeyDetails ozoneKeyDetails = ozoneBucketClient.getKey(keyName); assertEquals(keyName, ozoneKeyDetails.getName()); - assertEquals(ozoneBucketClient.getName(), + assertEquals(linkedBuckets.getOrDefault(ozoneBucketClient.getName(), ozoneBucketClient.getName()), ozoneKeyDetails.getBucketName()); assertEquals(ozoneBucketClient.getVolumeName(), ozoneKeyDetails.getVolumeName()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java index 4cf8ba9db957..41429fd3ba5c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.OzoneSnapshot; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; @@ -70,9 +71,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.UUID; @@ -119,6 +122,7 @@ public abstract class TestOmSnapshotFileSystem { private final boolean createLinkedBuckets; private FileSystem fs; private OzoneFileSystem o3fs; + private Map linkedBucketMaps = new HashMap<>(); private static final Logger LOG = LoggerFactory.getLogger(TestOmSnapshot.class); @@ -141,12 +145,20 @@ private void init() throws Exception { ozoneManager = cluster.getOzoneManager(); TestDataUtil.createVolume(client, VOLUME_NAME); - TestDataUtil.createBucket(client, VOLUME_NAME, + OzoneBucket bucket = TestDataUtil.createBucket(client, VOLUME_NAME, new BucketArgs.Builder().setBucketLayout(FILE_SYSTEM_OPTIMIZED).build(), BUCKET_NAME_FSO, createLinkedBuckets); - TestDataUtil.createBucket(client, VOLUME_NAME, + if (createLinkedBuckets) { + linkedBucketMaps.put(bucket.getName(), bucket.getSourceBucket()); + } + OzoneBucket ozoneBucket = TestDataUtil.createBucket(client, VOLUME_NAME, new BucketArgs.Builder().setBucketLayout(LEGACY).build(), BUCKET_NAME_LEGACY, createLinkedBuckets); + if (createLinkedBuckets) { + linkedBucketMaps.put(bucket.getName(), bucket.getSourceBucket()); + } + + // stop the deletion services so that keys can still be read KeyManagerImpl keyManager = (KeyManagerImpl) ozoneManager.getKeyManager(); @@ -276,7 +288,7 @@ public void testListKeysAtDifferentLevels() throws Exception { deleteSnapshot(snapshotName); String expectedMessage = String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - VOLUME_NAME, bucketName, snapshotName); + VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName); OMException exception = assertThrows(OMException.class, () -> ozoneBucket.listKeys(keyPrefix + "a/", null)); assertEquals(expectedMessage, exception.getMessage()); @@ -496,21 +508,21 @@ public void testListStatus() throws Exception { () -> fs.listStatus(snapshotRoot1)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - VOLUME_NAME, bucketName, snapshotName1), exception1.getMessage()); + VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName1), exception1.getMessage()); deleteSnapshot(snapshotName2); FileNotFoundException exception2 = assertThrows(FileNotFoundException.class, () -> fs.listStatus(snapshotRoot2)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - VOLUME_NAME, bucketName, snapshotName2), exception2.getMessage()); + VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName2), exception2.getMessage()); deleteSnapshot(snapshotName3); FileNotFoundException exception3 = assertThrows(FileNotFoundException.class, () -> fs.listStatus(snapshotParent3)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - VOLUME_NAME, bucketName, snapshotName3), exception3.getMessage()); + VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName3), exception3.getMessage()); } @Test @@ -545,7 +557,7 @@ public void testListStatusWithIntermediateDir() throws Exception { () -> fs.listStatus(snapshotParent)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage()); } @Test @@ -581,7 +593,7 @@ public void testGetFileStatus() throws Exception { () -> fs.listStatus(snapshotParent)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage()); } @Test @@ -622,7 +634,7 @@ void testReadFileFromSnapshot() throws Exception { () -> fs.open(fileInSnapshot)); assertEquals(String.format("FILE_NOT_FOUND: Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage()); } private void createAndCommitKey(String keyName) throws IOException { @@ -672,7 +684,7 @@ public void testListStatusOnRoot() throws Exception { () -> fs.listStatus(snapshotRoot)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage()); } /** @@ -729,7 +741,7 @@ public void testListStatusOnLargeDirectory() throws Exception { () -> fs.listStatus(snapshotRoot)); assertEquals(String.format("Unable to load snapshot. " + "Snapshot with table key '/%s/%s/%s' is no longer active", - VOLUME_NAME, bucketName, snapshotName), exception.getMessage()); + VOLUME_NAME, linkedBucketMaps.getOrDefault(bucketName, bucketName), snapshotName), exception.getMessage()); } private String createSnapshot(String snapshotName) @@ -739,9 +751,10 @@ private String createSnapshot(String snapshotName) writeClient.createSnapshot(VOLUME_NAME, bucketName, snapshotName); // wait till the snapshot directory exists + OzoneSnapshot snapshot = objectStore.getSnapshotInfo(VOLUME_NAME, bucketName, snapshotName); SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager() .getSnapshotInfoTable() - .get(SnapshotInfo.getTableKey(VOLUME_NAME, bucketName, snapshotName)); + .get(SnapshotInfo.getTableKey(snapshot.getVolumeName(), snapshot.getBucketName(), snapshotName)); String snapshotDirName = getSnapshotPath(conf, snapshotInfo) + OM_KEY_PREFIX + "CURRENT"; GenericTestUtils.waitFor(() -> new File(snapshotDirName).exists(), From a286afc640eecb04e757361d265e135da3aee5d4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 21:30:45 -0800 Subject: [PATCH 08/20] HDDS-11705. Address review comments and move resolve check outside to perform check only external requests Change-Id: If23f82aa5681588ac32bedb8def12d43c7bd8a5a --- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 4ca7da27f3ae..cf5263512539 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -625,7 +625,12 @@ public ReferenceCounted getActiveFsMetadataOrSnapshot( String[] keyParts = keyName.split(OM_KEY_PREFIX); if (isSnapshotKey(keyParts)) { String snapshotName = keyParts[1]; - + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket resolvedBucket = ozoneManager.resolveBucketLink(Pair.of(volumeName, + bucketName), false); + volumeName = resolvedBucket.realVolume(); + bucketName = resolvedBucket.realBucket(); return (ReferenceCounted) (ReferenceCounted) getActiveSnapshot(volumeName, bucketName, snapshotName); } else { @@ -657,12 +662,6 @@ private ReferenceCounted getSnapshot( // don't allow snapshot indicator without snapshot name throw new OMException(INVALID_KEY_NAME); } - // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a - // permission check, since linked bucket permissions and source bucket permissions could be different. - ResolvedBucket resolvedBucket = ozoneManager.resolveBucketLink(Pair.of(volumeName, - bucketName), false); - volumeName = resolvedBucket.realVolume(); - bucketName = resolvedBucket.realBucket(); String snapshotTableKey = SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName); From ce0fed04fae79dd1d0cf06459c4f7ed8afb8ac5a Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 22:35:14 -0800 Subject: [PATCH 09/20] HDDS-11705. Revert test case change Change-Id: I65418735f650ecf4f3e6901617f8b01e5702f146 --- .../org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 2f802a489873..48ca7fd4a94e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -238,7 +238,7 @@ private void init() throws Exception { // stop the deletion services so that keys can still be read stopKeyManager(); -// preFinalizationChecks(); + preFinalizationChecks(); finalizeOMUpgrade(); } From 50bc68b75997be7b3a87644c60132600a76ec83c Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 23:47:27 -0800 Subject: [PATCH 10/20] HDDS-11705. Fix bats Change-Id: I9cf926bfd4cf79dfd3709622108220128bbf5751 --- dev-support/ci/selective_ci_checks.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/ci/selective_ci_checks.bats b/dev-support/ci/selective_ci_checks.bats index 6edd38d68fe8..529a5d1f34df 100644 --- a/dev-support/ci/selective_ci_checks.bats +++ b/dev-support/ci/selective_ci_checks.bats @@ -190,7 +190,7 @@ load bats-assert/load.bash } @test "native test in other module" { - run dev-support/ci/selective_ci_checks.sh 822c0dee1a + run dev-support/ci/selective_ci_checks.sh 3115158830483c527fb978e1edc15473d81191d2 assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native"]' assert_output -p needs-build=true From d638499b39d2fdac940864161dc6857a5dde5acc Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 14 Nov 2024 23:49:18 -0800 Subject: [PATCH 11/20] HDDS-11705. Fix bats Change-Id: I4e4bfc336e70a4d5f99cb02701e494d5ea64ec4b --- .../hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java index 41429fd3ba5c..c43ec9c33c19 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystem.java @@ -151,7 +151,7 @@ private void init() throws Exception { if (createLinkedBuckets) { linkedBucketMaps.put(bucket.getName(), bucket.getSourceBucket()); } - OzoneBucket ozoneBucket = TestDataUtil.createBucket(client, VOLUME_NAME, + bucket = TestDataUtil.createBucket(client, VOLUME_NAME, new BucketArgs.Builder().setBucketLayout(LEGACY).build(), BUCKET_NAME_LEGACY, createLinkedBuckets); if (createLinkedBuckets) { From 82154a68c59b60fb0e5ec3429e481e4439a697be Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 00:10:58 -0800 Subject: [PATCH 12/20] HDDS-11705. fix bats Change-Id: I6fd76cdbc5bce189e0d12e2295e4119f9082a317 --- dev-support/ci/selective_ci_checks.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/ci/selective_ci_checks.bats b/dev-support/ci/selective_ci_checks.bats index 529a5d1f34df..d5618d80305b 100644 --- a/dev-support/ci/selective_ci_checks.bats +++ b/dev-support/ci/selective_ci_checks.bats @@ -197,7 +197,7 @@ load bats-assert/load.bash assert_output -p needs-compile=true assert_output -p needs-compose-tests=false assert_output -p needs-dependency-check=true - assert_output -p needs-integration-tests=false + assert_output -p needs-integration-tests=true assert_output -p needs-kubernetes-tests=false } From d0d5e9136b6ca71918056e65e4f2f357f19a3b2e Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 07:22:23 -0800 Subject: [PATCH 13/20] HDDS-11705. Add linked bucket info in snapshot info Change-Id: I43d6a8b49ecdb4981774fa0bb53b50159f730668 --- .../hadoop/ozone/om/helpers/SnapshotInfo.java | 71 ++++++++++++++++++- .../ozone/om/snapshot/TestOmSnapshot.java | 4 +- .../src/main/proto/OmClientProtocol.proto | 6 ++ .../snapshot/OMSnapshotCreateRequest.java | 30 +++++--- .../snapshot/TestOMSnapshotCreateRequest.java | 50 +++++++++++-- .../snapshot/TestOMSnapshotDeleteRequest.java | 28 ++++++++ .../snapshot/TestOMSnapshotRenameRequest.java | 28 ++++++++ .../ozone/om/snapshot/TestSnapshotInfo.java | 3 + .../TestSnapshotRequestAndResponse.java | 6 ++ 9 files changed, 209 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index cf0a60dd3530..8bfac49e4e23 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -114,6 +114,9 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) { private UUID globalPreviousSnapshotId; private String snapshotPath; // snapshot mask private String checkpointDir; + private boolean isLinked; + private String linkedVolumeName; + private String linkedBucketName; /** * RocksDB's transaction sequence number at the time of checkpoint creation. */ @@ -148,6 +151,9 @@ private SnapshotInfo(Builder b) { this.exclusiveReplicatedSize = b.exclusiveReplicatedSize; this.deepCleanedDeletedDir = b.deepCleanedDeletedDir; this.lastTransactionInfo = b.lastTransactionInfo; + this.linkedVolumeName = b.linkedVolumeName; + this.linkedBucketName = b.linkedBucketName; + this.isLinked = b.isLinked; } public void setName(String name) { @@ -246,6 +252,18 @@ public void setSstFiltered(boolean sstFiltered) { this.sstFiltered = sstFiltered; } + public void setLinked(boolean linked) { + isLinked = linked; + } + + public void setLinkedBucketName(String linkedBucketName) { + this.linkedBucketName = linkedBucketName; + } + + public void setLinkedVolumeName(String linkedVolumeName) { + this.linkedVolumeName = linkedVolumeName; + } + public static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.Builder newBuilder() { return new org.apache.hadoop.ozone.om.helpers.SnapshotInfo.Builder(); @@ -272,7 +290,10 @@ public SnapshotInfo.Builder toBuilder() { .setExclusiveSize(exclusiveSize) .setExclusiveReplicatedSize(exclusiveReplicatedSize) .setDeepCleanedDeletedDir(deepCleanedDeletedDir) - .setLastTransactionInfo(lastTransactionInfo); + .setLastTransactionInfo(lastTransactionInfo) + .setLinked(isLinked) + .setLinkedVolumeName(linkedVolumeName) + .setLinkedBucketName(linkedBucketName); } /** @@ -299,6 +320,9 @@ public static class Builder { private long exclusiveReplicatedSize; private boolean deepCleanedDeletedDir; private ByteString lastTransactionInfo; + private boolean isLinked = false; + private String linkedVolumeName; + private String linkedBucketName; public Builder() { // default values @@ -422,6 +446,21 @@ public Builder setLastTransactionInfo(ByteString lastTransactionInfo) { return this; } + public Builder setLinked(boolean linked) { + isLinked = linked; + return this; + } + + public Builder setLinkedBucketName(String linkedBucketName) { + this.linkedBucketName = linkedBucketName; + return this; + } + + public Builder setLinkedVolumeName(String linkedVolumeName) { + this.linkedVolumeName = linkedVolumeName; + return this; + } + public SnapshotInfo build() { Preconditions.checkNotNull(name); return new SnapshotInfo(this); @@ -459,6 +498,15 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() { if (lastTransactionInfo != null) { sib.setLastTransactionInfo(lastTransactionInfo); } + sib.setIsLinked(isLinked); + + if (linkedVolumeName != null) { + sib.setLinkedVolumeName(linkedVolumeName); + } + + if (linkedBucketName != null) { + sib.setLinkedBucketName(linkedBucketName); + } sib.setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) @@ -532,6 +580,16 @@ public static SnapshotInfo getFromProtobuf( osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo()); } + if (snapshotInfoProto.hasLinkedVolumeName()) { + osib.setLinkedVolumeName(snapshotInfoProto.getLinkedVolumeName()); + } + + if (snapshotInfoProto.hasLinkedBucketName()) { + osib.setLinkedBucketName(snapshotInfoProto.getLinkedBucketName()); + } + + osib.setLinked(snapshotInfoProto.hasIsLinked() && snapshotInfoProto.getIsLinked()); + osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) .setCheckpointDir(snapshotInfoProto.getCheckpointDir()) .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber()); @@ -701,7 +759,10 @@ public boolean equals(Object o) { exclusiveSize == that.exclusiveSize && exclusiveReplicatedSize == that.exclusiveReplicatedSize && deepCleanedDeletedDir == that.deepCleanedDeletedDir && - Objects.equals(lastTransactionInfo, that.lastTransactionInfo); + Objects.equals(lastTransactionInfo, that.lastTransactionInfo) && + isLinked == that.isLinked && + Objects.equals(linkedVolumeName, that.linkedVolumeName) && + Objects.equals(linkedBucketName, that.linkedBucketName); } @Override @@ -712,7 +773,8 @@ public int hashCode() { globalPreviousSnapshotId, snapshotPath, checkpointDir, deepClean, sstFiltered, referencedSize, referencedReplicatedSize, - exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir, lastTransactionInfo); + exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir, lastTransactionInfo, + linkedBucketName, linkedVolumeName, isLinked); } /** @@ -746,6 +808,9 @@ public String toString() { ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' + ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' + ", lastTransactionInfo: '" + lastTransactionInfo + '\'' + + ", isLinked: '" + isLinked + '\'' + + ", linkedVolumeName: '" + linkedVolumeName + '\'' + + ", linkedBucketName: '" + linkedBucketName + '\'' + '}'; } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 48ca7fd4a94e..bd93d8ae34f0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -238,7 +238,7 @@ private void init() throws Exception { // stop the deletion services so that keys can still be read stopKeyManager(); - preFinalizationChecks(); +// preFinalizationChecks(); finalizeOMUpgrade(); } @@ -1918,7 +1918,7 @@ public void testSnapshotQuotaHandling() throws Exception { OzoneVolume volume1 = store.getVolume(volume); createBucket(volume1, bucket); OzoneBucket bucket1 = volume1.getBucket(bucket); - OzoneBucket originalBucket1 = volume1.getBucket(linkedBuckets.get(bucket)); + OzoneBucket originalBucket1 = volume1.getBucket(linkedBuckets.getOrDefault(bucket, bucket)); originalBucket1.setQuota(OzoneQuota.parseQuota("102400000", "500")); volume1.setQuota(OzoneQuota.parseQuota("204800000", "1000")); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index f71dc44fec51..6e0eeec998dc 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -886,6 +886,9 @@ message SnapshotInfo { // note: shared sizes can be calculated from: referenced - exclusive optional bool deepCleanedDeletedDir = 19; optional bytes lastTransactionInfo = 20; + optional bool isLinked = 21; + optional string linkedVolumeName = 22; + optional string linkedBucketName = 23; } message SnapshotDiffJobProto { @@ -1920,6 +1923,9 @@ message CreateSnapshotRequest { optional string snapshotName = 3; optional hadoop.hdds.UUID snapshotId = 4; optional uint64 creationTime = 5; + optional bool isLinked = 6; + optional string linkedVolumeName = 7; + optional string linkedBucketName = 8; } message RenameSnapshotRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 1598f228d36b..71c2d5c322f2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -122,14 +122,17 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { "Only bucket owners and Ozone admins can create snapshots", OMException.ResultCodes.PERMISSION_DENIED); } - - return omRequest.toBuilder().setCreateSnapshotRequest( - omRequest.getCreateSnapshotRequest().toBuilder() - .setSnapshotId(toProtobuf(UUID.randomUUID())) - .setVolumeName(volumeName) - .setBucketName(this.bucketName) - .setCreationTime(Time.now()) - .build()).build(); + CreateSnapshotRequest.Builder createSnapshotRequest = omRequest.getCreateSnapshotRequest().toBuilder() + .setSnapshotId(toProtobuf(UUID.randomUUID())) + .setVolumeName(volumeName) + .setBucketName(this.bucketName) + .setCreationTime(Time.now()); + if (bucket.isLink()) { + createSnapshotRequest.setIsLinked(true); + createSnapshotRequest.setLinkedVolumeName(bucket.requestedVolume()); + createSnapshotRequest.setLinkedBucketName(bucket.requestedBucket()); + } + return omRequest.toBuilder().setCreateSnapshotRequest(createSnapshotRequest.build()).build(); } @Override @@ -169,6 +172,17 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn throw new OMException("Snapshot already exists", FILE_ALREADY_EXISTS); } + CreateSnapshotRequest createSnapshotRequest = getOmRequest().getCreateSnapshotRequest(); + if (createSnapshotRequest.hasIsLinked()) { + snapshotInfo.setLinked(true); + } + if (createSnapshotRequest.hasLinkedVolumeName()) { + snapshotInfo.setLinkedVolumeName(createSnapshotRequest.getLinkedVolumeName()); + } + if (createSnapshotRequest.hasLinkedBucketName()) { + snapshotInfo.setLinkedBucketName(createSnapshotRequest.getLinkedBucketName()); + } + // Note down RDB latest transaction sequence number, which is used // as snapshot generation in the Differ. final long dbLatestSequenceNumber = diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index af904382256b..cf8252a62f0c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -18,15 +18,18 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse; @@ -86,6 +89,31 @@ public void testPreExecute(String snapshotName) throws Exception { doPreExecute(omRequest); } + @ValueSource(strings = { + // '-' is allowed. + "9cdf0e8a-6946-41ad-a2d1-9eb724fab126", + // 3 chars name is allowed. + "sn1", + // less than or equal to 63 chars are allowed. + "snap75795657617173401188448010125899089001363595171500499231286" + }) + @ParameterizedTest + public void testPreExecuteWithLinkedBucket(String snapshotName) throws Exception { + when(getOzoneManager().isOwner(any(), any())).thenReturn(true); + String resolvedBucketName = getBucketName() + "1"; + String resolvedVolumeName = getVolumeName() + "1"; + when(getOzoneManager().resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) + .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), Pair.of(resolvedVolumeName, resolvedBucketName) + , "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); + OMRequest omRequest = createSnapshotRequest(getVolumeName(), + getBucketName(), snapshotName); + OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); + assertEquals(resolvedVolumeName, omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getVolumeName()); + assertEquals(resolvedBucketName, omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getBucketName()); + assertEquals(getBucketName(), omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getLinkedBucketName()); + assertEquals(getVolumeName(), omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getLinkedVolumeName()); + } + @ValueSource(strings = { // ? is not allowed in snapshot name. "a?b", @@ -119,11 +147,18 @@ public void testPreExecuteBadOwner() { omException.getMessage()); } - @Test - public void testValidateAndUpdateCache() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testValidateAndUpdateCache(boolean isLinked) throws Exception { when(getOzoneManager().isAdmin(any())).thenReturn(true); - OMRequest omRequest = createSnapshotRequest(getVolumeName(), - getBucketName(), snapshotName1); + String linkedBucketName = getBucketName() + "1"; + String linkedVolumeName = getVolumeName() + "1"; + if (isLinked) { + when(getOzoneManager().resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) + .thenAnswer(i -> new ResolvedBucket(Pair.of(linkedVolumeName, linkedBucketName), + Pair.of(getVolumeName(), getBucketName()), "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); + } + OMRequest omRequest = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName1); OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); String key = getTableKey(getVolumeName(), getBucketName(), snapshotName1); String bucketKey = getOmMetadataManager().getBucketKey(getVolumeName(), getBucketName()); @@ -159,6 +194,13 @@ public void testValidateAndUpdateCache() throws Exception { .getCreateSnapshotResponse() .getSnapshotInfo(); + if (isLinked) { + assertEquals(isLinked, snapshotInfoProto.getIsLinked()); + assertEquals(linkedVolumeName, snapshotInfoProto.getLinkedVolumeName()); + assertEquals(linkedBucketName, snapshotInfoProto.getLinkedBucketName()); + } else { + assertEquals(isLinked, snapshotInfoProto.hasIsLinked() && snapshotInfoProto.getIsLinked()); + } assertEquals(bucketDataSize, snapshotInfoProto.getReferencedSize()); assertEquals(bucketUsedBytes, snapshotInfoProto.getReferencedReplicatedSize()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java index 4c5dc2e77f0f..d80c2d740fd4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java @@ -19,10 +19,14 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.snapshot.TestSnapshotRequestAndResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -34,6 +38,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.io.IOException; import java.util.UUID; import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; @@ -81,6 +86,29 @@ public void testPreExecute(String deleteSnapshotName) throws Exception { doPreExecute(omRequest); } + @ValueSource(strings = { + // '-' is allowed. + "9cdf0e8a-6946-41ad-a2d1-9eb724fab126", + // 3 chars name is allowed. + "sn1", + // less than or equal to 63 chars are allowed. + "snap75795657617173401188448010125899089001363595171500499231286" + }) + @ParameterizedTest + public void testPreExecuteWithLinkedBuckets(String deleteSnapshotName) throws Exception { + when(getOzoneManager().isOwner(any(), any())).thenReturn(true); + String resolvedBucketName = getBucketName() + "1"; + String resolvedVolumeName = getVolumeName() + "1"; + when(getOzoneManager().resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) + .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), Pair.of(resolvedVolumeName, resolvedBucketName) + , "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); + OMRequest omRequest = deleteSnapshotRequest(getVolumeName(), + getBucketName(), deleteSnapshotName); + OMSnapshotDeleteRequest omSnapshotDeleteRequest = doPreExecute(omRequest); + assertEquals(resolvedVolumeName, omSnapshotDeleteRequest.getOmRequest().getDeleteSnapshotRequest().getVolumeName()); + assertEquals(resolvedBucketName, omSnapshotDeleteRequest.getOmRequest().getDeleteSnapshotRequest().getBucketName()); + } + @ValueSource(strings = { // ? is not allowed in snapshot name. "a?b", diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java index a746597288aa..41c6ee0384af 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java @@ -16,14 +16,18 @@ */ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.snapshot.TestSnapshotRequestAndResponse; @@ -83,6 +87,30 @@ public void testPreExecute(String toSnapshotName) throws Exception { doPreExecute(omRequest); } + @ValueSource(strings = { + // '-' is allowed. + "9cdf0e8a-6946-41ad-a2d1-9eb724fab126", + // 3 chars name is allowed. + "sn1", + // less than or equal to 63 chars are allowed. + "snap75795657617173401188448010125899089001363595171500499231286" + }) + @ParameterizedTest + public void testPreExecuteWithLinkedBucket(String toSnapshotName) throws Exception { + when(getOzoneManager().isOwner(any(), any())).thenReturn(true); + String resolvedBucketName = getBucketName() + "1"; + String resolvedVolumeName = getVolumeName() + "1"; + when(getOzoneManager().resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) + .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), Pair.of(resolvedVolumeName, resolvedBucketName) + , "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); + String currentSnapshotName = "current"; + OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(getVolumeName(), + getBucketName(), currentSnapshotName, toSnapshotName); + OMSnapshotRenameRequest omSnapshotRenameRequest = doPreExecute(omRequest); + assertEquals(resolvedVolumeName, omSnapshotRenameRequest.getOmRequest().getRenameSnapshotRequest().getVolumeName()); + assertEquals(resolvedBucketName, omSnapshotRenameRequest.getOmRequest().getRenameSnapshotRequest().getBucketName()); + } + @ValueSource(strings = { // ? is not allowed in snapshot name. "a?b", diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java index 29e0115861fe..9197caec0424 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java @@ -78,6 +78,9 @@ private SnapshotInfo createSnapshotInfo() { .setGlobalPreviousSnapshotId(EXPECTED_PREVIOUS_SNAPSHOT_ID) .setSnapshotPath("test/path") .setCheckpointDir("checkpoint.testdir") + .setLinked(true) + .setLinkedVolumeName("vol1") + .setLinkedBucketName("bucket1") .build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java index e60e23de22a3..b037b68fd72b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotRequestAndResponse.java @@ -32,9 +32,12 @@ import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ResolvedBucket; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; import org.apache.hadoop.ozone.om.request.snapshot.TestOMSnapshotCreateRequest; @@ -140,6 +143,9 @@ public void baseSetup() throws Exception { omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager); when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); + when(ozoneManager.resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) + .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), + i.getArgument(0), "dummyBucketOwner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); when(ozoneManager.getMetrics()).thenReturn(omMetrics); when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); when(ozoneManager.isRatisEnabled()).thenReturn(true); From 029655ddd380c2fbf1520618e6e7e0defd08dd5b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 07:26:25 -0800 Subject: [PATCH 14/20] HDDS-11705. Fix checkstyle Change-Id: I290d4a4ca4eb386fe9d4ccf7a8e0111d5b4017b6 --- .../request/snapshot/TestOMSnapshotCreateRequest.java | 10 ++++++---- .../request/snapshot/TestOMSnapshotDeleteRequest.java | 5 ++--- .../request/snapshot/TestOMSnapshotRenameRequest.java | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index cf8252a62f0c..31be7b9e5aca 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -103,15 +103,17 @@ public void testPreExecuteWithLinkedBucket(String snapshotName) throws Exception String resolvedBucketName = getBucketName() + "1"; String resolvedVolumeName = getVolumeName() + "1"; when(getOzoneManager().resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) - .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), Pair.of(resolvedVolumeName, resolvedBucketName) - , "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); + .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), Pair.of(resolvedVolumeName, resolvedBucketName), + "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); OMRequest omRequest = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName); OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); assertEquals(resolvedVolumeName, omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getVolumeName()); assertEquals(resolvedBucketName, omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getBucketName()); - assertEquals(getBucketName(), omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getLinkedBucketName()); - assertEquals(getVolumeName(), omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getLinkedVolumeName()); + assertEquals(getBucketName(), + omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getLinkedBucketName()); + assertEquals(getVolumeName(), + omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getLinkedVolumeName()); } @ValueSource(strings = { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java index d80c2d740fd4..9e19e5948436 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java @@ -38,7 +38,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import java.io.IOException; import java.util.UUID; import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE; @@ -100,8 +99,8 @@ public void testPreExecuteWithLinkedBuckets(String deleteSnapshotName) throws Ex String resolvedBucketName = getBucketName() + "1"; String resolvedVolumeName = getVolumeName() + "1"; when(getOzoneManager().resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) - .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), Pair.of(resolvedVolumeName, resolvedBucketName) - , "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); + .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), Pair.of(resolvedVolumeName, resolvedBucketName), + "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); OMRequest omRequest = deleteSnapshotRequest(getVolumeName(), getBucketName(), deleteSnapshotName); OMSnapshotDeleteRequest omSnapshotDeleteRequest = doPreExecute(omRequest); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java index 41c6ee0384af..8059c3ce501a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotRenameRequest.java @@ -101,8 +101,8 @@ public void testPreExecuteWithLinkedBucket(String toSnapshotName) throws Excepti String resolvedBucketName = getBucketName() + "1"; String resolvedVolumeName = getVolumeName() + "1"; when(getOzoneManager().resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) - .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), Pair.of(resolvedVolumeName, resolvedBucketName) - , "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); + .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), Pair.of(resolvedVolumeName, resolvedBucketName), + "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); String currentSnapshotName = "current"; OzoneManagerProtocolProtos.OMRequest omRequest = renameSnapshotRequest(getVolumeName(), getBucketName(), currentSnapshotName, toSnapshotName); From e5353c7540f961c82df272583a31fce00db47933 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 07:29:34 -0800 Subject: [PATCH 15/20] HDDS-11705. Revert commented code Change-Id: I580ee29560a4e7b22ea80a8654dab1e392f03935 --- .../org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index bd93d8ae34f0..4d0e56dec01e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -238,7 +238,7 @@ private void init() throws Exception { // stop the deletion services so that keys can still be read stopKeyManager(); -// preFinalizationChecks(); + preFinalizationChecks(); finalizeOMUpgrade(); } From 0105fade4c46a7d59717ac1d9f21de132d3ac85c Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 08:10:16 -0800 Subject: [PATCH 16/20] HDDS-11705. Revert renaming of WithoutLinkedBuckets tests Change-Id: I29f07e09efb60a3d76c9b5312af2f82f805153ad --- dev-support/ci/selective_ci_checks.bats | 4 ++-- ...outLinkedBuckets.java => TestOmSnapshotFileSystemFso.java} | 4 ++-- ...LinkedBuckets.java => TestOmSnapshotFileSystemLegacy.java} | 4 ++-- ...LinkedBuckets.java => TestOmSnapshotFsoWithNativeLib.java} | 4 ++-- ...kedBuckets.java => TestOmSnapshotFsoWithoutNativeLib.java} | 4 ++-- ...thoutLinkedBuckets.java => TestOmSnapshotObjectStore.java} | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotFileSystemFsoWithoutLinkedBuckets.java => TestOmSnapshotFileSystemFso.java} (85%) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets.java => TestOmSnapshotFileSystemLegacy.java} (85%) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets.java => TestOmSnapshotFsoWithNativeLib.java} (89%) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets.java => TestOmSnapshotFsoWithoutNativeLib.java} (86%) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/{TestOmSnapshotObjectStoreWithoutLinkedBuckets.java => TestOmSnapshotObjectStore.java} (87%) diff --git a/dev-support/ci/selective_ci_checks.bats b/dev-support/ci/selective_ci_checks.bats index d5618d80305b..6edd38d68fe8 100644 --- a/dev-support/ci/selective_ci_checks.bats +++ b/dev-support/ci/selective_ci_checks.bats @@ -190,14 +190,14 @@ load bats-assert/load.bash } @test "native test in other module" { - run dev-support/ci/selective_ci_checks.sh 3115158830483c527fb978e1edc15473d81191d2 + run dev-support/ci/selective_ci_checks.sh 822c0dee1a assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","native"]' assert_output -p needs-build=true assert_output -p needs-compile=true assert_output -p needs-compose-tests=false assert_output -p needs-dependency-check=true - assert_output -p needs-integration-tests=true + assert_output -p needs-integration-tests=false assert_output -p needs-kubernetes-tests=false } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java similarity index 85% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithoutLinkedBuckets.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java index c063ad548500..17adf6cce72f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFsoWithoutLinkedBuckets.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemFso.java @@ -24,8 +24,8 @@ * OmSnapshot file system tests for FSO. */ @Timeout(120) -public class TestOmSnapshotFileSystemFsoWithoutLinkedBuckets extends TestOmSnapshotFileSystem { - TestOmSnapshotFileSystemFsoWithoutLinkedBuckets() throws Exception { +public class TestOmSnapshotFileSystemFso extends TestOmSnapshotFileSystem { + TestOmSnapshotFileSystemFso() throws Exception { super(BUCKET_NAME_FSO, false); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacy.java similarity index 85% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacy.java index 2dc2f5e1e145..effaaa5d4e7f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFileSystemLegacy.java @@ -24,8 +24,8 @@ * OmSnapshot file system tests for Legacy. */ @Timeout(120) -public class TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets extends TestOmSnapshotFileSystem { - TestOmSnapshotFileSystemLegacyWithoutLinkedBuckets() throws Exception { +public class TestOmSnapshotFileSystemLegacy extends TestOmSnapshotFileSystem { + TestOmSnapshotFileSystemLegacy() throws Exception { super(BUCKET_NAME_LEGACY, false); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java similarity index 89% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java index 2fae92026a52..c303b24ad24b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java @@ -29,8 +29,8 @@ */ @Native(ROCKS_TOOLS_NATIVE_LIBRARY_NAME) @Timeout(300) -class TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets extends TestOmSnapshot { - TestOmSnapshotFsoWithNativeLibWithoutLinkedBuckets() throws Exception { +class TestOmSnapshotFsoWithNativeLib extends TestOmSnapshot { + TestOmSnapshotFsoWithNativeLib() throws Exception { super(FILE_SYSTEM_OPTIMIZED, false, false, false, false); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java similarity index 86% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java index 772dc73ff2d5..26262916cb85 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLib.java @@ -26,9 +26,9 @@ * Test OmSnapshot for FSO bucket type when native lib is disabled. */ @Timeout(300) -public class TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets extends TestOmSnapshot { +public class TestOmSnapshotFsoWithoutNativeLib extends TestOmSnapshot { - public TestOmSnapshotFsoWithoutNativeLibWithoutLinkedBuckets() throws Exception { + public TestOmSnapshotFsoWithoutNativeLib() throws Exception { super(FILE_SYSTEM_OPTIMIZED, false, false, true, false); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithoutLinkedBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java similarity index 87% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithoutLinkedBuckets.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java index 755e7e1412b8..bad51103a55b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithoutLinkedBuckets.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java @@ -26,9 +26,9 @@ * Test OmSnapshot for Object Store bucket type. */ @Timeout(300) -public class TestOmSnapshotObjectStoreWithoutLinkedBuckets extends TestOmSnapshot { +public class TestOmSnapshotObjectStore extends TestOmSnapshot { - public TestOmSnapshotObjectStoreWithoutLinkedBuckets() throws Exception { + public TestOmSnapshotObjectStore() throws Exception { super(OBJECT_STORE, false, false, false, true); } } From 798487e3fb75d4376090d8add58395db2669b860 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 09:33:04 -0800 Subject: [PATCH 17/20] HDDS-11705. Fix test case Change-Id: I9abc563cf4c0888125a7b1f9513a785b4deee01e --- .../apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java index dd9cf34c8e29..904740ff7a71 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java @@ -73,6 +73,9 @@ private SnapshotInfo createSnapshotInfo() { .setExclusiveSize(1000L) .setExclusiveReplicatedSize(3000L) .setDeepCleanedDeletedDir(false) + .setLinked(true) + .setLinkedVolumeName(VOLUME_NAME) + .setLinkedBucketName(BUCKET_NAME) .build(); } @@ -97,6 +100,9 @@ private OzoneManagerProtocolProtos.SnapshotInfo createSnapshotInfoProto() { .setExclusiveSize(1000L) .setExclusiveReplicatedSize(3000L) .setDeepCleanedDeletedDir(false) + .setLinkedBucketName(BUCKET_NAME) + .setLinkedVolumeName(VOLUME_NAME) + .setIsLinked(true) .build(); } From 85daf589b02783bba7dd0a6e6e4170b2ef94372b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 11:40:39 -0800 Subject: [PATCH 18/20] HDDS-11705. Fix test case Change-Id: Ibaf927b11d9f617ac738921f675a0e6d3631b71f --- .../hadoop/ozone/om/helpers/SnapshotInfo.java | 71 +------------------ .../ozone/om/helpers/TestOmSnapshotInfo.java | 6 -- .../src/main/proto/OmClientProtocol.proto | 6 -- .../apache/hadoop/ozone/om/OzoneManager.java | 22 +++--- .../snapshot/OMSnapshotCreateRequest.java | 16 ----- .../snapshot/TestOMSnapshotCreateRequest.java | 22 ++---- .../ozone/om/snapshot/TestSnapshotInfo.java | 3 - 7 files changed, 21 insertions(+), 125 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index 8bfac49e4e23..cf0a60dd3530 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -114,9 +114,6 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) { private UUID globalPreviousSnapshotId; private String snapshotPath; // snapshot mask private String checkpointDir; - private boolean isLinked; - private String linkedVolumeName; - private String linkedBucketName; /** * RocksDB's transaction sequence number at the time of checkpoint creation. */ @@ -151,9 +148,6 @@ private SnapshotInfo(Builder b) { this.exclusiveReplicatedSize = b.exclusiveReplicatedSize; this.deepCleanedDeletedDir = b.deepCleanedDeletedDir; this.lastTransactionInfo = b.lastTransactionInfo; - this.linkedVolumeName = b.linkedVolumeName; - this.linkedBucketName = b.linkedBucketName; - this.isLinked = b.isLinked; } public void setName(String name) { @@ -252,18 +246,6 @@ public void setSstFiltered(boolean sstFiltered) { this.sstFiltered = sstFiltered; } - public void setLinked(boolean linked) { - isLinked = linked; - } - - public void setLinkedBucketName(String linkedBucketName) { - this.linkedBucketName = linkedBucketName; - } - - public void setLinkedVolumeName(String linkedVolumeName) { - this.linkedVolumeName = linkedVolumeName; - } - public static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.Builder newBuilder() { return new org.apache.hadoop.ozone.om.helpers.SnapshotInfo.Builder(); @@ -290,10 +272,7 @@ public SnapshotInfo.Builder toBuilder() { .setExclusiveSize(exclusiveSize) .setExclusiveReplicatedSize(exclusiveReplicatedSize) .setDeepCleanedDeletedDir(deepCleanedDeletedDir) - .setLastTransactionInfo(lastTransactionInfo) - .setLinked(isLinked) - .setLinkedVolumeName(linkedVolumeName) - .setLinkedBucketName(linkedBucketName); + .setLastTransactionInfo(lastTransactionInfo); } /** @@ -320,9 +299,6 @@ public static class Builder { private long exclusiveReplicatedSize; private boolean deepCleanedDeletedDir; private ByteString lastTransactionInfo; - private boolean isLinked = false; - private String linkedVolumeName; - private String linkedBucketName; public Builder() { // default values @@ -446,21 +422,6 @@ public Builder setLastTransactionInfo(ByteString lastTransactionInfo) { return this; } - public Builder setLinked(boolean linked) { - isLinked = linked; - return this; - } - - public Builder setLinkedBucketName(String linkedBucketName) { - this.linkedBucketName = linkedBucketName; - return this; - } - - public Builder setLinkedVolumeName(String linkedVolumeName) { - this.linkedVolumeName = linkedVolumeName; - return this; - } - public SnapshotInfo build() { Preconditions.checkNotNull(name); return new SnapshotInfo(this); @@ -498,15 +459,6 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() { if (lastTransactionInfo != null) { sib.setLastTransactionInfo(lastTransactionInfo); } - sib.setIsLinked(isLinked); - - if (linkedVolumeName != null) { - sib.setLinkedVolumeName(linkedVolumeName); - } - - if (linkedBucketName != null) { - sib.setLinkedBucketName(linkedBucketName); - } sib.setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) @@ -580,16 +532,6 @@ public static SnapshotInfo getFromProtobuf( osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo()); } - if (snapshotInfoProto.hasLinkedVolumeName()) { - osib.setLinkedVolumeName(snapshotInfoProto.getLinkedVolumeName()); - } - - if (snapshotInfoProto.hasLinkedBucketName()) { - osib.setLinkedBucketName(snapshotInfoProto.getLinkedBucketName()); - } - - osib.setLinked(snapshotInfoProto.hasIsLinked() && snapshotInfoProto.getIsLinked()); - osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) .setCheckpointDir(snapshotInfoProto.getCheckpointDir()) .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber()); @@ -759,10 +701,7 @@ public boolean equals(Object o) { exclusiveSize == that.exclusiveSize && exclusiveReplicatedSize == that.exclusiveReplicatedSize && deepCleanedDeletedDir == that.deepCleanedDeletedDir && - Objects.equals(lastTransactionInfo, that.lastTransactionInfo) && - isLinked == that.isLinked && - Objects.equals(linkedVolumeName, that.linkedVolumeName) && - Objects.equals(linkedBucketName, that.linkedBucketName); + Objects.equals(lastTransactionInfo, that.lastTransactionInfo); } @Override @@ -773,8 +712,7 @@ public int hashCode() { globalPreviousSnapshotId, snapshotPath, checkpointDir, deepClean, sstFiltered, referencedSize, referencedReplicatedSize, - exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir, lastTransactionInfo, - linkedBucketName, linkedVolumeName, isLinked); + exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir, lastTransactionInfo); } /** @@ -808,9 +746,6 @@ public String toString() { ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' + ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' + ", lastTransactionInfo: '" + lastTransactionInfo + '\'' + - ", isLinked: '" + isLinked + '\'' + - ", linkedVolumeName: '" + linkedVolumeName + '\'' + - ", linkedBucketName: '" + linkedBucketName + '\'' + '}'; } } diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java index 904740ff7a71..dd9cf34c8e29 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java @@ -73,9 +73,6 @@ private SnapshotInfo createSnapshotInfo() { .setExclusiveSize(1000L) .setExclusiveReplicatedSize(3000L) .setDeepCleanedDeletedDir(false) - .setLinked(true) - .setLinkedVolumeName(VOLUME_NAME) - .setLinkedBucketName(BUCKET_NAME) .build(); } @@ -100,9 +97,6 @@ private OzoneManagerProtocolProtos.SnapshotInfo createSnapshotInfoProto() { .setExclusiveSize(1000L) .setExclusiveReplicatedSize(3000L) .setDeepCleanedDeletedDir(false) - .setLinkedBucketName(BUCKET_NAME) - .setLinkedVolumeName(VOLUME_NAME) - .setIsLinked(true) .build(); } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 6e0eeec998dc..f71dc44fec51 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -886,9 +886,6 @@ message SnapshotInfo { // note: shared sizes can be calculated from: referenced - exclusive optional bool deepCleanedDeletedDir = 19; optional bytes lastTransactionInfo = 20; - optional bool isLinked = 21; - optional string linkedVolumeName = 22; - optional string linkedBucketName = 23; } message SnapshotDiffJobProto { @@ -1923,9 +1920,6 @@ message CreateSnapshotRequest { optional string snapshotName = 3; optional hadoop.hdds.UUID snapshotId = 4; optional uint64 creationTime = 5; - optional bool isLinked = 6; - optional string linkedVolumeName = 7; - optional string linkedBucketName = 8; } message RenameSnapshotRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 77ae7aefb0c6..dc99e1d880c6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2970,10 +2970,14 @@ public SnapshotInfo getSnapshotInfo(String volumeName, String bucketName, metrics.incNumSnapshotInfos(); // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a // permission check, since linked bucket permissions and source bucket permissions could be different. - ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); - Map auditMap = buildAuditMap(resolvedBucket.realVolume()); - auditMap.put(OzoneConsts.BUCKET, resolvedBucket.realBucket()); + Map auditMap = buildAuditMap(volumeName); + auditMap.put(OzoneConsts.BUCKET, bucketName); try { + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); + auditMap = buildAuditMap(resolvedBucket.realVolume()); + auditMap.put(OzoneConsts.BUCKET, resolvedBucket.realBucket()); SnapshotInfo snapshotInfo = metadataManager.getSnapshotInfo(resolvedBucket.realVolume(), resolvedBucket.realBucket(), snapshotName); @@ -2993,12 +2997,14 @@ public ListSnapshotResponse listSnapshot( String volumeName, String bucketName, String snapshotPrefix, String prevSnapshot, int maxListResult) throws IOException { metrics.incNumSnapshotLists(); - // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a - // permission check, since linked bucket permissions and source bucket permissions could be different. - ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); - Map auditMap = buildAuditMap(resolvedBucket.realVolume()); - auditMap.put(OzoneConsts.BUCKET, resolvedBucket.realBucket()); + Map auditMap = buildAuditMap(volumeName); + auditMap.put(OzoneConsts.BUCKET, bucketName); try { + // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a + // permission check, since linked bucket permissions and source bucket permissions could be different. + ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); + auditMap = buildAuditMap(resolvedBucket.realVolume()); + auditMap.put(OzoneConsts.BUCKET, resolvedBucket.realBucket()); if (isAclEnabled) { omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST, resolvedBucket.realVolume(), resolvedBucket.realBucket(), null); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 71c2d5c322f2..59cc02b6fdb5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -127,11 +127,6 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { .setVolumeName(volumeName) .setBucketName(this.bucketName) .setCreationTime(Time.now()); - if (bucket.isLink()) { - createSnapshotRequest.setIsLinked(true); - createSnapshotRequest.setLinkedVolumeName(bucket.requestedVolume()); - createSnapshotRequest.setLinkedBucketName(bucket.requestedBucket()); - } return omRequest.toBuilder().setCreateSnapshotRequest(createSnapshotRequest.build()).build(); } @@ -172,17 +167,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn throw new OMException("Snapshot already exists", FILE_ALREADY_EXISTS); } - CreateSnapshotRequest createSnapshotRequest = getOmRequest().getCreateSnapshotRequest(); - if (createSnapshotRequest.hasIsLinked()) { - snapshotInfo.setLinked(true); - } - if (createSnapshotRequest.hasLinkedVolumeName()) { - snapshotInfo.setLinkedVolumeName(createSnapshotRequest.getLinkedVolumeName()); - } - if (createSnapshotRequest.hasLinkedBucketName()) { - snapshotInfo.setLinkedBucketName(createSnapshotRequest.getLinkedBucketName()); - } - // Note down RDB latest transaction sequence number, which is used // as snapshot generation in the Differ. final long dbLatestSequenceNumber = diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index 31be7b9e5aca..dde557f628ca 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -149,18 +149,11 @@ public void testPreExecuteBadOwner() { omException.getMessage()); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - public void testValidateAndUpdateCache(boolean isLinked) throws Exception { + @Test + public void testValidateAndUpdateCache() throws Exception { when(getOzoneManager().isAdmin(any())).thenReturn(true); - String linkedBucketName = getBucketName() + "1"; - String linkedVolumeName = getVolumeName() + "1"; - if (isLinked) { - when(getOzoneManager().resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) - .thenAnswer(i -> new ResolvedBucket(Pair.of(linkedVolumeName, linkedBucketName), - Pair.of(getVolumeName(), getBucketName()), "owner", BucketLayout.FILE_SYSTEM_OPTIMIZED)); - } - OMRequest omRequest = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName1); + OMRequest omRequest = createSnapshotRequest(getVolumeName(), + getBucketName(), snapshotName1); OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); String key = getTableKey(getVolumeName(), getBucketName(), snapshotName1); String bucketKey = getOmMetadataManager().getBucketKey(getVolumeName(), getBucketName()); @@ -196,13 +189,6 @@ public void testValidateAndUpdateCache(boolean isLinked) throws Exception { .getCreateSnapshotResponse() .getSnapshotInfo(); - if (isLinked) { - assertEquals(isLinked, snapshotInfoProto.getIsLinked()); - assertEquals(linkedVolumeName, snapshotInfoProto.getLinkedVolumeName()); - assertEquals(linkedBucketName, snapshotInfoProto.getLinkedBucketName()); - } else { - assertEquals(isLinked, snapshotInfoProto.hasIsLinked() && snapshotInfoProto.getIsLinked()); - } assertEquals(bucketDataSize, snapshotInfoProto.getReferencedSize()); assertEquals(bucketUsedBytes, snapshotInfoProto.getReferencedReplicatedSize()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java index 9197caec0424..29e0115861fe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java @@ -78,9 +78,6 @@ private SnapshotInfo createSnapshotInfo() { .setGlobalPreviousSnapshotId(EXPECTED_PREVIOUS_SNAPSHOT_ID) .setSnapshotPath("test/path") .setCheckpointDir("checkpoint.testdir") - .setLinked(true) - .setLinkedVolumeName("vol1") - .setLinkedBucketName("bucket1") .build(); } From 0bee4e3c87981d46fd86344377ea59e5e7b7b6ef Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 12:32:24 -0800 Subject: [PATCH 19/20] HDDS-11705. Address review comments Change-Id: Ia080b9985766f9d6f9c4f45250db2d976334d943 --- .../main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index dc99e1d880c6..2facdaccd2f6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2968,14 +2968,12 @@ public ListKeysLightResult listKeysLight(String volumeName, public SnapshotInfo getSnapshotInfo(String volumeName, String bucketName, String snapshotName) throws IOException { metrics.incNumSnapshotInfos(); - // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a - // permission check, since linked bucket permissions and source bucket permissions could be different. Map auditMap = buildAuditMap(volumeName); auditMap.put(OzoneConsts.BUCKET, bucketName); try { // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a // permission check, since linked bucket permissions and source bucket permissions could be different. - ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); + ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName)); auditMap = buildAuditMap(resolvedBucket.realVolume()); auditMap.put(OzoneConsts.BUCKET, resolvedBucket.realBucket()); SnapshotInfo snapshotInfo = @@ -3002,7 +3000,7 @@ public ListSnapshotResponse listSnapshot( try { // Updating the volumeName & bucketName in case the bucket is a linked bucket. We need to do this before a // permission check, since linked bucket permissions and source bucket permissions could be different. - ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName), false); + ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(volumeName, bucketName)); auditMap = buildAuditMap(resolvedBucket.realVolume()); auditMap.put(OzoneConsts.BUCKET, resolvedBucket.realBucket()); if (isAclEnabled) { From a861e5a381ba4ae6c5e66df21efa2875a6fdca60 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 13:52:26 -0800 Subject: [PATCH 20/20] HDDS-11705. Fix build issue Change-Id: I5262aeaf17498582fc6c46d041a8a27f99dc73b8 --- .../om/request/snapshot/TestOMSnapshotCreateRequest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index dde557f628ca..b7b7ff0a4647 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -110,10 +110,6 @@ public void testPreExecuteWithLinkedBucket(String snapshotName) throws Exception OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); assertEquals(resolvedVolumeName, omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getVolumeName()); assertEquals(resolvedBucketName, omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getBucketName()); - assertEquals(getBucketName(), - omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getLinkedBucketName()); - assertEquals(getVolumeName(), - omSnapshotCreateRequest.getOmRequest().getCreateSnapshotRequest().getLinkedVolumeName()); } @ValueSource(strings = {