From 17f7cefff38c81ac65e7ebcfa1b608cb3186d428 Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Mon, 12 Aug 2024 17:18:12 +0530 Subject: [PATCH 01/16] Initial commit for listTrash and recoverTrash API deprecation --- .../ozone/client/protocol/ClientProtocol.java | 3 + .../hadoop/ozone/client/rpc/RpcClient.java | 13 +- .../java/org/apache/hadoop/ozone/OmUtils.java | 2 - .../om/protocol/OzoneManagerProtocol.java | 6 + ...ManagerProtocolClientSideTranslatorPB.java | 76 +--------- .../src/main/proto/OmClientProtocol.proto | 16 +- .../hadoop/ozone/om/OMMetadataManager.java | 6 + .../apache/hadoop/ozone/om/KeyManager.java | 3 + .../hadoop/ozone/om/KeyManagerImpl.java | 11 +- .../ozone/om/OmMetadataManagerImpl.java | 14 +- .../apache/hadoop/ozone/om/OzoneManager.java | 29 +--- .../OzoneManagerRequestHandler.java | 25 +--- .../hadoop/ozone/om/TestTrashService.java | 137 ------------------ .../ozone/client/ClientProtocolStub.java | 6 +- 14 files changed, 50 insertions(+), 297 deletions(-) delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 68812a7eb4d6..c747657b9ef5 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -527,7 +527,10 @@ List listKeys(String volumeName, String bucketName, * the cluster level set by admins. * @return The list of keys that are deleted from the deleted table. * @throws IOException + * @deprecated HDDS-3367 implements a new way to store trashed files. + * Please avoid using this as it will be removed in a future release */ + @Deprecated List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 8445f9e6954c..37d50e52fc3a 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1762,20 +1762,15 @@ public List listKeys(String volumeName, String bucketName, @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - - return ozoneManagerClient.listTrash(volumeName, bucketName, startKeyName, - keyPrefix, maxKeys); + // listTrash is deprecated + throw new UnsupportedOperationException(); } @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - - return ozoneManagerClient.recoverTrash(volumeName, bucketName, keyName, - destinationBucket); + //recoverTrash is deprecated + throw new UnsupportedOperationException(); } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 82030669c93a..560c25752d6b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -242,7 +242,6 @@ public static boolean isReadOnly( case LookupKey: case ListKeys: case ListKeysLight: - case ListTrash: case ServiceList: case ListOpenFiles: case ListMultiPartUploadParts: @@ -302,7 +301,6 @@ public static boolean isReadOnly( case SetAcl: case AddAcl: case PurgeKeys: - case RecoverTrash: case FinalizeUpgrade: case Prepare: case CancelPrepare: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 5f592663dad5..9a37a54f8e6c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -1068,7 +1068,10 @@ DBUpdates getDBUpdates( * the cluster level set by admins. * @return The list of keys that are deleted from the deleted table. * @throws IOException + * @deprecated HDDS-3367 implements a new way to store trashed files. + * Please avoid using this as it will be removed in a future release */ + @Deprecated List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException; @@ -1081,7 +1084,10 @@ List listTrash(String volumeName, String bucketName, * @param destinationBucket - The bucket user want to recover to. * @return The result of recovering operation is success or not. * @throws IOException + * @deprecated HDDS-3367 implements a new way to store trashed files. + * Please avoid using this as it will be removed in a future release */ + @Deprecated default boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { return false; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 9a965b7c3d06..73b152253779 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -149,8 +149,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileRequest; @@ -181,8 +179,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RefetchSecretKeyRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RefetchSecretKeyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclRequest; @@ -2443,79 +2439,15 @@ public List listStatus(OmKeyArgs args, boolean recursive, public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - - Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName), - "The volume name cannot be null or " + - "empty. Please enter a valid volume name or use '*' as a wild card"); - - Preconditions.checkArgument(Strings.isNullOrEmpty(bucketName), - "The bucket name cannot be null or " + - "empty. Please enter a valid bucket name or use '*' as a wild card"); - - ListTrashRequest trashRequest = ListTrashRequest.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setStartKeyName(startKeyName) - .setKeyPrefix(keyPrefix) - .setMaxKeys(maxKeys) - .build(); - - OMRequest omRequest = createOMRequest(Type.ListTrash) - .setListTrashRequest(trashRequest) - .build(); - - ListTrashResponse trashResponse = - handleError(submitRequest(omRequest)).getListTrashResponse(); - - List deletedKeyList = - new ArrayList<>(trashResponse.getDeletedKeysCount()); - - List list = new ArrayList<>(); - for (OzoneManagerProtocolProtos.RepeatedKeyInfo - repeatedKeyInfo : trashResponse.getDeletedKeysList()) { - RepeatedOmKeyInfo fromProto = - RepeatedOmKeyInfo.getFromProto(repeatedKeyInfo); - list.add(fromProto); - } - deletedKeyList.addAll(list); - - return deletedKeyList; + // listTrash is deprecated + throw new UnsupportedOperationException(); } @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - - Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName), - "The volume name cannot be null or empty. " + - "Please enter a valid volume name."); - - Preconditions.checkArgument(Strings.isNullOrEmpty(bucketName), - "The bucket name cannot be null or empty. " + - "Please enter a valid bucket name."); - - Preconditions.checkArgument(Strings.isNullOrEmpty(keyName), - "The key name cannot be null or empty. " + - "Please enter a valid key name."); - - Preconditions.checkArgument(Strings.isNullOrEmpty(destinationBucket), - "The destination bucket name cannot be null or empty. " + - "Please enter a valid destination bucket name."); - - RecoverTrashRequest.Builder req = RecoverTrashRequest.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDestinationBucket(destinationBucket); - - OMRequest omRequest = createOMRequest(Type.RecoverTrash) - .setRecoverTrashRequest(req) - .build(); - - RecoverTrashResponse recoverResponse = - handleError(submitRequest(omRequest)).getRecoverTrashResponse(); - - return recoverResponse.getResponse(); + //recoverTrash is deprecated + throw new UnsupportedOperationException(); } @Override diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 32bba266080a..e9ae77957e03 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -97,8 +97,8 @@ enum Type { ListMultipartUploads = 82; - ListTrash = 91; - RecoverTrash = 92; + ListTrash = 91 [deprecated = true]; + RecoverTrash = 92 [deprecated = true]; RevokeS3Secret = 93; @@ -232,8 +232,8 @@ message OMRequest { optional UpdateGetS3SecretRequest updateGetS3SecretRequest = 82; optional ListMultipartUploadsRequest listMultipartUploadsRequest = 83; - optional ListTrashRequest listTrashRequest = 91; - optional RecoverTrashRequest RecoverTrashRequest = 92; + optional ListTrashRequest listTrashRequest = 91 [deprecated = true]; + optional RecoverTrashRequest RecoverTrashRequest = 92 [deprecated = true]; optional RevokeS3SecretRequest RevokeS3SecretRequest = 93; @@ -360,8 +360,8 @@ message OMResponse { optional ListMultipartUploadsResponse listMultipartUploadsResponse = 82; - optional ListTrashResponse listTrashResponse = 91; - optional RecoverTrashResponse RecoverTrashResponse = 92; + optional ListTrashResponse listTrashResponse = 91 [deprecated = true]; + optional RecoverTrashResponse RecoverTrashResponse = 92 [deprecated = true]; optional PurgePathsResponse purgePathsResponse = 93 [deprecated = true]; optional PurgeDirectoriesResponse purgeDirectoriesResponse = 108; @@ -547,6 +547,7 @@ enum Status { in the deleted table on Ozone Manager. */ message ListTrashRequest { + option deprecated = true; required string volumeName = 1; required string bucketName = 2; optional string startKeyName = 3; @@ -555,6 +556,7 @@ message ListTrashRequest { } message ListTrashResponse { + option deprecated = true; repeated RepeatedKeyInfo deletedKeys = 1; } @@ -564,6 +566,7 @@ message ListTrashResponse { */ message RecoverTrashRequest { + option deprecated = true; required string volumeName = 1; required string bucketName = 2; required string keyName = 3; @@ -571,6 +574,7 @@ message RecoverTrashRequest { } message RecoverTrashResponse { + option deprecated = true; required bool response = 1; } diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index fb34d19a8bb1..8b57d1244c42 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -276,7 +276,10 @@ ListKeysResult listKeys(String volumeName, * the cluster level set by admins. * @return The list of keys that are deleted from the deleted table. * @throws IOException + * @deprecated HDDS-3367 implements a new way to store trashed files. + * Please avoid using this as it will be removed in a future release */ + @Deprecated List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException; @@ -312,7 +315,10 @@ ListSnapshotResponse listSnapshot( * @param keyName - The key user want to recover. * @param destinationBucket - The bucket user want to recover to. * @return The result of recovering operation is success or not. + * @deprecated HDDS-3367 implements a new way to store trashed files. + * Please avoid using this as it will be removed in a future release */ + @Deprecated boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 7a3312c0685a..e562c42208f7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -121,7 +121,10 @@ ListKeysResult listKeys(String volumeName, String bucketName, String startKey, * the cluster level set by admins. * @return The list of keys that are deleted from the deleted table. * @throws IOException + * @deprecated HDDS-3367 implements a new way to store trashed files. + * Please avoid using this as it will be removed in a future release */ + @Deprecated List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 2cb55135294a..b8098cf5a889 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -664,15 +664,8 @@ public ListKeysResult listKeys(String volumeName, String bucketName, public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - Preconditions.checkArgument(maxKeys <= listTrashKeysMax, - "The max keys limit specified is not less than the cluster " + - "allowed maximum limit."); - - return metadataManager.listTrash(volumeName, bucketName, - startKeyName, keyPrefix, maxKeys); + // listTrash is deprecated + throw new UnsupportedOperationException(); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 22d2b1e50b2a..252d9ce64f8f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1389,9 +1389,8 @@ public ListKeysResult listKeys(String volumeName, String bucketName, @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - - List deletedKeys = new ArrayList<>(); - return deletedKeys; + // listTrash is deprecated + throw new UnsupportedOperationException(); } @Override @@ -1473,13 +1472,8 @@ public ListSnapshotResponse listSnapshot( @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - - /* TODO: HDDS-2425 and HDDS-2426 - core logic stub would be added in later patch. - */ - - boolean recoverOperation = true; - return recoverOperation; + //recoverTrash is deprecated + throw new UnsupportedOperationException(); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 03896a042ec5..a4eda025f8b3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2946,33 +2946,8 @@ public ListKeysLightResult listKeysLight(String volumeName, public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - boolean auditSuccess = true; - Map auditMap = buildAuditMap(volumeName); - auditMap.put(OzoneConsts.BUCKET, bucketName); - auditMap.put(OzoneConsts.START_KEY, startKeyName); - auditMap.put(OzoneConsts.KEY_PREFIX, keyPrefix); - auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys)); - try { - if (isAclEnabled) { - omMetadataReader.checkAcls(ResourceType.BUCKET, - StoreType.OZONE, ACLType.LIST, - volumeName, bucketName, keyPrefix); - } - metrics.incNumTrashKeyLists(); - return keyManager.listTrash(volumeName, bucketName, - startKeyName, keyPrefix, maxKeys); - } catch (IOException ex) { - metrics.incNumTrashKeyListFails(); - auditSuccess = false; - AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_TRASH, - auditMap, ex)); - throw ex; - } finally { - if (auditSuccess) { - AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_TRASH, - auditMap)); - } - } + // listTrash is deprecated + throw new UnsupportedOperationException(); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index eb9410501fa0..816df4ae9fcc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -113,8 +113,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysLightResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupKeyRequest; @@ -237,11 +235,6 @@ public OMResponse handleReadRequest(OMRequest request) { request.getListKeysRequest()); responseBuilder.setListKeysLightResponse(listKeysLightResponse); break; - case ListTrash: - ListTrashResponse listTrashResponse = listTrash( - request.getListTrashRequest(), request.getVersion()); - responseBuilder.setListTrashResponse(listTrashResponse); - break; case ListMultiPartUploadParts: MultipartUploadListPartsResponse listPartsResponse = listParts(request.getListMultipartUploadPartsRequest()); @@ -830,22 +823,8 @@ public static OMResponse disallowListKeysWithBucketLayout( private ListTrashResponse listTrash(ListTrashRequest request, int clientVersion) throws IOException { - - ListTrashResponse.Builder resp = - ListTrashResponse.newBuilder(); - - List deletedKeys = impl.listTrash( - request.getVolumeName(), - request.getBucketName(), - request.getStartKeyName(), - request.getKeyPrefix(), - request.getMaxKeys()); - - for (RepeatedOmKeyInfo key: deletedKeys) { - resp.addDeletedKeys(key.getProto(false, clientVersion)); - } - - return resp.build(); + // listTrash is deprecated + throw new UnsupportedOperationException(); } @RequestFeatureValidator( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java deleted file mode 100644 index 4f0c15f15e53..000000000000 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.ozone.om; - - -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.server.ServerUtils; -import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.helpers.OpenKeySession; -import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.ratis.util.ExitUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; - -/** - * Test Key Trash Service. - *

- * This test does the things including: - * 1. UTs for list trash. - * 2. UTs for recover trash. - * 3. UTs for empty trash. - *

- */ -public class TestTrashService { - - @TempDir - private Path tempFolder; - - private KeyManager keyManager; - private OzoneManagerProtocol writeClient; - private OzoneManager om; - private String volumeName; - private String bucketName; - - @BeforeEach - void setup() throws Exception { - ExitUtils.disableSystemExit(); - OzoneConfiguration configuration = new OzoneConfiguration(); - - File folder = tempFolder.toFile(); - if (!folder.exists()) { - assertTrue(folder.mkdirs()); - } - System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); - ServerUtils.setOzoneMetaDirPath(configuration, folder.toString()); - - OmTestManagers omTestManagers - = new OmTestManagers(configuration); - keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); - volumeName = "volume"; - bucketName = "bucket"; - } - - @AfterEach - public void cleanup() throws Exception { - om.stop(); - } - - @Test - public void testRecoverTrash() throws IOException { - String keyName = "testKey"; - String destinationBucket = "destBucket"; - createAndDeleteKey(keyName); - - boolean recoverOperation = keyManager.getMetadataManager() - .recoverTrash(volumeName, bucketName, keyName, destinationBucket); - assertTrue(recoverOperation); - } - - private void createAndDeleteKey(String keyName) throws IOException { - - OMRequestTestUtils.addVolumeToOM(keyManager.getMetadataManager(), - OmVolumeArgs.newBuilder() - .setOwnerName("owner") - .setAdminName("admin") - .setVolume(volumeName) - .build()); - - OMRequestTestUtils.addBucketToOM(keyManager.getMetadataManager(), - OmBucketInfo.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .build()); - - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setAcls(Collections.emptyList()) - .setLocationInfoList(new ArrayList<>()) - .setReplicationConfig(StandaloneReplicationConfig - .getInstance(HddsProtos.ReplicationFactor.ONE)) - .setOwnerName(UserGroupInformation.getCurrentUser().getShortUserName()) - .build(); - - /* Create and delete key in the Key Manager. */ - OpenKeySession session = writeClient.openKey(keyArgs); - writeClient.commitKey(keyArgs, session.getId()); - writeClient.deleteKey(keyArgs); - } - -} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 21c3f8358f43..d236968e9086 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -305,14 +305,16 @@ public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - return null; + // listTrash is deprecated + throw new UnsupportedOperationException(); } @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - return false; + // recoverTrash is deprecated + throw new UnsupportedOperationException(); } @Override From 2be1dfec4a9af7833a63fc045391ae7f6de69f5c Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Mon, 12 Aug 2024 18:10:53 +0530 Subject: [PATCH 02/16] Fixed .proto deprecation for v2 --- .../src/main/proto/OmClientProtocol.proto | 34 +++++++++---------- .../OzoneManagerRequestHandler.java | 2 ++ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index e9ae77957e03..f22444f1e3c1 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -97,8 +97,8 @@ enum Type { ListMultipartUploads = 82; - ListTrash = 91 [deprecated = true]; - RecoverTrash = 92 [deprecated = true]; + ListTrash = 91; + RecoverTrash = 92; RevokeS3Secret = 93; @@ -547,17 +547,17 @@ enum Status { in the deleted table on Ozone Manager. */ message ListTrashRequest { - option deprecated = true; - required string volumeName = 1; - required string bucketName = 2; - optional string startKeyName = 3; - optional string keyPrefix = 4; - optional int32 maxKeys = 5; + // option deprecated = true; + required string volumeName = 1 [deprecated = true]; + required string bucketName = 2 [deprecated = true]; + optional string startKeyName = 3 [deprecated = true]; + optional string keyPrefix = 4 [deprecated = true]; + optional int32 maxKeys = 5 [deprecated = true]; } message ListTrashResponse { - option deprecated = true; - repeated RepeatedKeyInfo deletedKeys = 1; + // option deprecated = true; + repeated RepeatedKeyInfo deletedKeys = 1 [deprecated = true]; } /** @@ -566,16 +566,16 @@ message ListTrashResponse { */ message RecoverTrashRequest { - option deprecated = true; - required string volumeName = 1; - required string bucketName = 2; - required string keyName = 3; - required string destinationBucket = 4; + // option deprecated = true; + required string volumeName = 1 [deprecated = true]; + required string bucketName = 2 [deprecated = true]; + required string keyName = 3 [deprecated = true]; + required string destinationBucket = 4 [deprecated = true]; } message RecoverTrashResponse { - option deprecated = true; - required bool response = 1; + // option deprecated = true; + required bool response = 1 [deprecated = true]; } message VolumeInfo { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 816df4ae9fcc..cb15d7c1e940 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -113,6 +113,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysLightResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupKeyRequest; From b3f38f43f5ea7b0e7da981e1b7f5bb9e72da4a67 Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Tue, 13 Aug 2024 10:58:25 +0530 Subject: [PATCH 03/16] Removed unused import --- .../hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index cb15d7c1e940..f596ea28c8c4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -63,7 +63,6 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; From 86aafd5da9ab3a86af0041d73edba8117ce3e4ef Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Tue, 13 Aug 2024 12:22:51 +0530 Subject: [PATCH 04/16] Marked recoverTrash as deprecated in ClientProtocol --- .../apache/hadoop/ozone/client/protocol/ClientProtocol.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index c747657b9ef5..2640dcb795c6 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -545,7 +545,10 @@ List listTrash(String volumeName, String bucketName, * @param destinationBucket - The bucket user want to recover to. * @return The result of recovering operation is success or not. * @throws IOException + * @deprecated HDDS-3367 implements a new way to store trashed files. + * Please avoid using this as it will be removed in a future release */ + @Deprecated boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException; From 6d9118c77500ae8ff194541b06cb81ef9928766a Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Sat, 17 Aug 2024 12:02:14 +0530 Subject: [PATCH 05/16] Restore files and mark as deprecated --- .../hadoop/ozone/client/rpc/RpcClient.java | 15 +- ...ManagerProtocolClientSideTranslatorPB.java | 76 +++++++++- .../hadoop/ozone/om/KeyManagerImpl.java | 13 +- .../ozone/om/OmMetadataManagerImpl.java | 16 +- .../apache/hadoop/ozone/om/OzoneManager.java | 30 +++- .../OzoneManagerRequestHandler.java | 25 +++- .../hadoop/ozone/om/TestTrashService.java | 137 ++++++++++++++++++ .../ozone/client/ClientProtocolStub.java | 9 +- 8 files changed, 299 insertions(+), 22 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 37d50e52fc3a..bef97ca84bf4 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1759,18 +1759,25 @@ public List listKeys(String volumeName, String bucketName, } } + @Deprecated @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - // listTrash is deprecated - throw new UnsupportedOperationException(); + + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + + return ozoneManagerClient.listTrash(volumeName, bucketName, startKeyName, + keyPrefix, maxKeys); } + @Deprecated @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - //recoverTrash is deprecated - throw new UnsupportedOperationException(); + + return ozoneManagerClient.recoverTrash(volumeName, bucketName, keyName, + destinationBucket); } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 73b152253779..5ee09015ef79 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -149,6 +149,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileRequest; @@ -179,6 +181,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RefetchSecretKeyRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RefetchSecretKeyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclRequest; @@ -2435,19 +2439,83 @@ public List listStatus(OmKeyArgs args, boolean recursive, return listStatus(args, recursive, startKey, numEntries, false); } + @Deprecated @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - // listTrash is deprecated - throw new UnsupportedOperationException(); + + Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName), + "The volume name cannot be null or " + + "empty. Please enter a valid volume name or use '*' as a wild card"); + + Preconditions.checkArgument(Strings.isNullOrEmpty(bucketName), + "The bucket name cannot be null or " + + "empty. Please enter a valid bucket name or use '*' as a wild card"); + + ListTrashRequest trashRequest = ListTrashRequest.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setStartKeyName(startKeyName) + .setKeyPrefix(keyPrefix) + .setMaxKeys(maxKeys) + .build(); + + OMRequest omRequest = createOMRequest(Type.ListTrash) + .setListTrashRequest(trashRequest) + .build(); + + ListTrashResponse trashResponse = + handleError(submitRequest(omRequest)).getListTrashResponse(); + + List deletedKeyList = + new ArrayList<>(trashResponse.getDeletedKeysCount()); + + List list = new ArrayList<>(); + for (OzoneManagerProtocolProtos.RepeatedKeyInfo + repeatedKeyInfo : trashResponse.getDeletedKeysList()) { + RepeatedOmKeyInfo fromProto = + RepeatedOmKeyInfo.getFromProto(repeatedKeyInfo); + list.add(fromProto); + } + deletedKeyList.addAll(list); } + @Deprecated @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - //recoverTrash is deprecated - throw new UnsupportedOperationException(); + + Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName), + "The volume name cannot be null or empty. " + + "Please enter a valid volume name."); + + Preconditions.checkArgument(Strings.isNullOrEmpty(bucketName), + "The bucket name cannot be null or empty. " + + "Please enter a valid bucket name."); + + Preconditions.checkArgument(Strings.isNullOrEmpty(keyName), + "The key name cannot be null or empty. " + + "Please enter a valid key name."); + + Preconditions.checkArgument(Strings.isNullOrEmpty(destinationBucket), + "The destination bucket name cannot be null or empty. " + + "Please enter a valid destination bucket name."); + + RecoverTrashRequest.Builder req = RecoverTrashRequest.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDestinationBucket(destinationBucket); + + OMRequest omRequest = createOMRequest(Type.RecoverTrash) + .setRecoverTrashRequest(req) + .build(); + + RecoverTrashResponse recoverResponse = + handleError(submitRequest(omRequest)).getRecoverTrashResponse(); + + return recoverResponse.getResponse(); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index b8098cf5a889..97c865737ea3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -660,12 +660,21 @@ public ListKeysResult listKeys(String volumeName, String bucketName, return listKeysResult; } + @Deprecated @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - // listTrash is deprecated - throw new UnsupportedOperationException(); + + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + Preconditions.checkArgument(maxKeys <= listTrashKeysMax, + "The max keys limit specified is not less than the cluster " + + "allowed maximum limit."); + + return metadataManager.listTrash(volumeName, bucketName, + startKeyName, keyPrefix, maxKeys); + } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 252d9ce64f8f..ce2b18d82a27 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1386,11 +1386,13 @@ public ListKeysResult listKeys(String volumeName, String bucketName, } // TODO: HDDS-2419 - Complete stub below for core logic + @Deprecated @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - // listTrash is deprecated - throw new UnsupportedOperationException(); + + List deletedKeys = new ArrayList<>(); + return deletedKeys; } @Override @@ -1469,11 +1471,17 @@ public ListSnapshotResponse listSnapshot( return new ListSnapshotResponse(snapshotInfos, lastSnapshot); } + @Deprecated @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - //recoverTrash is deprecated - throw new UnsupportedOperationException(); + + /* TODO: HDDS-2425 and HDDS-2426 + core logic stub would be added in later patch. + */ + + boolean recoverOperation = true; + return recoverOperation; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index a4eda025f8b3..63b7283c3b07 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2942,12 +2942,38 @@ public ListKeysLightResult listKeysLight(String volumeName, return new ListKeysLightResult(basicKeysList, listKeysResult.isTruncated()); } + @Deprecated @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - // listTrash is deprecated - throw new UnsupportedOperationException(); + boolean auditSuccess = true; + Map auditMap = buildAuditMap(volumeName); + auditMap.put(OzoneConsts.BUCKET, bucketName); + auditMap.put(OzoneConsts.START_KEY, startKeyName); + auditMap.put(OzoneConsts.KEY_PREFIX, keyPrefix); + auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys)); + try { + if (isAclEnabled) { + omMetadataReader.checkAcls(ResourceType.BUCKET, + StoreType.OZONE, ACLType.LIST, + volumeName, bucketName, keyPrefix); + } + metrics.incNumTrashKeyLists(); + return keyManager.listTrash(volumeName, bucketName, + startKeyName, keyPrefix, maxKeys); + } catch (IOException ex) { + metrics.incNumTrashKeyListFails(); + auditSuccess = false; + AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_TRASH, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_TRASH, + auditMap)); + } + } } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index f596ea28c8c4..04e947f03b83 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -63,6 +63,7 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; @@ -236,6 +237,11 @@ public OMResponse handleReadRequest(OMRequest request) { request.getListKeysRequest()); responseBuilder.setListKeysLightResponse(listKeysLightResponse); break; + case ListTrash: + ListTrashResponse listTrashResponse = listTrash( + request.getListTrashRequest(), request.getVersion()); + responseBuilder.setListTrashResponse(listTrashResponse); + break; case ListMultiPartUploadParts: MultipartUploadListPartsResponse listPartsResponse = listParts(request.getListMultipartUploadPartsRequest()); @@ -822,10 +828,25 @@ public static OMResponse disallowListKeysWithBucketLayout( return resp; } + @Deprecated private ListTrashResponse listTrash(ListTrashRequest request, int clientVersion) throws IOException { - // listTrash is deprecated - throw new UnsupportedOperationException(); + + ListTrashResponse.Builder resp = + ListTrashResponse.newBuilder(); + + List deletedKeys = impl.listTrash( + request.getVolumeName(), + request.getBucketName(), + request.getStartKeyName(), + request.getKeyPrefix(), + request.getMaxKeys()); + + for (RepeatedOmKeyInfo key: deletedKeys) { + resp.addDeletedKeys(key.getProto(false, clientVersion)); + } + + return resp.build(); } @RequestFeatureValidator( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java new file mode 100644 index 000000000000..4f0c15f15e53 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om; + + +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OpenKeySession; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ratis.util.ExitUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; + +/** + * Test Key Trash Service. + *

+ * This test does the things including: + * 1. UTs for list trash. + * 2. UTs for recover trash. + * 3. UTs for empty trash. + *

+ */ +public class TestTrashService { + + @TempDir + private Path tempFolder; + + private KeyManager keyManager; + private OzoneManagerProtocol writeClient; + private OzoneManager om; + private String volumeName; + private String bucketName; + + @BeforeEach + void setup() throws Exception { + ExitUtils.disableSystemExit(); + OzoneConfiguration configuration = new OzoneConfiguration(); + + File folder = tempFolder.toFile(); + if (!folder.exists()) { + assertTrue(folder.mkdirs()); + } + System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); + ServerUtils.setOzoneMetaDirPath(configuration, folder.toString()); + + OmTestManagers omTestManagers + = new OmTestManagers(configuration); + keyManager = omTestManagers.getKeyManager(); + writeClient = omTestManagers.getWriteClient(); + om = omTestManagers.getOzoneManager(); + volumeName = "volume"; + bucketName = "bucket"; + } + + @AfterEach + public void cleanup() throws Exception { + om.stop(); + } + + @Test + public void testRecoverTrash() throws IOException { + String keyName = "testKey"; + String destinationBucket = "destBucket"; + createAndDeleteKey(keyName); + + boolean recoverOperation = keyManager.getMetadataManager() + .recoverTrash(volumeName, bucketName, keyName, destinationBucket); + assertTrue(recoverOperation); + } + + private void createAndDeleteKey(String keyName) throws IOException { + + OMRequestTestUtils.addVolumeToOM(keyManager.getMetadataManager(), + OmVolumeArgs.newBuilder() + .setOwnerName("owner") + .setAdminName("admin") + .setVolume(volumeName) + .build()); + + OMRequestTestUtils.addBucketToOM(keyManager.getMetadataManager(), + OmBucketInfo.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .build()); + + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setAcls(Collections.emptyList()) + .setLocationInfoList(new ArrayList<>()) + .setReplicationConfig(StandaloneReplicationConfig + .getInstance(HddsProtos.ReplicationFactor.ONE)) + .setOwnerName(UserGroupInformation.getCurrentUser().getShortUserName()) + .build(); + + /* Create and delete key in the Key Manager. */ + OpenKeySession session = writeClient.openKey(keyArgs); + writeClient.commitKey(keyArgs, session.getId()); + writeClient.deleteKey(keyArgs); + } + +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index d236968e9086..dc0cfe3ecac5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -300,21 +300,22 @@ public List listKeys(String volumeName, String bucketName, return null; } + @Deprecated @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - // listTrash is deprecated - throw new UnsupportedOperationException(); + + return null; } + @Deprecated @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - // recoverTrash is deprecated - throw new UnsupportedOperationException(); + return false; } @Override From b645e5448bf06a866dfff844e640a6bd64053bf9 Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Sat, 17 Aug 2024 12:05:38 +0530 Subject: [PATCH 06/16] Fixed blank spaces --- .../java/org/apache/hadoop/ozone/client/rpc/RpcClient.java | 6 +++--- .../OzoneManagerProtocolClientSideTranslatorPB.java | 4 ++-- .../java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 3 +-- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 4 ++-- .../hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index bef97ca84bf4..17ca54e83a51 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1763,7 +1763,7 @@ public List listKeys(String volumeName, String bucketName, @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - + Preconditions.checkNotNull(volumeName); Preconditions.checkNotNull(bucketName); @@ -1775,9 +1775,9 @@ public List listTrash(String volumeName, String bucketName, @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - + return ozoneManagerClient.recoverTrash(volumeName, bucketName, keyName, - destinationBucket); + destinationBucket); } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 5ee09015ef79..be39ded5bc20 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -2444,7 +2444,7 @@ public List listStatus(OmKeyArgs args, boolean recursive, public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - + Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName), "The volume name cannot be null or " + "empty. Please enter a valid volume name or use '*' as a wild card"); @@ -2485,7 +2485,7 @@ public List listTrash(String volumeName, @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - + Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName), "The volume name cannot be null or empty. " + "Please enter a valid volume name."); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 97c865737ea3..0915a0365ee0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -665,7 +665,7 @@ public ListKeysResult listKeys(String volumeName, String bucketName, public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - + Preconditions.checkNotNull(volumeName); Preconditions.checkNotNull(bucketName); Preconditions.checkArgument(maxKeys <= listTrashKeysMax, @@ -674,7 +674,6 @@ public List listTrash(String volumeName, return metadataManager.listTrash(volumeName, bucketName, startKeyName, keyPrefix, maxKeys); - } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index ce2b18d82a27..e19a156bb93f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1390,7 +1390,7 @@ public ListKeysResult listKeys(String volumeName, String bucketName, @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException { - + List deletedKeys = new ArrayList<>(); return deletedKeys; } @@ -1475,7 +1475,7 @@ public ListSnapshotResponse listSnapshot( @Override public boolean recoverTrash(String volumeName, String bucketName, String keyName, String destinationBucket) throws IOException { - + /* TODO: HDDS-2425 and HDDS-2426 core logic stub would be added in later patch. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 04e947f03b83..001b3de56fa9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -831,7 +831,7 @@ public static OMResponse disallowListKeysWithBucketLayout( @Deprecated private ListTrashResponse listTrash(ListTrashRequest request, int clientVersion) throws IOException { - + ListTrashResponse.Builder resp = ListTrashResponse.newBuilder(); From 35922e1f665c3e9a48c705a4114fec99826c8334 Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Sun, 18 Aug 2024 19:43:06 +0530 Subject: [PATCH 07/16] Removed listTrash and recoverTrash definitions --- .../apache/hadoop/ozone/OzoneConfigKeys.java | 4 - .../ozone/client/protocol/ClientProtocol.java | 39 ----- .../hadoop/ozone/client/rpc/RpcClient.java | 21 --- .../om/protocol/OzoneManagerProtocol.java | 39 ----- ...ManagerProtocolClientSideTranslatorPB.java | 83 ----------- .../apache/hadoop/ozone/om/TestOmMetrics.java | 8 - .../src/main/proto/OmClientProtocol.proto | 6 + .../hadoop/ozone/om/OMMetadataManager.java | 36 ----- .../apache/hadoop/ozone/om/KeyManager.java | 21 --- .../hadoop/ozone/om/KeyManagerImpl.java | 22 --- .../ozone/om/OmMetadataManagerImpl.java | 23 --- .../apache/hadoop/ozone/om/OzoneManager.java | 34 ----- .../OzoneManagerRequestHandler.java | 28 ---- .../hadoop/ozone/om/TestTrashService.java | 137 ------------------ .../ozone/client/ClientProtocolStub.java | 18 --- 15 files changed, 6 insertions(+), 513 deletions(-) delete mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 38ebc80b27e6..1f304fed8fad 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -535,10 +535,6 @@ public final class OzoneConfigKeys { public static final int OZONE_MANAGER_STRIPED_LOCK_SIZE_DEFAULT = 512; - public static final String OZONE_CLIENT_LIST_TRASH_KEYS_MAX = - "ozone.client.list.trash.keys.max"; - public static final int OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT = 1000; - public static final String OZONE_HTTP_BASEDIR = "ozone.http.basedir"; public static final String OZONE_HTTP_POLICY_KEY = diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 2640dcb795c6..8ea9efefca96 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -513,45 +513,6 @@ List listKeys(String volumeName, String bucketName, String keyPrefix, String prevKey, int maxListResult) throws IOException; - /** - * List trash allows the user to list the keys that were marked as deleted, - * but not actually deleted by Ozone Manager. This allows a user to recover - * keys within a configurable window. - * @param volumeName - The volume name, which can also be a wild card - * using '*'. - * @param bucketName - The bucket name, which can also be a wild card - * using '*'. - * @param startKeyName - List keys from a specific key name. - * @param keyPrefix - List keys using a specific prefix. - * @param maxKeys - The number of keys to be returned. This must be below - * the cluster level set by admins. - * @return The list of keys that are deleted from the deleted table. - * @throws IOException - * @deprecated HDDS-3367 implements a new way to store trashed files. - * Please avoid using this as it will be removed in a future release - */ - @Deprecated - List listTrash(String volumeName, String bucketName, - String startKeyName, String keyPrefix, - int maxKeys) - throws IOException; - - /** - * Recover trash allows the user to recover keys that were marked as deleted, - * but not actually deleted by Ozone Manager. - * @param volumeName - The volume name. - * @param bucketName - The bucket name. - * @param keyName - The key user want to recover. - * @param destinationBucket - The bucket user want to recover to. - * @return The result of recovering operation is success or not. - * @throws IOException - * @deprecated HDDS-3367 implements a new way to store trashed files. - * Please avoid using this as it will be removed in a future release - */ - @Deprecated - boolean recoverTrash(String volumeName, String bucketName, String keyName, - String destinationBucket) throws IOException; - /** * Get OzoneKey. * @param volumeName Name of the Volume diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 17ca54e83a51..8f64ad5320e0 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1759,27 +1759,6 @@ public List listKeys(String volumeName, String bucketName, } } - @Deprecated - @Override - public List listTrash(String volumeName, String bucketName, - String startKeyName, String keyPrefix, int maxKeys) throws IOException { - - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - - return ozoneManagerClient.listTrash(volumeName, bucketName, startKeyName, - keyPrefix, maxKeys); - } - - @Deprecated - @Override - public boolean recoverTrash(String volumeName, String bucketName, - String keyName, String destinationBucket) throws IOException { - - return ozoneManagerClient.recoverTrash(volumeName, bucketName, keyName, - destinationBucket); - } - @Override public OzoneKeyDetails getKeyDetails( String volumeName, String bucketName, String keyName) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 9a37a54f8e6c..01ee3a89fd1c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -1054,45 +1054,6 @@ DBUpdates getDBUpdates( OzoneManagerProtocolProtos.DBUpdatesRequest dbUpdatesRequest) throws IOException; - /** - * List trash allows the user to list the keys that were marked as deleted, - * but not actually deleted by Ozone Manager. This allows a user to recover - * keys within a configurable window. - * @param volumeName - The volume name, which can also be a wild card - * using '*'. - * @param bucketName - The bucket name, which can also be a wild card - * using '*'. - * @param startKeyName - List keys from a specific key name. - * @param keyPrefix - List keys using a specific prefix. - * @param maxKeys - The number of keys to be returned. This must be below - * the cluster level set by admins. - * @return The list of keys that are deleted from the deleted table. - * @throws IOException - * @deprecated HDDS-3367 implements a new way to store trashed files. - * Please avoid using this as it will be removed in a future release - */ - @Deprecated - List listTrash(String volumeName, String bucketName, - String startKeyName, String keyPrefix, int maxKeys) throws IOException; - - /** - * Recover trash allows the user to recover keys that were marked as deleted, - * but not actually deleted by Ozone Manager. - * @param volumeName - The volume name. - * @param bucketName - The bucket name. - * @param keyName - The key user want to recover. - * @param destinationBucket - The bucket user want to recover to. - * @return The result of recovering operation is success or not. - * @throws IOException - * @deprecated HDDS-3367 implements a new way to store trashed files. - * Please avoid using this as it will be removed in a future release - */ - @Deprecated - default boolean recoverTrash(String volumeName, String bucketName, - String keyName, String destinationBucket) throws IOException { - return false; - } - /** * * @param txnApplyWaitTimeoutSeconds Max time in SECONDS to wait for all diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index be39ded5bc20..1b33b675ad23 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -149,8 +149,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileRequest; @@ -181,8 +179,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverTrashResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RefetchSecretKeyRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RefetchSecretKeyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclRequest; @@ -2439,85 +2435,6 @@ public List listStatus(OmKeyArgs args, boolean recursive, return listStatus(args, recursive, startKey, numEntries, false); } - @Deprecated - @Override - public List listTrash(String volumeName, - String bucketName, String startKeyName, String keyPrefix, int maxKeys) - throws IOException { - - Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName), - "The volume name cannot be null or " + - "empty. Please enter a valid volume name or use '*' as a wild card"); - - Preconditions.checkArgument(Strings.isNullOrEmpty(bucketName), - "The bucket name cannot be null or " + - "empty. Please enter a valid bucket name or use '*' as a wild card"); - - ListTrashRequest trashRequest = ListTrashRequest.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setStartKeyName(startKeyName) - .setKeyPrefix(keyPrefix) - .setMaxKeys(maxKeys) - .build(); - - OMRequest omRequest = createOMRequest(Type.ListTrash) - .setListTrashRequest(trashRequest) - .build(); - - ListTrashResponse trashResponse = - handleError(submitRequest(omRequest)).getListTrashResponse(); - - List deletedKeyList = - new ArrayList<>(trashResponse.getDeletedKeysCount()); - - List list = new ArrayList<>(); - for (OzoneManagerProtocolProtos.RepeatedKeyInfo - repeatedKeyInfo : trashResponse.getDeletedKeysList()) { - RepeatedOmKeyInfo fromProto = - RepeatedOmKeyInfo.getFromProto(repeatedKeyInfo); - list.add(fromProto); - } - deletedKeyList.addAll(list); - } - - @Deprecated - @Override - public boolean recoverTrash(String volumeName, String bucketName, - String keyName, String destinationBucket) throws IOException { - - Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName), - "The volume name cannot be null or empty. " + - "Please enter a valid volume name."); - - Preconditions.checkArgument(Strings.isNullOrEmpty(bucketName), - "The bucket name cannot be null or empty. " + - "Please enter a valid bucket name."); - - Preconditions.checkArgument(Strings.isNullOrEmpty(keyName), - "The key name cannot be null or empty. " + - "Please enter a valid key name."); - - Preconditions.checkArgument(Strings.isNullOrEmpty(destinationBucket), - "The destination bucket name cannot be null or empty. " + - "Please enter a valid destination bucket name."); - - RecoverTrashRequest.Builder req = RecoverTrashRequest.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDestinationBucket(destinationBucket); - - OMRequest omRequest = createOMRequest(Type.RecoverTrash) - .setRecoverTrashRequest(req) - .build(); - - RecoverTrashResponse recoverResponse = - handleError(submitRequest(omRequest)).getRecoverTrashResponse(); - - return recoverResponse.getResponse(); - } - @Override public long prepareOzoneManager( long txnApplyWaitTimeoutSeconds, long txnApplyCheckIntervalSeconds) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java index 4619af1baa29..3b680a780584 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java @@ -409,8 +409,6 @@ public void testKeyOps() throws Exception { doThrow(exception).when(mockKm).lookupKey(any(), any(), any()); doThrow(exception).when(mockKm).listKeys( any(), any(), any(), any(), anyInt()); - doThrow(exception).when(mockKm).listTrash( - any(), any(), any(), any(), anyInt()); OmMetadataReader omMetadataReader = (OmMetadataReader) ozoneManager.getOmMetadataReader().get(); HddsWhiteboxTestUtils.setInternalState( @@ -843,12 +841,6 @@ private void doKeyOps(OmKeyArgs keyArgs) { } catch (IOException ignored) { } - try { - ozoneManager.listTrash(keyArgs.getVolumeName(), - keyArgs.getBucketName(), null, null, 0); - } catch (IOException ignored) { - } - try { writeClient.deleteKey(keyArgs); } catch (IOException ignored) { diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index f22444f1e3c1..0c3212f63104 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -97,6 +97,7 @@ enum Type { ListMultipartUploads = 82; + // Not used anymore due to HDDS-11251 ListTrash = 91; RecoverTrash = 92; @@ -232,6 +233,7 @@ message OMRequest { optional UpdateGetS3SecretRequest updateGetS3SecretRequest = 82; optional ListMultipartUploadsRequest listMultipartUploadsRequest = 83; + // Not used anymore due to HDDS-11251 optional ListTrashRequest listTrashRequest = 91 [deprecated = true]; optional RecoverTrashRequest RecoverTrashRequest = 92 [deprecated = true]; @@ -360,8 +362,10 @@ message OMResponse { optional ListMultipartUploadsResponse listMultipartUploadsResponse = 82; + // Not used anymore due to HDDS-11251 optional ListTrashResponse listTrashResponse = 91 [deprecated = true]; optional RecoverTrashResponse RecoverTrashResponse = 92 [deprecated = true]; + optional PurgePathsResponse purgePathsResponse = 93 [deprecated = true]; optional PurgeDirectoriesResponse purgeDirectoriesResponse = 108; @@ -545,6 +549,7 @@ enum Status { /** This command acts as a list command for deleted keys that are still present in the deleted table on Ozone Manager. + Not used anymore due to HDDS-11251 */ message ListTrashRequest { // option deprecated = true; @@ -563,6 +568,7 @@ message ListTrashResponse { /** This command acts as a recover command for deleted keys that are still in deleted table on Ozone Manager. + Not used anymore due to HDDS-11251 */ message RecoverTrashRequest { diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 8b57d1244c42..cf0819ca527c 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -262,27 +262,6 @@ ListKeysResult listKeys(String volumeName, int maxKeys) throws IOException; - /** - * List trash allows the user to list the keys that were marked as deleted, - * but not actually deleted by Ozone Manager. This allows a user to recover - * keys within a configurable window. - * @param volumeName - The volume name, which can also be a wild card - * using '*'. - * @param bucketName - The bucket name, which can also be a wild card - * using '*'. - * @param startKeyName - List keys from a specific key name. - * @param keyPrefix - List keys using a specific prefix. - * @param maxKeys - The number of keys to be returned. This must be below - * the cluster level set by admins. - * @return The list of keys that are deleted from the deleted table. - * @throws IOException - * @deprecated HDDS-3367 implements a new way to store trashed files. - * Please avoid using this as it will be removed in a future release - */ - @Deprecated - List listTrash(String volumeName, String bucketName, - String startKeyName, String keyPrefix, int maxKeys) throws IOException; - /** * Returns snapshot info for volume/bucket snapshot path. * @param volumeName volume name @@ -307,21 +286,6 @@ ListSnapshotResponse listSnapshot( String volumeName, String bucketName, String snapshotPrefix, String prevSnapshot, int maxListResult) throws IOException; - /** - * Recover trash allows the user to recover the keys - * that were marked as deleted, but not actually deleted by Ozone Manager. - * @param volumeName - The volume name. - * @param bucketName - The bucket name. - * @param keyName - The key user want to recover. - * @param destinationBucket - The bucket user want to recover to. - * @return The result of recovering operation is success or not. - * @deprecated HDDS-3367 implements a new way to store trashed files. - * Please avoid using this as it will be removed in a future release - */ - @Deprecated - boolean recoverTrash(String volumeName, String bucketName, - String keyName, String destinationBucket) throws IOException; - /** * Returns a list of volumes owned by a given user; if user is null, returns * all volumes. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index e562c42208f7..6377d7f643ac 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -107,27 +107,6 @@ ListKeysResult listKeys(String volumeName, String bucketName, String startKey, String keyPrefix, int maxKeys) throws IOException; - /** - * List trash allows the user to list the keys that were marked as deleted, - * but not actually deleted by Ozone Manager. This allows a user to recover - * keys within a configurable window. - * @param volumeName - The volume name, which can also be a wild card - * using '*'. - * @param bucketName - The bucket name, which can also be a wild card - * using '*'. - * @param startKeyName - List keys from a specific key name. - * @param keyPrefix - List keys using a specific prefix. - * @param maxKeys - The number of keys to be returned. This must be below - * the cluster level set by admins. - * @return The list of keys that are deleted from the deleted table. - * @throws IOException - * @deprecated HDDS-3367 implements a new way to store trashed files. - * Please avoid using this as it will be removed in a future release - */ - @Deprecated - List listTrash(String volumeName, String bucketName, - String startKeyName, String keyPrefix, int maxKeys) throws IOException; - /** * Returns a PendingKeysDeletion. It has a list of pending deletion key info * that ups to the given count.Each entry is a {@link BlockGroup}, which diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 0915a0365ee0..db75f7565df9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -122,8 +122,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_TRASH_KEYS_MAX; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL; @@ -182,7 +180,6 @@ public class KeyManagerImpl implements KeyManager { private final ScmClient scmClient; private final OMMetadataManager metadataManager; private final long scmBlockSize; - private final int listTrashKeysMax; private final OzoneBlockTokenSecretManager secretManager; private final boolean grpcBlockTokenEnabled; @@ -218,9 +215,6 @@ public KeyManagerImpl(OzoneManager om, ScmClient scmClient, this.grpcBlockTokenEnabled = conf.getBoolean( HDDS_BLOCK_TOKEN_ENABLED, HDDS_BLOCK_TOKEN_ENABLED_DEFAULT); - this.listTrashKeysMax = conf.getInt( - OZONE_CLIENT_LIST_TRASH_KEYS_MAX, - OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT); this.enableFileSystemPaths = conf.getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); @@ -660,22 +654,6 @@ public ListKeysResult listKeys(String volumeName, String bucketName, return listKeysResult; } - @Deprecated - @Override - public List listTrash(String volumeName, - String bucketName, String startKeyName, String keyPrefix, - int maxKeys) throws IOException { - - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - Preconditions.checkArgument(maxKeys <= listTrashKeysMax, - "The max keys limit specified is not less than the cluster " + - "allowed maximum limit."); - - return metadataManager.listTrash(volumeName, bucketName, - startKeyName, keyPrefix, maxKeys); - } - @Override public PendingKeysDeletion getPendingDeletionKeys(final int count) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index e19a156bb93f..ee92dbc2fde9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1385,16 +1385,6 @@ public ListKeysResult listKeys(String volumeName, String bucketName, return new ListKeysResult(result, isTruncated); } - // TODO: HDDS-2419 - Complete stub below for core logic - @Deprecated - @Override - public List listTrash(String volumeName, String bucketName, - String startKeyName, String keyPrefix, int maxKeys) throws IOException { - - List deletedKeys = new ArrayList<>(); - return deletedKeys; - } - @Override public SnapshotInfo getSnapshotInfo(String volumeName, String bucketName, String snapshotName) throws IOException { @@ -1471,19 +1461,6 @@ public ListSnapshotResponse listSnapshot( return new ListSnapshotResponse(snapshotInfos, lastSnapshot); } - @Deprecated - @Override - public boolean recoverTrash(String volumeName, String bucketName, - String keyName, String destinationBucket) throws IOException { - - /* TODO: HDDS-2425 and HDDS-2426 - core logic stub would be added in later patch. - */ - - boolean recoverOperation = true; - return recoverOperation; - } - /** * @param userName volume owner, null for listing all volumes. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 63b7283c3b07..90c11212e171 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2942,40 +2942,6 @@ public ListKeysLightResult listKeysLight(String volumeName, return new ListKeysLightResult(basicKeysList, listKeysResult.isTruncated()); } - @Deprecated - @Override - public List listTrash(String volumeName, - String bucketName, String startKeyName, String keyPrefix, int maxKeys) - throws IOException { - boolean auditSuccess = true; - Map auditMap = buildAuditMap(volumeName); - auditMap.put(OzoneConsts.BUCKET, bucketName); - auditMap.put(OzoneConsts.START_KEY, startKeyName); - auditMap.put(OzoneConsts.KEY_PREFIX, keyPrefix); - auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys)); - try { - if (isAclEnabled) { - omMetadataReader.checkAcls(ResourceType.BUCKET, - StoreType.OZONE, ACLType.LIST, - volumeName, bucketName, keyPrefix); - } - metrics.incNumTrashKeyLists(); - return keyManager.listTrash(volumeName, bucketName, - startKeyName, keyPrefix, maxKeys); - } catch (IOException ex) { - metrics.incNumTrashKeyListFails(); - auditSuccess = false; - AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_TRASH, - auditMap, ex)); - throw ex; - } finally { - if (auditSuccess) { - AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_TRASH, - auditMap)); - } - } - } - @Override public SnapshotInfo getSnapshotInfo(String volumeName, String bucketName, String snapshotName) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 001b3de56fa9..ec0aadfb4db2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -113,8 +113,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysLightResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupKeyRequest; @@ -237,11 +235,6 @@ public OMResponse handleReadRequest(OMRequest request) { request.getListKeysRequest()); responseBuilder.setListKeysLightResponse(listKeysLightResponse); break; - case ListTrash: - ListTrashResponse listTrashResponse = listTrash( - request.getListTrashRequest(), request.getVersion()); - responseBuilder.setListTrashResponse(listTrashResponse); - break; case ListMultiPartUploadParts: MultipartUploadListPartsResponse listPartsResponse = listParts(request.getListMultipartUploadPartsRequest()); @@ -828,27 +821,6 @@ public static OMResponse disallowListKeysWithBucketLayout( return resp; } - @Deprecated - private ListTrashResponse listTrash(ListTrashRequest request, - int clientVersion) throws IOException { - - ListTrashResponse.Builder resp = - ListTrashResponse.newBuilder(); - - List deletedKeys = impl.listTrash( - request.getVolumeName(), - request.getBucketName(), - request.getStartKeyName(), - request.getKeyPrefix(), - request.getMaxKeys()); - - for (RepeatedOmKeyInfo key: deletedKeys) { - resp.addDeletedKeys(key.getProto(false, clientVersion)); - } - - return resp.build(); - } - @RequestFeatureValidator( conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java deleted file mode 100644 index 4f0c15f15e53..000000000000 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.ozone.om; - - -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.server.ServerUtils; -import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.helpers.OpenKeySession; -import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.ratis.util.ExitUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; - -/** - * Test Key Trash Service. - *

- * This test does the things including: - * 1. UTs for list trash. - * 2. UTs for recover trash. - * 3. UTs for empty trash. - *

- */ -public class TestTrashService { - - @TempDir - private Path tempFolder; - - private KeyManager keyManager; - private OzoneManagerProtocol writeClient; - private OzoneManager om; - private String volumeName; - private String bucketName; - - @BeforeEach - void setup() throws Exception { - ExitUtils.disableSystemExit(); - OzoneConfiguration configuration = new OzoneConfiguration(); - - File folder = tempFolder.toFile(); - if (!folder.exists()) { - assertTrue(folder.mkdirs()); - } - System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); - ServerUtils.setOzoneMetaDirPath(configuration, folder.toString()); - - OmTestManagers omTestManagers - = new OmTestManagers(configuration); - keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); - volumeName = "volume"; - bucketName = "bucket"; - } - - @AfterEach - public void cleanup() throws Exception { - om.stop(); - } - - @Test - public void testRecoverTrash() throws IOException { - String keyName = "testKey"; - String destinationBucket = "destBucket"; - createAndDeleteKey(keyName); - - boolean recoverOperation = keyManager.getMetadataManager() - .recoverTrash(volumeName, bucketName, keyName, destinationBucket); - assertTrue(recoverOperation); - } - - private void createAndDeleteKey(String keyName) throws IOException { - - OMRequestTestUtils.addVolumeToOM(keyManager.getMetadataManager(), - OmVolumeArgs.newBuilder() - .setOwnerName("owner") - .setAdminName("admin") - .setVolume(volumeName) - .build()); - - OMRequestTestUtils.addBucketToOM(keyManager.getMetadataManager(), - OmBucketInfo.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .build()); - - OmKeyArgs keyArgs = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setAcls(Collections.emptyList()) - .setLocationInfoList(new ArrayList<>()) - .setReplicationConfig(StandaloneReplicationConfig - .getInstance(HddsProtos.ReplicationFactor.ONE)) - .setOwnerName(UserGroupInformation.getCurrentUser().getShortUserName()) - .build(); - - /* Create and delete key in the Key Manager. */ - OpenKeySession session = writeClient.openKey(keyArgs); - writeClient.commitKey(keyArgs, session.getId()); - writeClient.deleteKey(keyArgs); - } - -} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index dc0cfe3ecac5..5ef35ab937ae 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -300,24 +300,6 @@ public List listKeys(String volumeName, String bucketName, return null; } - @Deprecated - @Override - public List listTrash(String volumeName, String bucketName, - String startKeyName, - String keyPrefix, int maxKeys) - throws IOException { - - return null; - } - - @Deprecated - @Override - public boolean recoverTrash(String volumeName, String bucketName, - String keyName, String destinationBucket) - throws IOException { - return false; - } - @Override public OzoneKeyDetails getKeyDetails(String volumeName, String bucketName, String keyName) throws IOException { From 4a46f709bd7c4cb7cb89c855db26592a5524bdbb Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Sun, 18 Aug 2024 22:57:49 +0530 Subject: [PATCH 08/16] Fixed unused imports --- .../org/apache/hadoop/ozone/client/protocol/ClientProtocol.java | 1 - .../main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java | 1 - .../apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java | 1 - .../protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java | 1 - .../src/main/java/org/apache/hadoop/ozone/om/KeyManager.java | 1 - .../src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 1 - .../src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 1 - .../hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java | 1 - .../java/org/apache/hadoop/ozone/client/ClientProtocolStub.java | 1 - 9 files changed, 9 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 8ea9efefca96..8eb1f1b35673 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -58,7 +58,6 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.TenantStateList; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 8f64ad5320e0..586fc855db41 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -121,7 +121,6 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 01ee3a89fd1c..c9cdd7a68518 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -55,7 +55,6 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 1b33b675ad23..4dabed1596b4 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -71,7 +71,6 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 6377d7f643ac..b7fa5d746fb0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -28,7 +28,6 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; import org.apache.hadoop.ozone.om.fs.OzoneManagerFS; import org.apache.hadoop.hdds.utils.BackgroundService; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.om.service.SnapshotDirectoryCleaningService; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index db75f7565df9..6d276d95284e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -85,7 +85,6 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 90c11212e171..7608affcdd90 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -169,7 +169,6 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index ec0aadfb4db2..e8c07210758a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -63,7 +63,6 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 5ef35ab937ae..3555eea7e468 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -42,7 +42,6 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; -import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.TenantStateList; From 8c97ac5c822ee87eedf504b6e95b13c446858382 Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Mon, 19 Aug 2024 11:55:12 +0530 Subject: [PATCH 09/16] Removed trash key property from ozone-default.xml --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 186b5aaded9f..b4870d3a0669 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3356,14 +3356,6 @@ unhealthy will each have their own limit. - - ozone.client.list.trash.keys.max - 1000 - OZONE, CLIENT - - The maximum number of keys to return for a list trash request. - - ozone.http.basedir From 21290b753520a1a943a655c7c7b0ebd8de81282d Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Mon, 19 Aug 2024 13:03:49 +0530 Subject: [PATCH 10/16] Added deprecated comment markers to listTrash and recoverTrash enum --- .../src/main/java/org/apache/hadoop/ozone/OmUtils.java | 10 ++++++++++ .../src/main/proto/OmClientProtocol.proto | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 560c25752d6b..76d0079bf6b5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -242,6 +242,11 @@ public static boolean isReadOnly( case LookupKey: case ListKeys: case ListKeysLight: + case ListTrash: + // This is deprecated by HDDS-11251. Keeping this in here + // As protobuf currently doesn't support deprecating enum fields + // TODO: Remove once migrated to proto3 and mark fields in proto + // as deprecated case ServiceList: case ListOpenFiles: case ListMultiPartUploadParts: @@ -301,6 +306,11 @@ public static boolean isReadOnly( case SetAcl: case AddAcl: case PurgeKeys: + case RecoverTrash: + // This is deprecated by HDDS-11251. Keeping this in here + // As protobuf currently doesn't support deprecating enum fields + // TODO: Remove once migrated to proto3 and mark fields in proto + // as deprecated case FinalizeUpgrade: case Prepare: case CancelPrepare: diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 0c3212f63104..8ae026115f5b 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -98,8 +98,8 @@ enum Type { ListMultipartUploads = 82; // Not used anymore due to HDDS-11251 - ListTrash = 91; - RecoverTrash = 92; + ListTrash = 91; // [deprecated = true] + RecoverTrash = 92; // [deprecated = true] RevokeS3Secret = 93; From aa5e8b6046d0208524ae49c16de227a2492e8c5f Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Wed, 21 Aug 2024 12:54:53 +0530 Subject: [PATCH 11/16] Removed trashKey and trashKeyFail metrics --- .../org/apache/hadoop/ozone/om/OMMetrics.java | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java index 1c0ec78cfb22..b85b29399e8b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java @@ -61,7 +61,6 @@ public class OMMetrics implements OmMetadataReaderMetrics { private @Metric MutableCounterLong numKeyDeletes; private @Metric MutableCounterLong numBucketLists; private @Metric MutableCounterLong numKeyLists; - private @Metric MutableCounterLong numTrashKeyLists; private @Metric MutableCounterLong numVolumeLists; private @Metric MutableCounterLong numKeyCommits; private @Metric MutableCounterLong numKeyHSyncs; @@ -120,7 +119,6 @@ public class OMMetrics implements OmMetadataReaderMetrics { private @Metric MutableCounterLong numKeyDeleteFails; private @Metric MutableCounterLong numBucketListFails; private @Metric MutableCounterLong numKeyListFails; - private @Metric MutableCounterLong numTrashKeyListFails; private @Metric MutableCounterLong numVolumeListFails; private @Metric MutableCounterLong numKeyCommitFails; private @Metric MutableCounterLong numBlockAllocationFails; @@ -420,11 +418,6 @@ public void incNumKeyLists() { numKeyLists.incr(); } - public void incNumTrashKeyLists() { - numKeyOps.incr(); - numTrashKeyLists.incr(); - } - public void incNumVolumeLists() { numVolumeOps.incr(); numVolumeLists.incr(); @@ -836,14 +829,6 @@ public void incNumKeyListFails() { numKeyListFails.incr(); } - public void incNumTrashKeyListFails() { - numTrashKeyListFails.incr(); - } - - public void incNumVolumeListFails() { - numVolumeListFails.incr(); - } - public void incNumGetServiceListFails() { numGetServiceListFails.incr(); } @@ -994,11 +979,6 @@ public long getNumKeyLists() { return numKeyLists.value(); } - @VisibleForTesting - public long getNumTrashKeyLists() { - return numTrashKeyLists.value(); - } - @VisibleForTesting public long getNumGetServiceLists() { return numGetServiceLists.value(); @@ -1099,11 +1079,6 @@ public long getNumKeyListFails() { return numKeyListFails.value(); } - @VisibleForTesting - public long getNumTrashKeyListFails() { - return numTrashKeyListFails.value(); - } - @VisibleForTesting public long getNumFSOps() { return numFSOps.value(); From 96529cc8a3dc09d4d946ca962e18d60621892f40 Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Wed, 21 Aug 2024 13:04:36 +0530 Subject: [PATCH 12/16] Addressed build failures --- .../src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java index b85b29399e8b..cbe5205c10ba 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java @@ -829,6 +829,10 @@ public void incNumKeyListFails() { numKeyListFails.incr(); } + public void incNumVolumeListFails() { + numVolumeListFails.incr(); + } + public void incNumGetServiceListFails() { numGetServiceListFails.incr(); } From 76c77cbef94cb76e08cc87ce343aba59d6b7558c Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Wed, 21 Aug 2024 13:04:36 +0530 Subject: [PATCH 13/16] Addressed build failures --- .../test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java index 3b680a780584..5ddf554ddcb6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java @@ -338,7 +338,6 @@ public void testKeyOps() throws Exception { long initialNumKeyLookup = getLongCounter("NumKeyLookup", omMetrics); long initialNumKeyDeletes = getLongCounter("NumKeyDeletes", omMetrics); long initialNumKeyLists = getLongCounter("NumKeyLists", omMetrics); - long initialNumTrashKeyLists = getLongCounter("NumTrashKeyLists", omMetrics); long initialNumKeys = getLongCounter("NumKeys", omMetrics); long initialNumInitiateMultipartUploads = getLongCounter("NumInitiateMultipartUploads", omMetrics); @@ -346,7 +345,6 @@ public void testKeyOps() throws Exception { long initialNumKeyAllocateFails = getLongCounter("NumKeyAllocateFails", omMetrics); long initialNumKeyLookupFails = getLongCounter("NumKeyLookupFails", omMetrics); long initialNumKeyDeleteFails = getLongCounter("NumKeyDeleteFails", omMetrics); - long initialNumTrashKeyListFails = getLongCounter("NumTrashKeyListFails", omMetrics); long initialNumInitiateMultipartUploadFails = getLongCounter("NumInitiateMultipartUploadFails", omMetrics); long initialNumBlockAllocationFails = getLongCounter("NumBlockAllocationFails", omMetrics); long initialNumKeyListFails = getLongCounter("NumKeyListFails", omMetrics); @@ -365,7 +363,6 @@ public void testKeyOps() throws Exception { assertEquals(initialNumKeyLookup + 1, getLongCounter("NumKeyLookup", omMetrics)); assertEquals(initialNumKeyDeletes + 1, getLongCounter("NumKeyDeletes", omMetrics)); assertEquals(initialNumKeyLists + 1, getLongCounter("NumKeyLists", omMetrics)); - assertEquals(initialNumTrashKeyLists + 1, getLongCounter("NumTrashKeyLists", omMetrics)); assertEquals(initialNumKeys, getLongCounter("NumKeys", omMetrics)); assertEquals(initialNumInitiateMultipartUploads + 1, getLongCounter("NumInitiateMultipartUploads", omMetrics)); @@ -429,14 +426,12 @@ public void testKeyOps() throws Exception { assertEquals(initialNumKeyLookup + 3, getLongCounter("NumKeyLookup", omMetrics)); assertEquals(initialNumKeyDeletes + 4, getLongCounter("NumKeyDeletes", omMetrics)); assertEquals(initialNumKeyLists + 3, getLongCounter("NumKeyLists", omMetrics)); - assertEquals(initialNumTrashKeyLists + 3, getLongCounter("NumTrashKeyLists", omMetrics)); assertEquals(initialNumInitiateMultipartUploads + 3, getLongCounter("NumInitiateMultipartUploads", omMetrics)); assertEquals(initialNumKeyAllocateFails + 1, getLongCounter("NumKeyAllocateFails", omMetrics)); assertEquals(initialNumKeyLookupFails + 1, getLongCounter("NumKeyLookupFails", omMetrics)); assertEquals(initialNumKeyDeleteFails + 1, getLongCounter("NumKeyDeleteFails", omMetrics)); assertEquals(initialNumKeyListFails + 1, getLongCounter("NumKeyListFails", omMetrics)); - assertEquals(initialNumTrashKeyListFails + 1, getLongCounter("NumTrashKeyListFails", omMetrics)); assertEquals(initialNumInitiateMultipartUploadFails + 1, getLongCounter( "NumInitiateMultipartUploadFails", omMetrics)); assertEquals(initialNumKeys + 2, getLongCounter("NumKeys", omMetrics)); From 163c5521e41b4cf05eebd81d5bbc176978a7185f Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Thu, 22 Aug 2024 16:49:32 +0530 Subject: [PATCH 14/16] Fixed keyOps count --- .../test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java index 5ddf554ddcb6..87a599e0348c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java @@ -354,11 +354,11 @@ public void testKeyOps() throws Exception { TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, BucketLayout.LEGACY); OmKeyArgs keyArgs = createKeyArgs(volumeName, bucketName, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); - doKeyOps(keyArgs); + doKeyOps(keyArgs); // This will perform 7 different operations on the key omMetrics = getMetrics("OMMetrics"); - assertEquals(initialNumKeyOps + 8, getLongCounter("NumKeyOps", omMetrics)); + assertEquals(initialNumKeyOps + 7, getLongCounter("NumKeyOps", omMetrics)); assertEquals(initialNumKeyAllocate + 1, getLongCounter("NumKeyAllocate", omMetrics)); assertEquals(initialNumKeyLookup + 1, getLongCounter("NumKeyLookup", omMetrics)); assertEquals(initialNumKeyDeletes + 1, getLongCounter("NumKeyDeletes", omMetrics)); From 817ec783b73e1afdcba482ae4d61b88e19308e33 Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Mon, 26 Aug 2024 23:35:59 +0530 Subject: [PATCH 15/16] Fixed numKeyOps test --- .../src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java index 87a599e0348c..0481ee4a867c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java @@ -421,7 +421,7 @@ public void testKeyOps() throws Exception { doKeyOps(keyArgs); omMetrics = getMetrics("OMMetrics"); - assertEquals(initialNumKeyOps + 31, getLongCounter("NumKeyOps", omMetrics)); + assertEquals(initialNumKeyOps + 28, getLongCounter("NumKeyOps", omMetrics)); assertEquals(initialNumKeyAllocate + 6, getLongCounter("NumKeyAllocate", omMetrics)); assertEquals(initialNumKeyLookup + 3, getLongCounter("NumKeyLookup", omMetrics)); assertEquals(initialNumKeyDeletes + 4, getLongCounter("NumKeyDeletes", omMetrics)); From 3172f3c9f13cfc3d4779871bb8e484f0aca889c5 Mon Sep 17 00:00:00 2001 From: Abhishek Pal Date: Tue, 27 Aug 2024 11:36:29 +0530 Subject: [PATCH 16/16] Addressed review comments --- .../common/src/main/java/org/apache/hadoop/ozone/OmUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 76d0079bf6b5..32f258eaaba6 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -243,7 +243,7 @@ public static boolean isReadOnly( case ListKeys: case ListKeysLight: case ListTrash: - // This is deprecated by HDDS-11251. Keeping this in here + // ListTrash is deprecated by HDDS-11251. Keeping this in here // As protobuf currently doesn't support deprecating enum fields // TODO: Remove once migrated to proto3 and mark fields in proto // as deprecated @@ -307,7 +307,7 @@ public static boolean isReadOnly( case AddAcl: case PurgeKeys: case RecoverTrash: - // This is deprecated by HDDS-11251. Keeping this in here + // RecoverTrash is deprecated by HDDS-11251. Keeping this in here // As protobuf currently doesn't support deprecating enum fields // TODO: Remove once migrated to proto3 and mark fields in proto // as deprecated