From c825d0faf657f85a58fdabe1e485c816047bd7cf Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Fri, 15 Sep 2023 13:07:46 -0700 Subject: [PATCH 1/6] HDDS-7743. [Snapshot] Implement snapshot disk usage. --- .../java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../src/main/proto/OmClientProtocol.proto | 17 ++ .../ratis/utils/OzoneManagerRatisUtils.java | 3 + .../snapshot/OMSnapshotUpdateSizeRequest.java | 98 ++++++++++ .../OMSnapshotUpdateSizeResponse.java | 67 +++++++ .../ozone/om/service/KeyDeletingService.java | 159 +++++++++++++++ .../TestOMSnapshotSizeRequestAndResponse.java | 185 ++++++++++++++++++ .../om/service/TestKeyDeletingService.java | 152 +++++++++++++- 8 files changed, 679 insertions(+), 3 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotUpdateSizeRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotUpdateSizeResponse.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 711db38bcb81..89e0e3d79718 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -321,6 +321,7 @@ public static boolean isReadOnly( case SnapshotPurge: case RecoverLease: case SetTimes: + case SnapshotUpdateSize: case UnknownCommand: return false; default: diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index ace702812842..d53a0ebb2c6f 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -141,6 +141,7 @@ enum Type { CancelSnapshotDiff = 123; SetSafeMode = 124; PrintCompactionLogDag = 125; + SnapshotUpdateSize = 126; } enum SafeMode { @@ -272,6 +273,7 @@ message OMRequest { optional CancelSnapshotDiffRequest CancelSnapshotDiffRequest = 123; optional SetSafeModeRequest SetSafeModeRequest = 124; optional PrintCompactionLogDagRequest PrintCompactionLogDagRequest = 125; + optional SnapshotUpdateSizeRequest SnapshotUpdateSizeRequest = 126; } message OMResponse { @@ -388,6 +390,7 @@ message OMResponse { optional CancelSnapshotDiffResponse cancelSnapshotDiffResponse = 123; optional SetSafeModeResponse SetSafeModeResponse = 124; optional PrintCompactionLogDagResponse PrintCompactionLogDagResponse = 125; + optional SnapshotUpdateSizeResponse SnapshotUpdateSizeResponse = 126; } enum Status { @@ -1815,6 +1818,16 @@ message SnapshotPurgeRequest { repeated string updatedSnapshotDBKey = 2; } +message SnapshotUpdateSizeRequest { + repeated SnapshotSize snapshotSize = 1; +} + +message SnapshotSize { + optional string snapshotKey = 1; + optional uint64 exclusiveSize = 2; + optional uint64 exclusiveReplicatedSize = 3; +} + message DeleteTenantRequest { optional string tenantId = 1; } @@ -1900,6 +1913,10 @@ message SnapshotPurgeResponse { } +message SnapshotUpdateSizeResponse { + +} + message SnapshotDiffReportProto { optional string volumeName = 1; optional string bucketName = 2; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 399b2cecc7c0..7397dc01ccab 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -77,6 +77,7 @@ import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotDeleteRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveDeletedKeysRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotPurgeRequest; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotUpdateSizeRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMCancelPrepareRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMFinalizeUpgradeRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMPrepareRequest; @@ -224,6 +225,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMSnapshotMoveDeletedKeysRequest(omRequest); case SnapshotPurge: return new OMSnapshotPurgeRequest(omRequest); + case SnapshotUpdateSize: + return new OMSnapshotUpdateSizeRequest(omRequest); case DeleteOpenKeys: BucketLayout bktLayout = BucketLayout.DEFAULT; if (omRequest.getDeleteOpenKeysRequest().hasBucketLayout()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotUpdateSizeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotUpdateSizeRequest.java new file mode 100644 index 000000000000..c5ddb301649e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotUpdateSizeRequest.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.om.request.snapshot; + +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotUpdateSizeResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Updates the exclusive size of the snapshot. + */ +public class OMSnapshotUpdateSizeRequest extends OMClientRequest { + + public OMSnapshotUpdateSizeRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { + + OMClientResponse omClientResponse = null; + OMMetadataManager metadataManager = ozoneManager.getMetadataManager(); + + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); + OzoneManagerProtocolProtos.SnapshotUpdateSizeRequest + snapshotUpdateSizeRequest = getOmRequest() + .getSnapshotUpdateSizeRequest(); + + List snapshotSizeList = snapshotUpdateSizeRequest + .getSnapshotSizeList(); + Map updatedSnapInfos = new HashMap<>(); + + try { + for (SnapshotSize snapshotSize: snapshotSizeList) { + String snapshotKey = snapshotSize.getSnapshotKey(); + long exclusiveSize = snapshotSize.getExclusiveSize(); + long exclusiveReplicatedSize = snapshotSize + .getExclusiveReplicatedSize(); + SnapshotInfo snapshotInfo = metadataManager + .getSnapshotInfoTable().get(snapshotKey); + + if (snapshotInfo == null) { + continue; + } + + // Set Exclusive size. + snapshotInfo.setExclusiveSize(exclusiveSize); + snapshotInfo.setExclusiveReplicatedSize(exclusiveReplicatedSize); + // Update Table Cache + metadataManager.getSnapshotInfoTable().addCacheEntry( + new CacheKey<>(snapshotKey), + CacheValue.get(trxnLogIndex, snapshotInfo)); + updatedSnapInfos.put(snapshotKey, snapshotInfo); + } + omClientResponse = new OMSnapshotUpdateSizeResponse( + omResponse.build(), updatedSnapInfos); + } catch (IOException ex) { + omClientResponse = new OMSnapshotUpdateSizeResponse( + createErrorOMResponse(omResponse, ex)); + } finally { + addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, + omDoubleBufferHelper); + } + return omClientResponse; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotUpdateSizeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotUpdateSizeResponse.java new file mode 100644 index 000000000000..8ee240203f30 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotUpdateSizeResponse.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.snapshot; + +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.Map; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; + +/** + * Response for OMSnapshotUpdateSizeRequest. + */ +@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) +public class OMSnapshotUpdateSizeResponse extends OMClientResponse { + private final Map updatedSnapInfos; + + public OMSnapshotUpdateSizeResponse( + @Nonnull OMResponse omResponse, + Map updatedSnapInfos) { + super(omResponse); + this.updatedSnapInfos = updatedSnapInfos; + } + + public OMSnapshotUpdateSizeResponse(@Nonnull OMResponse omResponse) { + super(omResponse); + checkStatusNotOK(); + this.updatedSnapInfos = null; + } + + @Override + protected void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) + throws IOException { + OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) + omMetadataManager; + for (Map.Entry entry : updatedSnapInfos.entrySet()) { + metadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, + entry.getKey(), entry.getValue()); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 5a1f8336fc0c..463c1e13a458 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -19,7 +19,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -43,6 +46,8 @@ import org.apache.hadoop.ozone.om.snapshot.SnapshotCache; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotUpdateSizeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; @@ -88,6 +93,9 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private final int keyLimitPerTask; private final AtomicLong deletedKeyCount; private final AtomicBoolean suspended; + private final Map exclusiveSizeList; + private final Map exclusiveReplicatedSizeList; + private final Set completedExclusiveSizeList; public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, @@ -101,6 +109,9 @@ public KeyDeletingService(OzoneManager ozoneManager, OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); this.deletedKeyCount = new AtomicLong(0); this.suspended = new AtomicBoolean(false); + this.exclusiveSizeList = new HashMap<>(); + this.exclusiveReplicatedSizeList = new HashMap<>(); + this.completedExclusiveSizeList = new HashSet<>(); } /** @@ -212,6 +223,7 @@ public BackgroundTaskResult call() { return EmptyTaskResult.newResult(); } + @SuppressWarnings("checkstyle:MethodLength") private void processSnapshotDeepClean(int delCount) throws IOException { OmSnapshotManager omSnapshotManager = @@ -269,7 +281,15 @@ private void processSnapshotDeepClean(int delCount) String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; SnapshotInfo previousSnapshot = getPreviousActiveSnapshot( currSnapInfo, snapChainManager, omSnapshotManager); + SnapshotInfo previousToPrevSnapshot = null; + + if (previousSnapshot != null) { + previousToPrevSnapshot = getPreviousActiveSnapshot( + previousSnapshot, snapChainManager, omSnapshotManager); + } + Table previousKeyTable = null; + Table prevRenamedTable = null; ReferenceCounted rcPrevOmSnapshot = null; OmSnapshot omPreviousSnapshot = null; @@ -285,6 +305,25 @@ private void processSnapshotDeepClean(int delCount) previousKeyTable = omPreviousSnapshot.getMetadataManager() .getKeyTable(bucketInfo.getBucketLayout()); + prevRenamedTable = omPreviousSnapshot + .getMetadataManager().getSnapshotRenamedTable(); + } + + Table previousToPrevKeyTable = null; + ReferenceCounted + rcPrevToPrevOmSnapshot = null; + OmSnapshot omPreviousToPrevSnapshot = null; + if (previousToPrevSnapshot != null) { + rcPrevToPrevOmSnapshot = omSnapshotManager.checkForSnapshot( + previousToPrevSnapshot.getVolumeName(), + previousToPrevSnapshot.getBucketName(), + getSnapshotPrefix(previousToPrevSnapshot.getName()), true); + omPreviousToPrevSnapshot = (OmSnapshot) + rcPrevToPrevOmSnapshot.get(); + + previousToPrevKeyTable = omPreviousToPrevSnapshot + .getMetadataManager() + .getKeyTable(bucketInfo.getBucketLayout()); } try (TableIterator blocksForKeyDelete = currOmSnapshot @@ -338,6 +418,14 @@ private void processSnapshotDeepClean(int delCount) if (delCount < keyLimitPerTask) { // Deep clean is completed, we can update the SnapInfo. deepCleanedSnapshots.add(currSnapInfo.getTableKey()); + // exclusiveSizeList contains check is used to prevent + // case where there is no entry in deletedTable, this + // will throw NPE when we submit request. + if (previousSnapshot != null && exclusiveSizeList + .containsKey(previousSnapshot.getTableKey())) { + completedExclusiveSizeList.add( + previousSnapshot.getTableKey()); + } } if (!keysToPurge.isEmpty()) { @@ -348,14 +436,85 @@ private void processSnapshotDeepClean(int delCount) if (previousSnapshot != null) { rcPrevOmSnapshot.close(); } + if (previousToPrevSnapshot != null) { + rcPrevToPrevOmSnapshot.close(); + } } } } } + + updateSnapshotExclusiveSize(); updateDeepCleanedSnapshots(deepCleanedSnapshots); } + private OmKeyInfo getPreviousSnapshotKeyName( + OmKeyInfo keyInfo, OmBucketInfo bucketInfo, long volumeId, + Table snapRenamedTable, + Table previousKeyTable) throws IOException { + + if (keyInfo == null) { + return null; + } + + String dbKeyPrevSnap; + if (bucketInfo.getBucketLayout().isFileSystemOptimized()) { + dbKeyPrevSnap = getOzoneManager().getMetadataManager().getOzonePathKey( + volumeId, + bucketInfo.getObjectID(), + keyInfo.getParentObjectID(), + keyInfo.getFileName()); + } else { + dbKeyPrevSnap = getOzoneManager().getMetadataManager().getOzoneKey( + keyInfo.getVolumeName(), + keyInfo.getBucketName(), + keyInfo.getKeyName()); + } + + String dbRenameKey = getOzoneManager().getMetadataManager().getRenameKey( + keyInfo.getVolumeName(), + keyInfo.getBucketName(), + keyInfo.getObjectID()); + + String renamedKey = snapRenamedTable.getIfExist(dbRenameKey); + dbKeyPrevSnap = renamedKey != null ? renamedKey : dbKeyPrevSnap; + + return previousKeyTable.get(dbKeyPrevSnap); + } + + private void updateSnapshotExclusiveSize() { + + List snapshotSizeList = new ArrayList<>(); + for (String dbKey: completedExclusiveSizeList) { + SnapshotSize snapshotSize = SnapshotSize.newBuilder() + .setSnapshotKey(dbKey) + .setExclusiveSize(exclusiveSizeList.get(dbKey)) + .setExclusiveReplicatedSize( + exclusiveReplicatedSizeList.get(dbKey)) + .build(); + snapshotSizeList.add(snapshotSize); + exclusiveSizeList.remove(dbKey); + exclusiveReplicatedSizeList.remove(dbKey); + } + + if (!snapshotSizeList.isEmpty()) { + SnapshotUpdateSizeRequest snapshotUpdateSizeRequest = + SnapshotUpdateSizeRequest.newBuilder() + .addAllSnapshotSize(snapshotSizeList) + .build(); + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(Type.SnapshotUpdateSize) + .setSnapshotUpdateSizeRequest(snapshotUpdateSizeRequest) + .setClientId(clientId.toString()) + .build(); + + submitRequest(omRequest); + } + completedExclusiveSizeList.clear(); + } + private void updateDeepCleanedSnapshots(List deepCleanedSnapshots) { if (!deepCleanedSnapshots.isEmpty()) { SnapshotPurgeRequest snapshotPurgeRequest = SnapshotPurgeRequest diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java new file mode 100644 index 000000000000..39c2dd3c51d1 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.request.snapshot; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; + +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotUpdateSizeResponse; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotUpdateSizeRequest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Tests TestOMSnapshotSizeRequest TestOMSnapshotSizeResponse class. + */ +public class TestOMSnapshotSizeRequestAndResponse { + private BatchOperation batchOperation; + private OzoneManager ozoneManager; + private OMMetadataManager omMetadataManager; + + private String volumeName; + private String bucketName; + private String snapName; + private long exclusiveSize; + private long exclusiveSizeAfterRepl; + + // Just setting ozoneManagerDoubleBuffer which does nothing. + private static final OzoneManagerDoubleBufferHelper + DOUBLE_BUFFER_HELPER = ((response, transactionIndex) -> null); + + @BeforeEach + void setup(@TempDir File testDir) throws Exception { + ozoneManager = Mockito.mock(OzoneManager.class); + OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); + when(lvm.isAllowed(anyString())).thenReturn(true); + when(ozoneManager.getVersionManager()).thenReturn(lvm); + when(ozoneManager.isRatisEnabled()).thenReturn(true); + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, + testDir.getAbsolutePath()); + ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, + testDir.getAbsolutePath()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, + ozoneManager); + when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); + + volumeName = UUID.randomUUID().toString(); + bucketName = UUID.randomUUID().toString(); + snapName = UUID.randomUUID().toString(); + exclusiveSize = 2000L; + exclusiveSizeAfterRepl = 6000L; + } + + @Test + public void testValidateAndUpdateCache() throws IOException { + createSnapshotDataForTest(); + assertFalse(omMetadataManager.getSnapshotInfoTable().isEmpty()); + OzoneManagerProtocolProtos.OMRequest snapshotUpdateSizeRequest = + createSnapshotUpdateSizeRequest(); + + // Pre-Execute + OMSnapshotUpdateSizeRequest omSnapshotUpdateSizeRequest = new + OMSnapshotUpdateSizeRequest(snapshotUpdateSizeRequest); + OMRequest modifiedOmRequest = omSnapshotUpdateSizeRequest + .preExecute(ozoneManager); + omSnapshotUpdateSizeRequest = new + OMSnapshotUpdateSizeRequest(modifiedOmRequest); + + // Validate and Update Cache + OMSnapshotUpdateSizeResponse omSnapshotUpdateSizeResponse = + (OMSnapshotUpdateSizeResponse) omSnapshotUpdateSizeRequest + .validateAndUpdateCache(ozoneManager, 200L, + DOUBLE_BUFFER_HELPER); + + // Commit to DB. + batchOperation = omMetadataManager.getStore().initBatchOperation(); + omSnapshotUpdateSizeResponse.checkAndUpdateDB(omMetadataManager, + batchOperation); + omMetadataManager.getStore().commitBatchOperation(batchOperation); + + // Check if the exclusive size is set. + try (TableIterator> + iterator = omMetadataManager.getSnapshotInfoTable().iterator()) { + while (iterator.hasNext()) { + Table.KeyValue snapshotEntry = iterator.next(); + assertCacheValues(snapshotEntry.getKey()); + assertEquals(exclusiveSize, snapshotEntry.getValue(). + getExclusiveSize()); + assertEquals(exclusiveSizeAfterRepl, snapshotEntry.getValue() + .getExclusiveReplicatedSize()); + } + } + } + + private void assertCacheValues(String dbKey) { + CacheValue cacheValue = omMetadataManager + .getSnapshotInfoTable() + .getCacheValue(new CacheKey<>(dbKey)); + assertEquals(exclusiveSize, cacheValue.getCacheValue().getExclusiveSize()); + assertEquals(exclusiveSizeAfterRepl, cacheValue.getCacheValue() + .getExclusiveReplicatedSize()); + } + + private OMRequest createSnapshotUpdateSizeRequest() throws IOException { + List snapshotSizeList = new ArrayList<>(); + try (TableIterator> + iterator = omMetadataManager.getSnapshotInfoTable().iterator()) { + while (iterator.hasNext()) { + String snapDbKey = iterator.next().getKey(); + SnapshotSize snapshotSize = SnapshotSize.newBuilder() + .setSnapshotKey(snapDbKey) + .setExclusiveSize(exclusiveSize) + .setExclusiveReplicatedSize(exclusiveSizeAfterRepl) + .build(); + snapshotSizeList.add(snapshotSize); + } + } + SnapshotUpdateSizeRequest snapshotUpdateSizeRequest = + SnapshotUpdateSizeRequest.newBuilder() + .addAllSnapshotSize(snapshotSizeList) + .build(); + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SnapshotUpdateSize) + .setSnapshotUpdateSizeRequest(snapshotUpdateSizeRequest) + .setClientId(UUID.randomUUID().toString()) + .build(); + + return omRequest; + } + + private void createSnapshotDataForTest() throws IOException { + // Create 10 Snapshots + for (int i = 0; i < 10; i++) { + OMRequestTestUtils.addSnapshotToTableCache(volumeName, bucketName, + snapName + i, omMetadataManager); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java index 5b1c1325d669..340ce3b6bdf2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java @@ -24,12 +24,15 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.IOmMetadataReader; @@ -71,6 +74,7 @@ import org.apache.commons.lang3.RandomStringUtils; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; @@ -550,15 +554,126 @@ volumeName, bucketName, getSnapshotPrefix("snap3"), true)) { // 5 keys can be deep cleaned as it was stuck previously assertTableRowCount(snap3deletedTable, 10, metadataManager); - checkSnapDeepCleanStatus(snapshotInfoTable, false); writeClient.deleteSnapshot(volumeName, bucketName, "snap2"); assertTableRowCount(snapshotInfoTable, 2, metadataManager); assertTableRowCount(snap3deletedTable, 0, metadataManager); assertTableRowCount(deletedTable, 0, metadataManager); + checkSnapDeepCleanStatus(snapshotInfoTable, false); + } + + } + + @Test + public void testSnapshotExclusiveSize() throws Exception { + OzoneConfiguration conf = createConfAndInitValues(); + OmTestManagers omTestManagers + = new OmTestManagers(conf); + KeyManager keyManager = omTestManagers.getKeyManager(); + writeClient = omTestManagers.getWriteClient(); + om = omTestManagers.getOzoneManager(); + OMMetadataManager metadataManager = omTestManagers.getMetadataManager(); + Table snapshotInfoTable = + om.getMetadataManager().getSnapshotInfoTable(); + Table deletedTable = + om.getMetadataManager().getDeletedTable(); + Table renamedTable = + om.getMetadataManager().getSnapshotRenamedTable(); + Table keyTable = + om.getMetadataManager().getKeyTable(BucketLayout.DEFAULT); + + KeyDeletingService keyDeletingService = keyManager.getDeletingService(); + // Supspend KDS + keyDeletingService.suspend(); + + String volumeName = "volume1"; + String bucketName = "bucket1"; + String keyName = "key"; + + // Create Volume and Buckets + createVolumeAndBucket(keyManager, volumeName, bucketName, false); + + // Create 3 keys + for (int i = 1; i <= 3; i++) { + createAndCommitKey(keyManager, volumeName, bucketName, keyName + i, 3); + } + assertTableRowCount(keyTable, 3, metadataManager); + + // Create Snapshot1 + writeClient.createSnapshot(volumeName, bucketName, "snap1"); + assertTableRowCount(snapshotInfoTable, 1, metadataManager); + assertTableRowCount(deletedTable, 0, metadataManager); + + // Create 2 keys + for (int i = 4; i <= 5; i++) { + createAndCommitKey(keyManager, volumeName, bucketName, keyName + i, 3); + } + // Delete a key, rename 2 keys. We will be using this to test + // how we handle renamed key for exclusive size calculation. + renameKey(volumeName, bucketName, keyName + 1, "renamedKey1"); + renameKey(volumeName, bucketName, keyName + 2, "renamedKey2"); + deleteKey(volumeName, bucketName, keyName + 3); + assertTableRowCount(deletedTable, 1, metadataManager); + assertTableRowCount(renamedTable, 2, metadataManager); + + // Create Snapshot2 + writeClient.createSnapshot(volumeName, bucketName, "snap2"); + assertTableRowCount(snapshotInfoTable, 2, metadataManager); + assertTableRowCount(deletedTable, 0, metadataManager); + + // Create 2 keys + for (int i = 6; i <= 7; i++) { + createAndCommitKey(keyManager, volumeName, bucketName, keyName + i, 3); } + deleteKey(volumeName, bucketName, "renamedKey1"); + deleteKey(volumeName, bucketName, "key4"); + // Do a second rename of already renamedKey2 + renameKey(volumeName, bucketName, "renamedKey2", "renamedKey22"); + assertTableRowCount(deletedTable, 2, metadataManager); + assertTableRowCount(renamedTable, 1, metadataManager); + + // Create Snapshot3 + writeClient.createSnapshot(volumeName, bucketName, "snap3"); + // Delete 4 keys + deleteKey(volumeName, bucketName, "renamedKey22"); + for (int i = 5; i <= 7; i++) { + deleteKey(volumeName, bucketName, keyName + i); + } + + // Create Snapshot4 + writeClient.createSnapshot(volumeName, bucketName, "snap4"); + createAndCommitKey(keyManager, volumeName, bucketName, "key8", 3); + keyDeletingService.resume(); + + Map expectedSize = new HashMap() {{ + put("snap1", 1000L); + put("snap2", 1000L); + put("snap3", 2000L); + put("snap4", 0L); + }}; + + long prevKdsRunCount = keyDeletingService.getRunCount().get(); + + // Let KeyDeletingService to run for some iterations + GenericTestUtils.waitFor( + () -> (keyDeletingService.getRunCount().get() > prevKdsRunCount + 5), + 100, 10000); + + // Check if the exclusive size is set. + try (TableIterator> + iterator = snapshotInfoTable.iterator()) { + while (iterator.hasNext()) { + Table.KeyValue snapshotEntry = iterator.next(); + String snapshotName = snapshotEntry.getValue().getName(); + assertEquals(expectedSize.get(snapshotName), snapshotEntry.getValue(). + getExclusiveSize()); + // Since for the test we are using RATIS/THREE + assertEquals(expectedSize.get(snapshotName) * 3, + snapshotEntry.getValue().getExclusiveReplicatedSize()); + } + } } private void checkSnapDeepCleanStatus(Table @@ -613,6 +728,37 @@ private void createVolumeAndBucket(KeyManager keyManager, String volumeName, .build()); } + private void deleteKey(String volumeName, + String bucketName, + String keyName) throws IOException { + OmKeyArgs keyArg = + new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setAcls(Collections.emptyList()) + .setReplicationConfig(StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE)) + .build(); + writeClient.deleteKey(keyArg); + } + + private void renameKey(String volumeName, + String bucketName, + String keyName, + String toKeyName) throws IOException { + OmKeyArgs keyArg = + new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setAcls(Collections.emptyList()) + .setReplicationConfig(StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE)) + .build(); + writeClient.renameKey(keyArg, toKeyName); + } + private OmKeyArgs createAndCommitKey(KeyManager keyManager, String volumeName, String bucketName, String keyName, int numBlocks) throws IOException { return createAndCommitKey(keyManager, volumeName, bucketName, keyName, @@ -630,8 +776,8 @@ private OmKeyArgs createAndCommitKey(KeyManager keyManager, String volumeName, .setBucketName(bucketName) .setKeyName(keyName) .setAcls(Collections.emptyList()) - .setReplicationConfig(StandaloneReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.ONE)) + .setReplicationConfig(RatisReplicationConfig.getInstance(THREE)) + .setDataSize(1000L) .setLocationInfoList(new ArrayList<>()) .build(); //Open and Commit the Key in the Key Manager. From 1ddeb65652612b767a274f2df5b34627762510cb Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Mon, 18 Sep 2023 13:34:18 -0700 Subject: [PATCH 2/6] Address Review Comments. --- .../java/org/apache/hadoop/ozone/OmUtils.java | 2 +- .../src/main/proto/OmClientProtocol.proto | 14 +++---- .../ratis/utils/OzoneManagerRatisUtils.java | 6 +-- ...java => OMSnapshotSetPropertyRequest.java} | 35 +++++++++-------- ...ava => OMSnapshotSetPropertyResponse.java} | 9 ++--- .../ozone/om/service/KeyDeletingService.java | 22 +++++------ .../TestOMSnapshotSizeRequestAndResponse.java | 38 +++++++++---------- 7 files changed, 65 insertions(+), 61 deletions(-) rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/{OMSnapshotUpdateSizeRequest.java => OMSnapshotSetPropertyRequest.java} (74%) rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/{OMSnapshotUpdateSizeResponse.java => OMSnapshotSetPropertyResponse.java} (90%) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 89e0e3d79718..c89619e3d1be 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -321,7 +321,7 @@ public static boolean isReadOnly( case SnapshotPurge: case RecoverLease: case SetTimes: - case SnapshotUpdateSize: + case SetSnapshotProperty: case UnknownCommand: return false; default: diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index d53a0ebb2c6f..96ca55d515cb 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -141,7 +141,7 @@ enum Type { CancelSnapshotDiff = 123; SetSafeMode = 124; PrintCompactionLogDag = 125; - SnapshotUpdateSize = 126; + SetSnapshotProperty = 126; } enum SafeMode { @@ -273,7 +273,7 @@ message OMRequest { optional CancelSnapshotDiffRequest CancelSnapshotDiffRequest = 123; optional SetSafeModeRequest SetSafeModeRequest = 124; optional PrintCompactionLogDagRequest PrintCompactionLogDagRequest = 125; - optional SnapshotUpdateSizeRequest SnapshotUpdateSizeRequest = 126; + optional SetSnapshotPropertyRequest SetSnapshotPropertyRequest = 126; } message OMResponse { @@ -390,7 +390,7 @@ message OMResponse { optional CancelSnapshotDiffResponse cancelSnapshotDiffResponse = 123; optional SetSafeModeResponse SetSafeModeResponse = 124; optional PrintCompactionLogDagResponse PrintCompactionLogDagResponse = 125; - optional SnapshotUpdateSizeResponse SnapshotUpdateSizeResponse = 126; + optional SetSnapshotPropertyResponse SetSnapshotPropertyResponse = 126; } enum Status { @@ -1818,11 +1818,11 @@ message SnapshotPurgeRequest { repeated string updatedSnapshotDBKey = 2; } -message SnapshotUpdateSizeRequest { - repeated SnapshotSize snapshotSize = 1; +message SetSnapshotPropertyRequest { + repeated SnapshotProperty snapshotProperty = 1; } -message SnapshotSize { +message SnapshotProperty { optional string snapshotKey = 1; optional uint64 exclusiveSize = 2; optional uint64 exclusiveReplicatedSize = 3; @@ -1913,7 +1913,7 @@ message SnapshotPurgeResponse { } -message SnapshotUpdateSizeResponse { +message SetSnapshotPropertyResponse { } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 7397dc01ccab..0146f1f50d54 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -77,7 +77,7 @@ import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotDeleteRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveDeletedKeysRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotPurgeRequest; -import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotUpdateSizeRequest; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotSetPropertyRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMCancelPrepareRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMFinalizeUpgradeRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMPrepareRequest; @@ -225,8 +225,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMSnapshotMoveDeletedKeysRequest(omRequest); case SnapshotPurge: return new OMSnapshotPurgeRequest(omRequest); - case SnapshotUpdateSize: - return new OMSnapshotUpdateSizeRequest(omRequest); + case SetSnapshotProperty: + return new OMSnapshotSetPropertyRequest(omRequest); case DeleteOpenKeys: BucketLayout bktLayout = BucketLayout.DEFAULT; if (omRequest.getDeleteOpenKeysRequest().hasBucketLayout()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotUpdateSizeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java similarity index 74% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotUpdateSizeRequest.java rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java index c5ddb301649e..af3fe5885f71 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotUpdateSizeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java @@ -26,10 +26,12 @@ import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotUpdateSizeResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotSetPropertyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashMap; @@ -39,9 +41,11 @@ /** * Updates the exclusive size of the snapshot. */ -public class OMSnapshotUpdateSizeRequest extends OMClientRequest { +public class OMSnapshotSetPropertyRequest extends OMClientRequest { + private static final Logger LOG = + LoggerFactory.getLogger(OMSnapshotSetPropertyRequest.class); - public OMSnapshotUpdateSizeRequest(OMRequest omRequest) { + public OMSnapshotSetPropertyRequest(OMRequest omRequest) { super(omRequest); } @@ -54,24 +58,25 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, OzoneManagerProtocolProtos.OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); - OzoneManagerProtocolProtos.SnapshotUpdateSizeRequest - snapshotUpdateSizeRequest = getOmRequest() - .getSnapshotUpdateSizeRequest(); + OzoneManagerProtocolProtos.SetSnapshotPropertyRequest + setSnapshotPropertyRequest = getOmRequest() + .getSetSnapshotPropertyRequest(); - List snapshotSizeList = snapshotUpdateSizeRequest - .getSnapshotSizeList(); + List snapshotPropertyList = setSnapshotPropertyRequest + .getSnapshotPropertyList(); Map updatedSnapInfos = new HashMap<>(); try { - for (SnapshotSize snapshotSize: snapshotSizeList) { - String snapshotKey = snapshotSize.getSnapshotKey(); - long exclusiveSize = snapshotSize.getExclusiveSize(); - long exclusiveReplicatedSize = snapshotSize + for (SnapshotProperty snapshotProperty: snapshotPropertyList) { + String snapshotKey = snapshotProperty.getSnapshotKey(); + long exclusiveSize = snapshotProperty.getExclusiveSize(); + long exclusiveReplicatedSize = snapshotProperty .getExclusiveReplicatedSize(); SnapshotInfo snapshotInfo = metadataManager .getSnapshotInfoTable().get(snapshotKey); if (snapshotInfo == null) { + LOG.error("SnapshotInfo for Snapshot: {} is not found", snapshotKey); continue; } @@ -84,10 +89,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, CacheValue.get(trxnLogIndex, snapshotInfo)); updatedSnapInfos.put(snapshotKey, snapshotInfo); } - omClientResponse = new OMSnapshotUpdateSizeResponse( + omClientResponse = new OMSnapshotSetPropertyResponse( omResponse.build(), updatedSnapInfos); } catch (IOException ex) { - omClientResponse = new OMSnapshotUpdateSizeResponse( + omClientResponse = new OMSnapshotSetPropertyResponse( createErrorOMResponse(omResponse, ex)); } finally { addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotUpdateSizeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java similarity index 90% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotUpdateSizeResponse.java rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java index 8ee240203f30..fe407188e0b6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotUpdateSizeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java @@ -26,7 +26,6 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; - import javax.annotation.Nonnull; import java.io.IOException; import java.util.Map; @@ -34,20 +33,20 @@ import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; /** - * Response for OMSnapshotUpdateSizeRequest. + * Response for OMSnapshotSetPropertyRequest. */ @CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) -public class OMSnapshotUpdateSizeResponse extends OMClientResponse { +public class OMSnapshotSetPropertyResponse extends OMClientResponse { private final Map updatedSnapInfos; - public OMSnapshotUpdateSizeResponse( + public OMSnapshotSetPropertyResponse( @Nonnull OMResponse omResponse, Map updatedSnapInfos) { super(omResponse); this.updatedSnapInfos = updatedSnapInfos; } - public OMSnapshotUpdateSizeResponse(@Nonnull OMResponse omResponse) { + public OMSnapshotSetPropertyResponse(@Nonnull OMResponse omResponse) { super(omResponse); checkStatusNotOK(); this.updatedSnapInfos = null; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 463c1e13a458..60533e60cc5d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -46,8 +46,8 @@ import org.apache.hadoop.ozone.om.snapshot.SnapshotCache; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotUpdateSizeRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotProperty; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; @@ -485,28 +485,28 @@ private OmKeyInfo getPreviousSnapshotKeyName( private void updateSnapshotExclusiveSize() { - List snapshotSizeList = new ArrayList<>(); + List snapshotPropertyList = new ArrayList<>(); for (String dbKey: completedExclusiveSizeList) { - SnapshotSize snapshotSize = SnapshotSize.newBuilder() + SnapshotProperty snapshotProperty = SnapshotProperty.newBuilder() .setSnapshotKey(dbKey) .setExclusiveSize(exclusiveSizeList.get(dbKey)) .setExclusiveReplicatedSize( exclusiveReplicatedSizeList.get(dbKey)) .build(); - snapshotSizeList.add(snapshotSize); + snapshotPropertyList.add(snapshotProperty); exclusiveSizeList.remove(dbKey); exclusiveReplicatedSizeList.remove(dbKey); } - if (!snapshotSizeList.isEmpty()) { - SnapshotUpdateSizeRequest snapshotUpdateSizeRequest = - SnapshotUpdateSizeRequest.newBuilder() - .addAllSnapshotSize(snapshotSizeList) + if (!snapshotPropertyList.isEmpty()) { + SetSnapshotPropertyRequest setSnapshotPropertyRequest = + SetSnapshotPropertyRequest.newBuilder() + .addAllSnapshotProperty(snapshotPropertyList) .build(); OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(Type.SnapshotUpdateSize) - .setSnapshotUpdateSizeRequest(snapshotUpdateSizeRequest) + .setCmdType(Type.SetSnapshotProperty) + .setSetSnapshotPropertyRequest(setSnapshotPropertyRequest) .setClientId(clientId.toString()) .build(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java index 39c2dd3c51d1..e4926c501902 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java @@ -33,12 +33,12 @@ import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotUpdateSizeResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotSetPropertyResponse; import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotSize; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotUpdateSizeRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotProperty; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSnapshotPropertyRequest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; @@ -105,22 +105,22 @@ public void testValidateAndUpdateCache() throws IOException { createSnapshotUpdateSizeRequest(); // Pre-Execute - OMSnapshotUpdateSizeRequest omSnapshotUpdateSizeRequest = new - OMSnapshotUpdateSizeRequest(snapshotUpdateSizeRequest); - OMRequest modifiedOmRequest = omSnapshotUpdateSizeRequest + OMSnapshotSetPropertyRequest omSnapshotSetPropertyRequest = new + OMSnapshotSetPropertyRequest(snapshotUpdateSizeRequest); + OMRequest modifiedOmRequest = omSnapshotSetPropertyRequest .preExecute(ozoneManager); - omSnapshotUpdateSizeRequest = new - OMSnapshotUpdateSizeRequest(modifiedOmRequest); + omSnapshotSetPropertyRequest = new + OMSnapshotSetPropertyRequest(modifiedOmRequest); // Validate and Update Cache - OMSnapshotUpdateSizeResponse omSnapshotUpdateSizeResponse = - (OMSnapshotUpdateSizeResponse) omSnapshotUpdateSizeRequest + OMSnapshotSetPropertyResponse omSnapshotSetPropertyResponse = + (OMSnapshotSetPropertyResponse) omSnapshotSetPropertyRequest .validateAndUpdateCache(ozoneManager, 200L, DOUBLE_BUFFER_HELPER); // Commit to DB. batchOperation = omMetadataManager.getStore().initBatchOperation(); - omSnapshotUpdateSizeResponse.checkAndUpdateDB(omMetadataManager, + omSnapshotSetPropertyResponse.checkAndUpdateDB(omMetadataManager, batchOperation); omMetadataManager.getStore().commitBatchOperation(batchOperation); @@ -148,27 +148,27 @@ private void assertCacheValues(String dbKey) { } private OMRequest createSnapshotUpdateSizeRequest() throws IOException { - List snapshotSizeList = new ArrayList<>(); + List snapshotPropertyList = new ArrayList<>(); try (TableIterator> iterator = omMetadataManager.getSnapshotInfoTable().iterator()) { while (iterator.hasNext()) { String snapDbKey = iterator.next().getKey(); - SnapshotSize snapshotSize = SnapshotSize.newBuilder() + SnapshotProperty snapshotSize = SnapshotProperty.newBuilder() .setSnapshotKey(snapDbKey) .setExclusiveSize(exclusiveSize) .setExclusiveReplicatedSize(exclusiveSizeAfterRepl) .build(); - snapshotSizeList.add(snapshotSize); + snapshotPropertyList.add(snapshotSize); } } - SnapshotUpdateSizeRequest snapshotUpdateSizeRequest = - SnapshotUpdateSizeRequest.newBuilder() - .addAllSnapshotSize(snapshotSizeList) + SetSnapshotPropertyRequest snapshotUpdateSizeRequest = + SetSnapshotPropertyRequest.newBuilder() + .addAllSnapshotProperty(snapshotPropertyList) .build(); OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(OzoneManagerProtocolProtos.Type.SnapshotUpdateSize) - .setSnapshotUpdateSizeRequest(snapshotUpdateSizeRequest) + .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) + .setSetSnapshotPropertyRequest(snapshotUpdateSizeRequest) .setClientId(UUID.randomUUID().toString()) .build(); From 99177da00232b64cf4c0a7b0427bc0f88c08c159 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Tue, 26 Sep 2023 12:38:44 -0700 Subject: [PATCH 3/6] Address review comments. --- .../OMSnapshotSetPropertyResponse.java | 2 +- .../ozone/om/service/KeyDeletingService.java | 123 ++++++++++-------- ...napshotSetPropertyRequestAndResponse.java} | 5 +- 3 files changed, 76 insertions(+), 54 deletions(-) rename hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/{TestOMSnapshotSizeRequestAndResponse.java => TestOMSnapshotSetPropertyRequestAndResponse.java} (98%) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java index fe407188e0b6..fb84dfab8edc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java @@ -41,7 +41,7 @@ public class OMSnapshotSetPropertyResponse extends OMClientResponse { public OMSnapshotSetPropertyResponse( @Nonnull OMResponse omResponse, - Map updatedSnapInfos) { + @Nonnull Map updatedSnapInfos) { super(omResponse); this.updatedSnapInfos = updatedSnapInfos; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 60533e60cc5d..4d9064caf728 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -30,6 +30,7 @@ import com.google.protobuf.ServiceException; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneConsts; @@ -93,9 +94,9 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private final int keyLimitPerTask; private final AtomicLong deletedKeyCount; private final AtomicBoolean suspended; - private final Map exclusiveSizeList; - private final Map exclusiveReplicatedSizeList; - private final Set completedExclusiveSizeList; + private final Map exclusiveSizeMap; + private final Map exclusiveReplicatedSizeMap; + private final Set completedExclusiveSizeMap; public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, @@ -109,9 +110,9 @@ public KeyDeletingService(OzoneManager ozoneManager, OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); this.deletedKeyCount = new AtomicLong(0); this.suspended = new AtomicBoolean(false); - this.exclusiveSizeList = new HashMap<>(); - this.exclusiveReplicatedSizeList = new HashMap<>(); - this.completedExclusiveSizeList = new HashSet<>(); + this.exclusiveSizeMap = new HashMap<>(); + this.exclusiveReplicatedSizeMap = new HashMap<>(); + this.completedExclusiveSizeMap = new HashSet<>(); } /** @@ -355,38 +356,18 @@ private void processSnapshotDeepClean(int delCount) // snapshot. Here since we are only iterating through // deletedTable we can check the previous and previous to // previous snapshot to achieve the same. + // previousSnapshot - Current Snapshot for which we are + // calculating exclusive size. + // currSnapshot - Snapshot's deletedTable which is used to + // calc current(prev) snapshot's exclusive size. + // previousToPrevSnapshot - Previous to the previous snapshot + // to check if it's exclusive to the current(prev) + // snapshot or not. if (previousSnapshot != null) { - String prevSnapKey = previousSnapshot.getTableKey(); - long exclusiveReplicatedSize = - exclusiveReplicatedSizeList.getOrDefault( - prevSnapKey, 0L) + keyInfo.getReplicatedSize(); - long exclusiveSize = exclusiveSizeList.getOrDefault( - prevSnapKey, 0L) + keyInfo.getDataSize(); - - // If there is no previous to previous snapshot, then - // the previous snapshot is the first snapshot. - if (previousToPrevSnapshot == null) { - exclusiveSizeList.put(prevSnapKey, exclusiveSize); - exclusiveReplicatedSizeList.put(prevSnapKey, - exclusiveReplicatedSize); - } else { - OmKeyInfo keyInfoPrevSnapshot = - getPreviousSnapshotKeyName( - keyInfo, bucketInfo, volumeId, - snapRenamedTable, previousKeyTable); - OmKeyInfo keyInfoPrevToPrevSnapshot = - getPreviousSnapshotKeyName( - keyInfoPrevSnapshot, bucketInfo, volumeId, - prevRenamedTable, previousToPrevKeyTable); - // If the previous to previous snapshot doesn't - // have the key, then it is exclusive size for the - // previous snapshot. - if (keyInfoPrevToPrevSnapshot == null) { - exclusiveSizeList.put(prevSnapKey, exclusiveSize); - exclusiveReplicatedSizeList.put(prevSnapKey, - exclusiveReplicatedSize); - } - } + calculateExclusiveSize(previousSnapshot, + previousToPrevSnapshot, keyInfo, bucketInfo, volumeId, + snapRenamedTable, previousKeyTable, prevRenamedTable, + previousToPrevKeyTable); } if (isKeyReclaimable(previousKeyTable, snapRenamedTable, @@ -421,9 +402,9 @@ private void processSnapshotDeepClean(int delCount) // exclusiveSizeList contains check is used to prevent // case where there is no entry in deletedTable, this // will throw NPE when we submit request. - if (previousSnapshot != null && exclusiveSizeList + if (previousSnapshot != null && exclusiveSizeMap .containsKey(previousSnapshot.getTableKey())) { - completedExclusiveSizeList.add( + completedExclusiveSizeMap.add( previousSnapshot.getTableKey()); } } @@ -433,12 +414,7 @@ private void processSnapshotDeepClean(int delCount) keysToModify, currSnapInfo.getTableKey()); } } finally { - if (previousSnapshot != null) { - rcPrevOmSnapshot.close(); - } - if (previousToPrevSnapshot != null) { - rcPrevToPrevOmSnapshot.close(); - } + IOUtils.closeQuietly(rcPrevOmSnapshot, rcPrevToPrevOmSnapshot); } } @@ -449,6 +425,47 @@ private void processSnapshotDeepClean(int delCount) updateDeepCleanedSnapshots(deepCleanedSnapshots); } + @SuppressWarnings("checkstyle:ParameterNumber") + private void calculateExclusiveSize( + SnapshotInfo previousSnapshot, + SnapshotInfo previousToPrevSnapshot, + OmKeyInfo keyInfo, + OmBucketInfo bucketInfo, long volumeId, + Table snapRenamedTable, + Table previousKeyTable, + Table prevRenamedTable, + Table previousToPrevKeyTable) throws IOException { + String prevSnapKey = previousSnapshot.getTableKey(); + long exclusiveReplicatedSize = + exclusiveReplicatedSizeMap.getOrDefault( + prevSnapKey, 0L) + keyInfo.getReplicatedSize(); + long exclusiveSize = exclusiveSizeMap.getOrDefault( + prevSnapKey, 0L) + keyInfo.getDataSize(); + + // If there is no previous to previous snapshot, then + // the previous snapshot is the first snapshot. + if (previousToPrevSnapshot == null) { + exclusiveSizeMap.put(prevSnapKey, exclusiveSize); + exclusiveReplicatedSizeMap.put(prevSnapKey, + exclusiveReplicatedSize); + } else { + OmKeyInfo keyInfoPrevSnapshot = getPreviousSnapshotKeyName( + keyInfo, bucketInfo, volumeId, + snapRenamedTable, previousKeyTable); + OmKeyInfo keyInfoPrevToPrevSnapshot = getPreviousSnapshotKeyName( + keyInfoPrevSnapshot, bucketInfo, volumeId, + prevRenamedTable, previousToPrevKeyTable); + // If the previous to previous snapshot doesn't + // have the key, then it is exclusive size for the + // previous snapshot. + if (keyInfoPrevToPrevSnapshot == null) { + exclusiveSizeMap.put(prevSnapKey, exclusiveSize); + exclusiveReplicatedSizeMap.put(prevSnapKey, + exclusiveReplicatedSize); + } + } + } + private OmKeyInfo getPreviousSnapshotKeyName( OmKeyInfo keyInfo, OmBucketInfo bucketInfo, long volumeId, Table snapRenamedTable, @@ -486,16 +503,20 @@ private OmKeyInfo getPreviousSnapshotKeyName( private void updateSnapshotExclusiveSize() { List snapshotPropertyList = new ArrayList<>(); - for (String dbKey: completedExclusiveSizeList) { + if (completedExclusiveSizeMap.isEmpty()) { + return; + } + + for (String dbKey: completedExclusiveSizeMap) { SnapshotProperty snapshotProperty = SnapshotProperty.newBuilder() .setSnapshotKey(dbKey) - .setExclusiveSize(exclusiveSizeList.get(dbKey)) + .setExclusiveSize(exclusiveSizeMap.get(dbKey)) .setExclusiveReplicatedSize( - exclusiveReplicatedSizeList.get(dbKey)) + exclusiveReplicatedSizeMap.get(dbKey)) .build(); snapshotPropertyList.add(snapshotProperty); - exclusiveSizeList.remove(dbKey); - exclusiveReplicatedSizeList.remove(dbKey); + exclusiveSizeMap.remove(dbKey); + exclusiveReplicatedSizeMap.remove(dbKey); } if (!snapshotPropertyList.isEmpty()) { @@ -512,7 +533,7 @@ private void updateSnapshotExclusiveSize() { submitRequest(omRequest); } - completedExclusiveSizeList.clear(); + completedExclusiveSizeMap.clear(); } private void updateDeepCleanedSnapshots(List deepCleanedSnapshots) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java similarity index 98% rename from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java index e4926c501902..0bcd0fa1c9e9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSizeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java @@ -57,9 +57,10 @@ import static org.mockito.Mockito.when; /** - * Tests TestOMSnapshotSizeRequest TestOMSnapshotSizeResponse class. + * Tests TestOMSnapshotSetPropertyRequest + * TestOMSnapshotSetPropertyResponse class. */ -public class TestOMSnapshotSizeRequestAndResponse { +public class TestOMSnapshotSetPropertyRequestAndResponse { private BatchOperation batchOperation; private OzoneManager ozoneManager; private OMMetadataManager omMetadataManager; From 0b9be1e6dd34ad2adcb09c292c54a32fb27cd0e9 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Thu, 28 Sep 2023 16:57:37 -0700 Subject: [PATCH 4/6] Change single API call to multiple API --- .../src/main/proto/OmClientProtocol.proto | 2 +- .../OMSnapshotSetPropertyRequest.java | 52 +++++++------- .../OMSnapshotSetPropertyResponse.java | 18 ++--- .../ozone/om/service/KeyDeletingService.java | 37 ++++------ ...SnapshotSetPropertyRequestAndResponse.java | 70 ++++++++++--------- 5 files changed, 83 insertions(+), 96 deletions(-) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index a0cafd929525..818164360f64 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1841,7 +1841,7 @@ message SnapshotPurgeRequest { } message SetSnapshotPropertyRequest { - repeated SnapshotProperty snapshotProperty = 1; + optional SnapshotProperty snapshotProperty = 1; } message SnapshotProperty { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java index af3fe5885f71..c0b1b4f3ae81 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotSetPropertyRequest.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.OMClientRequest; @@ -34,9 +35,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_SNAPSHOT_ERROR; /** * Updates the exclusive size of the snapshot. @@ -62,35 +62,33 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, setSnapshotPropertyRequest = getOmRequest() .getSetSnapshotPropertyRequest(); - List snapshotPropertyList = setSnapshotPropertyRequest - .getSnapshotPropertyList(); - Map updatedSnapInfos = new HashMap<>(); + SnapshotProperty snapshotProperty = setSnapshotPropertyRequest + .getSnapshotProperty(); + SnapshotInfo updatedSnapInfo = null; try { - for (SnapshotProperty snapshotProperty: snapshotPropertyList) { - String snapshotKey = snapshotProperty.getSnapshotKey(); - long exclusiveSize = snapshotProperty.getExclusiveSize(); - long exclusiveReplicatedSize = snapshotProperty - .getExclusiveReplicatedSize(); - SnapshotInfo snapshotInfo = metadataManager - .getSnapshotInfoTable().get(snapshotKey); - - if (snapshotInfo == null) { - LOG.error("SnapshotInfo for Snapshot: {} is not found", snapshotKey); - continue; - } + String snapshotKey = snapshotProperty.getSnapshotKey(); + long exclusiveSize = snapshotProperty.getExclusiveSize(); + long exclusiveReplicatedSize = snapshotProperty + .getExclusiveReplicatedSize(); + updatedSnapInfo = metadataManager.getSnapshotInfoTable() + .get(snapshotKey); - // Set Exclusive size. - snapshotInfo.setExclusiveSize(exclusiveSize); - snapshotInfo.setExclusiveReplicatedSize(exclusiveReplicatedSize); - // Update Table Cache - metadataManager.getSnapshotInfoTable().addCacheEntry( - new CacheKey<>(snapshotKey), - CacheValue.get(trxnLogIndex, snapshotInfo)); - updatedSnapInfos.put(snapshotKey, snapshotInfo); + if (updatedSnapInfo == null) { + LOG.error("SnapshotInfo for Snapshot: {} is not found", snapshotKey); + throw new OMException("SnapshotInfo for Snapshot: " + snapshotKey + + " is not found", INVALID_SNAPSHOT_ERROR); } + + // Set Exclusive size. + updatedSnapInfo.setExclusiveSize(exclusiveSize); + updatedSnapInfo.setExclusiveReplicatedSize(exclusiveReplicatedSize); + // Update Table Cache + metadataManager.getSnapshotInfoTable().addCacheEntry( + new CacheKey<>(snapshotKey), + CacheValue.get(trxnLogIndex, updatedSnapInfo)); omClientResponse = new OMSnapshotSetPropertyResponse( - omResponse.build(), updatedSnapInfos); + omResponse.build(), updatedSnapInfo); } catch (IOException ex) { omClientResponse = new OMSnapshotSetPropertyResponse( createErrorOMResponse(omResponse, ex)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java index fb84dfab8edc..ed2953b5415e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotSetPropertyResponse.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -28,7 +27,6 @@ import javax.annotation.Nonnull; import java.io.IOException; -import java.util.Map; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; @@ -37,30 +35,26 @@ */ @CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) public class OMSnapshotSetPropertyResponse extends OMClientResponse { - private final Map updatedSnapInfos; + private final SnapshotInfo updatedSnapInfo; public OMSnapshotSetPropertyResponse( @Nonnull OMResponse omResponse, - @Nonnull Map updatedSnapInfos) { + @Nonnull SnapshotInfo updatedSnapInfo) { super(omResponse); - this.updatedSnapInfos = updatedSnapInfos; + this.updatedSnapInfo = updatedSnapInfo; } public OMSnapshotSetPropertyResponse(@Nonnull OMResponse omResponse) { super(omResponse); checkStatusNotOK(); - this.updatedSnapInfos = null; + this.updatedSnapInfo = null; } @Override protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) - omMetadataManager; - for (Map.Entry entry : updatedSnapInfos.entrySet()) { - metadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, - entry.getKey(), entry.getValue()); - } + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, + updatedSnapInfo.getTableKey(), updatedSnapInfo); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 4d9064caf728..d1461f0f00ea 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -96,7 +96,7 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private final AtomicBoolean suspended; private final Map exclusiveSizeMap; private final Map exclusiveReplicatedSizeMap; - private final Set completedExclusiveSizeMap; + private final Set completedExclusiveSizeSet; public KeyDeletingService(OzoneManager ozoneManager, ScmBlockLocationProtocol scmClient, @@ -112,7 +112,7 @@ public KeyDeletingService(OzoneManager ozoneManager, this.suspended = new AtomicBoolean(false); this.exclusiveSizeMap = new HashMap<>(); this.exclusiveReplicatedSizeMap = new HashMap<>(); - this.completedExclusiveSizeMap = new HashSet<>(); + this.completedExclusiveSizeSet = new HashSet<>(); } /** @@ -356,13 +356,12 @@ private void processSnapshotDeepClean(int delCount) // snapshot. Here since we are only iterating through // deletedTable we can check the previous and previous to // previous snapshot to achieve the same. - // previousSnapshot - Current Snapshot for which we are - // calculating exclusive size. - // currSnapshot - Snapshot's deletedTable which is used to - // calc current(prev) snapshot's exclusive size. - // previousToPrevSnapshot - Previous to the previous snapshot - // to check if it's exclusive to the current(prev) - // snapshot or not. + // previousSnapshot - Snapshot for which exclusive size is + // getting calculating. + // currSnapshot - Snapshot's deletedTable is used to calculate + // previousSnapshot snapshot's exclusive size. + // previousToPrevSnapshot - Snapshot which is used to check + // if key is exclusive to previousSnapshot. if (previousSnapshot != null) { calculateExclusiveSize(previousSnapshot, previousToPrevSnapshot, keyInfo, bucketInfo, volumeId, @@ -404,7 +403,7 @@ private void processSnapshotDeepClean(int delCount) // will throw NPE when we submit request. if (previousSnapshot != null && exclusiveSizeMap .containsKey(previousSnapshot.getTableKey())) { - completedExclusiveSizeMap.add( + completedExclusiveSizeSet.add( previousSnapshot.getTableKey()); } } @@ -502,27 +501,20 @@ private OmKeyInfo getPreviousSnapshotKeyName( private void updateSnapshotExclusiveSize() { - List snapshotPropertyList = new ArrayList<>(); - if (completedExclusiveSizeMap.isEmpty()) { + if (completedExclusiveSizeSet.isEmpty()) { return; } - for (String dbKey: completedExclusiveSizeMap) { + for (String dbKey: completedExclusiveSizeSet) { SnapshotProperty snapshotProperty = SnapshotProperty.newBuilder() .setSnapshotKey(dbKey) .setExclusiveSize(exclusiveSizeMap.get(dbKey)) .setExclusiveReplicatedSize( exclusiveReplicatedSizeMap.get(dbKey)) .build(); - snapshotPropertyList.add(snapshotProperty); - exclusiveSizeMap.remove(dbKey); - exclusiveReplicatedSizeMap.remove(dbKey); - } - - if (!snapshotPropertyList.isEmpty()) { SetSnapshotPropertyRequest setSnapshotPropertyRequest = SetSnapshotPropertyRequest.newBuilder() - .addAllSnapshotProperty(snapshotPropertyList) + .setSnapshotProperty(snapshotProperty) .build(); OMRequest omRequest = OMRequest.newBuilder() @@ -530,10 +522,11 @@ private void updateSnapshotExclusiveSize() { .setSetSnapshotPropertyRequest(setSnapshotPropertyRequest) .setClientId(clientId.toString()) .build(); - submitRequest(omRequest); + exclusiveSizeMap.remove(dbKey); + exclusiveReplicatedSizeMap.remove(dbKey); } - completedExclusiveSizeMap.clear(); + completedExclusiveSizeSet.clear(); } private void updateDeepCleanedSnapshots(List deepCleanedSnapshots) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java index 0bcd0fa1c9e9..6ab86609dafe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotSetPropertyRequestAndResponse.java @@ -102,28 +102,30 @@ void setup(@TempDir File testDir) throws Exception { public void testValidateAndUpdateCache() throws IOException { createSnapshotDataForTest(); assertFalse(omMetadataManager.getSnapshotInfoTable().isEmpty()); - OzoneManagerProtocolProtos.OMRequest snapshotUpdateSizeRequest = + List snapshotUpdateSizeRequests = createSnapshotUpdateSizeRequest(); // Pre-Execute - OMSnapshotSetPropertyRequest omSnapshotSetPropertyRequest = new - OMSnapshotSetPropertyRequest(snapshotUpdateSizeRequest); - OMRequest modifiedOmRequest = omSnapshotSetPropertyRequest - .preExecute(ozoneManager); - omSnapshotSetPropertyRequest = new - OMSnapshotSetPropertyRequest(modifiedOmRequest); - - // Validate and Update Cache - OMSnapshotSetPropertyResponse omSnapshotSetPropertyResponse = - (OMSnapshotSetPropertyResponse) omSnapshotSetPropertyRequest - .validateAndUpdateCache(ozoneManager, 200L, - DOUBLE_BUFFER_HELPER); - - // Commit to DB. - batchOperation = omMetadataManager.getStore().initBatchOperation(); - omSnapshotSetPropertyResponse.checkAndUpdateDB(omMetadataManager, - batchOperation); - omMetadataManager.getStore().commitBatchOperation(batchOperation); + for (OMRequest request: snapshotUpdateSizeRequests) { + OMSnapshotSetPropertyRequest omSnapshotSetPropertyRequest = new + OMSnapshotSetPropertyRequest(request); + OMRequest modifiedOmRequest = omSnapshotSetPropertyRequest + .preExecute(ozoneManager); + omSnapshotSetPropertyRequest = new + OMSnapshotSetPropertyRequest(modifiedOmRequest); + + // Validate and Update Cache + OMSnapshotSetPropertyResponse omSnapshotSetPropertyResponse = + (OMSnapshotSetPropertyResponse) omSnapshotSetPropertyRequest + .validateAndUpdateCache(ozoneManager, 200L, + DOUBLE_BUFFER_HELPER); + + // Commit to DB. + batchOperation = omMetadataManager.getStore().initBatchOperation(); + omSnapshotSetPropertyResponse.checkAndUpdateDB(omMetadataManager, + batchOperation); + omMetadataManager.getStore().commitBatchOperation(batchOperation); + } // Check if the exclusive size is set. try (TableIterator> @@ -148,8 +150,9 @@ private void assertCacheValues(String dbKey) { .getExclusiveReplicatedSize()); } - private OMRequest createSnapshotUpdateSizeRequest() throws IOException { - List snapshotPropertyList = new ArrayList<>(); + private List createSnapshotUpdateSizeRequest() + throws IOException { + List omRequests = new ArrayList<>(); try (TableIterator> iterator = omMetadataManager.getSnapshotInfoTable().iterator()) { while (iterator.hasNext()) { @@ -159,21 +162,20 @@ private OMRequest createSnapshotUpdateSizeRequest() throws IOException { .setExclusiveSize(exclusiveSize) .setExclusiveReplicatedSize(exclusiveSizeAfterRepl) .build(); - snapshotPropertyList.add(snapshotSize); + SetSnapshotPropertyRequest snapshotUpdateSizeRequest = + SetSnapshotPropertyRequest.newBuilder() + .setSnapshotProperty(snapshotSize) + .build(); + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) + .setSetSnapshotPropertyRequest(snapshotUpdateSizeRequest) + .setClientId(UUID.randomUUID().toString()) + .build(); + omRequests.add(omRequest); } } - SetSnapshotPropertyRequest snapshotUpdateSizeRequest = - SetSnapshotPropertyRequest.newBuilder() - .addAllSnapshotProperty(snapshotPropertyList) - .build(); - - OMRequest omRequest = OMRequest.newBuilder() - .setCmdType(OzoneManagerProtocolProtos.Type.SetSnapshotProperty) - .setSetSnapshotPropertyRequest(snapshotUpdateSizeRequest) - .setClientId(UUID.randomUUID().toString()) - .build(); - - return omRequest; + return omRequests; } private void createSnapshotDataForTest() throws IOException { From 545181217fa9e9f416d6412a093b516049382320 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Mon, 2 Oct 2023 12:04:13 -0700 Subject: [PATCH 5/6] Address review comments. --- .../ozone/om/service/KeyDeletingService.java | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index d1461f0f00ea..429d5aa31b34 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -293,7 +294,6 @@ private void processSnapshotDeepClean(int delCount) Table prevRenamedTable = null; ReferenceCounted rcPrevOmSnapshot = null; - OmSnapshot omPreviousSnapshot = null; // Split RepeatedOmKeyInfo and update current snapshot // deletedKeyTable and next snapshot deletedKeyTable. @@ -302,7 +302,8 @@ private void processSnapshotDeepClean(int delCount) previousSnapshot.getVolumeName(), previousSnapshot.getBucketName(), getSnapshotPrefix(previousSnapshot.getName()), true); - omPreviousSnapshot = (OmSnapshot) rcPrevOmSnapshot.get(); + OmSnapshot omPreviousSnapshot = (OmSnapshot) + rcPrevOmSnapshot.get(); previousKeyTable = omPreviousSnapshot.getMetadataManager() .getKeyTable(bucketInfo.getBucketLayout()); @@ -313,13 +314,12 @@ private void processSnapshotDeepClean(int delCount) Table previousToPrevKeyTable = null; ReferenceCounted rcPrevToPrevOmSnapshot = null; - OmSnapshot omPreviousToPrevSnapshot = null; if (previousToPrevSnapshot != null) { rcPrevToPrevOmSnapshot = omSnapshotManager.checkForSnapshot( previousToPrevSnapshot.getVolumeName(), previousToPrevSnapshot.getBucketName(), getSnapshotPrefix(previousToPrevSnapshot.getName()), true); - omPreviousToPrevSnapshot = (OmSnapshot) + OmSnapshot omPreviousToPrevSnapshot = (OmSnapshot) rcPrevToPrevOmSnapshot.get(); previousToPrevKeyTable = omPreviousToPrevSnapshot @@ -349,20 +349,9 @@ private void processSnapshotDeepClean(int delCount) RepeatedOmKeyInfo newRepeatedOmKeyInfo = new RepeatedOmKeyInfo(); for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) { - // To calculate Exclusive Size for current snapshot, Check - // the next snapshot deletedTable if the deleted key is - // referenced in current snapshot and not referenced in the - // previous snapshot then that key is exclusive to the current - // snapshot. Here since we are only iterating through - // deletedTable we can check the previous and previous to - // previous snapshot to achieve the same. - // previousSnapshot - Snapshot for which exclusive size is - // getting calculating. - // currSnapshot - Snapshot's deletedTable is used to calculate - // previousSnapshot snapshot's exclusive size. - // previousToPrevSnapshot - Snapshot which is used to check - // if key is exclusive to previousSnapshot. if (previousSnapshot != null) { + // Calculates the exclusive size for the previous + // snapshot. See Java Doc for more info. calculateExclusiveSize(previousSnapshot, previousToPrevSnapshot, keyInfo, bucketInfo, volumeId, snapRenamedTable, previousKeyTable, prevRenamedTable, @@ -424,6 +413,21 @@ private void processSnapshotDeepClean(int delCount) updateDeepCleanedSnapshots(deepCleanedSnapshots); } + /** + * To calculate Exclusive Size for current snapshot, Check + * the next snapshot deletedTable if the deleted key is + * referenced in current snapshot and not referenced in the + * previous snapshot then that key is exclusive to the current + * snapshot. Here since we are only iterating through + * deletedTable we can check the previous and previous to + * previous snapshot to achieve the same. + * previousSnapshot - Snapshot for which exclusive size is + * getting calculating. + * currSnapshot - Snapshot's deletedTable is used to calculate + * previousSnapshot snapshot's exclusive size. + * previousToPrevSnapshot - Snapshot which is used to check + * if key is exclusive to previousSnapshot. + */ @SuppressWarnings("checkstyle:ParameterNumber") private void calculateExclusiveSize( SnapshotInfo previousSnapshot, @@ -505,7 +509,10 @@ private void updateSnapshotExclusiveSize() { return; } - for (String dbKey: completedExclusiveSizeSet) { + Iterator completedSnapshotIterator = + completedExclusiveSizeSet.iterator(); + while (completedSnapshotIterator.hasNext()) { + String dbKey = completedSnapshotIterator.next(); SnapshotProperty snapshotProperty = SnapshotProperty.newBuilder() .setSnapshotKey(dbKey) .setExclusiveSize(exclusiveSizeMap.get(dbKey)) @@ -525,8 +532,8 @@ private void updateSnapshotExclusiveSize() { submitRequest(omRequest); exclusiveSizeMap.remove(dbKey); exclusiveReplicatedSizeMap.remove(dbKey); + completedSnapshotIterator.remove(); } - completedExclusiveSizeSet.clear(); } private void updateDeepCleanedSnapshots(List deepCleanedSnapshots) { From 6d3918b55e0da51f28cbdd8007baf86e170ee7ce Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Tue, 3 Oct 2023 18:34:08 -0700 Subject: [PATCH 6/6] Use different client id for requests. --- .../hadoop/ozone/om/service/KeyDeletingService.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java index 429d5aa31b34..f03484347e82 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java @@ -91,7 +91,6 @@ public class KeyDeletingService extends AbstractKeyDeletingService { private static final int KEY_DELETING_CORE_POOL_SIZE = 1; private final KeyManager manager; - private static ClientId clientId = ClientId.randomId(); private final int keyLimitPerTask; private final AtomicLong deletedKeyCount; private final AtomicBoolean suspended; @@ -512,6 +511,7 @@ private void updateSnapshotExclusiveSize() { Iterator completedSnapshotIterator = completedExclusiveSizeSet.iterator(); while (completedSnapshotIterator.hasNext()) { + ClientId clientId = ClientId.randomId(); String dbKey = completedSnapshotIterator.next(); SnapshotProperty snapshotProperty = SnapshotProperty.newBuilder() .setSnapshotKey(dbKey) @@ -529,7 +529,7 @@ private void updateSnapshotExclusiveSize() { .setSetSnapshotPropertyRequest(setSnapshotPropertyRequest) .setClientId(clientId.toString()) .build(); - submitRequest(omRequest); + submitRequest(omRequest, clientId); exclusiveSizeMap.remove(dbKey); exclusiveReplicatedSizeMap.remove(dbKey); completedSnapshotIterator.remove(); @@ -538,6 +538,7 @@ private void updateSnapshotExclusiveSize() { private void updateDeepCleanedSnapshots(List deepCleanedSnapshots) { if (!deepCleanedSnapshots.isEmpty()) { + ClientId clientId = ClientId.randomId(); SnapshotPurgeRequest snapshotPurgeRequest = SnapshotPurgeRequest .newBuilder() .addAllUpdatedSnapshotDBKey(deepCleanedSnapshots) @@ -549,11 +550,11 @@ private void updateDeepCleanedSnapshots(List deepCleanedSnapshots) { .setClientId(clientId.toString()) .build(); - submitRequest(omRequest); + submitRequest(omRequest, clientId); } } - public void submitRequest(OMRequest omRequest) { + public void submitRequest(OMRequest omRequest, ClientId clientId) { try { if (isRatisEnabled()) { OzoneManagerRatisServer server = getOzoneManager().getOmRatisServer();