diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index d4ebf0be1b38..4f6ddd76bafd 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1625,8 +1625,8 @@ public OzoneInputStream getKey( .setParentObjectID(keyInfo.getParentObjectID()) .setFileChecksum(keyInfo.getFileChecksum()) .setOwnerName(keyInfo.getOwnerName()) + .addAllMetadata(keyInfo.getMetadata()) .build(); - dnKeyInfo.setMetadata(keyInfo.getMetadata()); dnKeyInfo.setKeyLocationVersions(keyLocationInfoGroups); blocks.put(dn, createInputStream(dnKeyInfo, Function.identity())); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 4c20b3808654..7c3e1b12b330 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -541,23 +541,29 @@ public static File createOMDir(String dirPath) { */ public static RepeatedOmKeyInfo prepareKeyForDelete(long bucketId, OmKeyInfo keyInfo, long trxnLogIndex) { + OmKeyInfo sanitizedKeyInfo = keyInfo; // If this key is in a GDPR enforced bucket, then before moving // KeyInfo to deletedTable, remove the GDPR related metadata and // FileEncryptionInfo from KeyInfo. if (Boolean.parseBoolean( keyInfo.getMetadata().get(OzoneConsts.GDPR_FLAG)) ) { - keyInfo.getMetadata().remove(OzoneConsts.GDPR_FLAG); - keyInfo.getMetadata().remove(OzoneConsts.GDPR_ALGORITHM); - keyInfo.getMetadata().remove(OzoneConsts.GDPR_SECRET); - keyInfo.clearFileEncryptionInfo(); + sanitizedKeyInfo = sanitizedKeyInfo.withMetadataMutations(metadata -> { + metadata.remove(OzoneConsts.GDPR_FLAG); + metadata.remove(OzoneConsts.GDPR_ALGORITHM); + metadata.remove(OzoneConsts.GDPR_SECRET); + }); + sanitizedKeyInfo.clearFileEncryptionInfo(); } // Set the updateID - keyInfo.setUpdateID(trxnLogIndex); + sanitizedKeyInfo.setUpdateID(trxnLogIndex); + if (sanitizedKeyInfo != keyInfo) { + keyInfo.setUpdateID(trxnLogIndex); + } //The key doesn't exist in deletedTable, so create a new instance. - return new RepeatedOmKeyInfo(keyInfo, bucketId); + return new RepeatedOmKeyInfo(sanitizedKeyInfo, bucketId); } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index 29c25778ed7c..0cc2a8ff4b96 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileChecksum; @@ -193,12 +194,38 @@ public String getOwnerName() { return ownerName; } - public void setCommittedKeyDeletedFlag(boolean val) { + public OmKeyInfo withCommittedKeyDeletedFlag(boolean val) { if (val) { - this.getMetadata().put(COMMITTED_KEY_DELETED_FLAG, "true"); - } else { - this.getMetadata().remove(COMMITTED_KEY_DELETED_FLAG); + return withMetadataMutations( + metadata -> metadata.put(COMMITTED_KEY_DELETED_FLAG, "true")); } + return withMetadataMutations( + metadata -> metadata.remove(COMMITTED_KEY_DELETED_FLAG)); + } + + /** + * Returns a new {@link OmKeyInfo} instance with metadata updated by the + * provided mutator. + * + * @param metadataUpdater a function that applies mutations to a copy of the metadata + * @return a new {@link OmKeyInfo} instance with updated metadata + */ + public OmKeyInfo withMetadataMutations( + Consumer> metadataUpdater) { + Objects.requireNonNull(metadataUpdater, "metadataUpdater == null"); + Map metadataCopy = new HashMap<>(getMetadata()); + metadataUpdater.accept(metadataCopy); + return toBuilder().setMetadata(metadataCopy).build(); + } + + /** + * Returns a new {@link OmKeyInfo} with metadata replaced by the provided + * map. + * @param metadata the metadata to set + * @return a new {@link OmKeyInfo} + */ + public OmKeyInfo withMetadata(Map metadata) { + return toBuilder().setMetadata(metadata).build(); } public boolean isDeletedKeyCommitted() { @@ -499,6 +526,28 @@ public Builder() { public Builder(OmKeyInfo obj) { super(obj); + this.volumeName = obj.volumeName; + this.bucketName = obj.bucketName; + this.keyName = obj.keyName; + this.ownerName = obj.ownerName; + this.dataSize = obj.dataSize; + this.creationTime = obj.creationTime; + this.modificationTime = obj.modificationTime; + this.replicationConfig = obj.replicationConfig; + this.encInfo = obj.encInfo; + this.fileName = obj.fileName; + this.fileChecksum = obj.fileChecksum; + this.isFile = obj.isFile; + this.expectedDataGeneration = obj.expectedDataGeneration; + if (obj.getTags() != null) { + this.tags.putAll(obj.getTags()); + } + this.acls.addAll(obj.getAcls()); + obj.keyLocationVersions.forEach(keyLocationVersion -> + this.omKeyLocationInfoGroups.add( + new OmKeyLocationInfoGroup(keyLocationVersion.getVersion(), + keyLocationVersion.getLocationList(), + keyLocationVersion.isMultipartKey()))); } public Builder setVolumeName(String volume) { @@ -569,6 +618,12 @@ public Builder addAllMetadata(Map newMetadata) { return this; } + @Override + public Builder setMetadata(Map map) { + super.setMetadata(map); + return this; + } + public Builder setFileEncryptionInfo(FileEncryptionInfo feInfo) { this.encInfo = feInfo; return this; @@ -882,44 +937,13 @@ public int hashCode() { /** * Return a new copy of the object. */ + public Builder toBuilder() { + return new Builder(this); + } + @Override public OmKeyInfo copyObject() { - OmKeyInfo.Builder builder = new OmKeyInfo.Builder(this) - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setOwnerName(ownerName) - .setCreationTime(creationTime) - .setModificationTime(modificationTime) - .setDataSize(dataSize) - .setReplicationConfig(replicationConfig) - .setFileEncryptionInfo(encInfo) - .setAcls(acls) - .setFileName(fileName) - .setFile(isFile); - - keyLocationVersions.forEach(keyLocationVersion -> - builder.addOmKeyLocationInfoGroup( - new OmKeyLocationInfoGroup(keyLocationVersion.getVersion(), - keyLocationVersion.getLocationList(), - keyLocationVersion.isMultipartKey()))); - - if (getMetadata() != null) { - getMetadata().forEach(builder::addMetadata); - } - - if (getTags() != null) { - getTags().forEach(builder::addTag); - } - - if (fileChecksum != null) { - builder.setFileChecksum(fileChecksum); - } - if (expectedDataGeneration != null) { - builder.setExpectedDataGeneration(expectedDataGeneration); - } - - return builder.build(); + return new Builder(this).build(); } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java index 85a102c9cc90..956fa5753a92 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithMetadata.java @@ -17,26 +17,31 @@ package org.apache.hadoop.ozone.om.helpers; +import com.google.common.collect.ImmutableMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import net.jcip.annotations.Immutable; /** * Mixin class to handle custom metadata. */ +@Immutable public abstract class WithMetadata { - private Map metadata; + private final Map metadata; protected WithMetadata() { - metadata = new ConcurrentHashMap<>(); + metadata = ImmutableMap.of(); } protected WithMetadata(Builder b) { - metadata = b.metadata; + metadata = b.metadata == null ? ImmutableMap.of() + : ImmutableMap.copyOf(b.metadata); } protected WithMetadata(WithMetadata other) { - metadata = new ConcurrentHashMap<>(other.getMetadata()); + metadata = other.getMetadata() == null ? ImmutableMap.of() + : ImmutableMap.copyOf(other.getMetadata()); } /** @@ -46,13 +51,6 @@ public final Map getMetadata() { return metadata; } - /** - * Set custom key value metadata. - */ - public final void setMetadata(Map metadata) { - this.metadata = metadata; - } - /** Builder for {@link WithMetadata}. */ public static class Builder { private final Map metadata; diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java index 5a23df4d9688..e35a2518cfe7 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java @@ -64,7 +64,8 @@ public void protobufConversion() throws IOException { assertEquals(key, keyAfterSerialization); assertFalse(key.isHsync()); - key.getMetadata().put(OzoneConsts.HSYNC_CLIENT_ID, "clientid"); + key = key.withMetadataMutations( + metadata -> metadata.put(OzoneConsts.HSYNC_CLIENT_ID, "clientid")); assertTrue(key.isHsync()); assertEquals(5678L, key.getExpectedDataGeneration()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index 34a5f4b74595..5cc2e8ec4352 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@ -4454,7 +4454,8 @@ public void testKeyReadWriteForGDPR() throws Exception { OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(BucketLayout.OBJECT_STORE) .get(omMetadataManager.getOzoneKey(volumeName, bucketName, keyName)); - omKeyInfo.getMetadata().remove(OzoneConsts.GDPR_FLAG); + omKeyInfo = omKeyInfo.withMetadataMutations( + metadata -> metadata.remove(OzoneConsts.GDPR_FLAG)); omMetadataManager.getKeyTable(BucketLayout.OBJECT_STORE) .put(omMetadataManager.getOzoneKey(volumeName, bucketName, keyName), diff --git a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfo.java b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfo.java index 4b8679408649..5fcaed544af5 100644 --- a/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfo.java +++ b/hadoop-ozone/interface-storage/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmPrefixInfo.java @@ -131,7 +131,9 @@ public void testGetProtobuf() { IAccessAuthorizer.ACLIdentityType.USER, username, IAccessAuthorizer.ACLType.WRITE, ACCESS); - omPrefixInfo.getMetadata().put("key", "value"); + omPrefixInfo = new OmPrefixInfo.Builder(omPrefixInfo) + .addMetadata("key", "value") + .build(); OzoneManagerStorageProtos.PersistedPrefixInfo pi = omPrefixInfo.getProtobuf(); assertEquals(testPath, pi.getName()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java index 5c96ae67fbe7..c3effc5d291a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMRecoverLeaseRequest.java @@ -228,7 +228,9 @@ private RecoverLeaseResponse doWork(OzoneManager ozoneManager, throw new OMException("Open Key " + keyName + " updated recently and is inside soft limit period", KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD); } - openKeyInfo.getMetadata().put(OzoneConsts.LEASE_RECOVERY, "true"); + openKeyInfo = openKeyInfo.toBuilder() + .addMetadata(OzoneConsts.LEASE_RECOVERY, "true") + .build(); openKeyInfo.setUpdateID(transactionLogIndex); openKeyInfo.setModificationTime(Time.now()); // add to cache. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index ed6185141d6d..aa197108195e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -173,7 +173,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut parentId, processed.keyInfo.getFileName(), hsyncClientId); OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKey); if (openKeyInfo != null) { - openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyInfo = openKeyInfo.withMetadataMutations( + metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); openKeyInfoMap.put(dbOpenKey, openKeyInfo); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 31f1d9d71801..a106903bce79 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -274,7 +274,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut dbOpenKeyToDeleteKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, Long.parseLong(keyToDelete.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID))); openKeyToDelete = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(dbOpenKeyToDeleteKey); - openKeyToDelete.getMetadata().put(OzoneConsts.OVERWRITTEN_HSYNC_KEY, "true"); + openKeyToDelete = openKeyToDelete.toBuilder() + .addMetadata(OzoneConsts.OVERWRITTEN_HSYNC_KEY, "true") + .build(); openKeyToDelete.setModificationTime(Time.now()); openKeyToDelete.setUpdateID(trxnLogIndex); omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( @@ -288,7 +290,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut if (isHSync) { if (!OmKeyHSyncUtil.isHSyncedPreviously(omKeyInfo, clientIdString, dbOpenKey)) { // Update open key as well if it is the first hsync of this key - omKeyInfo.getMetadata().put(OzoneConsts.HSYNC_CLIENT_ID, clientIdString); + omKeyInfo = omKeyInfo.withMetadataMutations( + metadata -> metadata.put(OzoneConsts.HSYNC_CLIENT_ID, clientIdString)); newOpenKeyInfo = omKeyInfo.copyObject(); } } @@ -298,8 +301,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // not persisted in the key table. omKeyInfo.setExpectedDataGeneration(null); - omKeyInfo.getMetadata().putAll(KeyValueUtil.getFromProtobuf( - commitKeyArgs.getMetadataList())); + omKeyInfo = omKeyInfo.withMetadataMutations(metadata -> + metadata.putAll(KeyValueUtil.getFromProtobuf( + commitKeyArgs.getMetadataList()))); omKeyInfo.setDataSize(commitKeyArgs.getDataSize()); // Update the block length for each block, return the allocated but @@ -343,9 +347,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut long totalNamespace = 0; if (!oldVerKeyInfo.getOmKeyInfoList().isEmpty()) { oldKeyVersionsToDeleteMap.put(delKeyName, oldVerKeyInfo); - for (OmKeyInfo olderKeyVersions : oldVerKeyInfo.getOmKeyInfoList()) { - olderKeyVersions.setCommittedKeyDeletedFlag(true); - totalSize += sumBlockLengths(olderKeyVersions); + List oldKeys = oldVerKeyInfo.getOmKeyInfoList(); + for (int i = 0; i < oldKeys.size(); i++) { + OmKeyInfo updatedOlderKeyVersions = + oldKeys.get(i).withCommittedKeyDeletedFlag(true); + oldKeys.set(i, updatedOlderKeyVersions); + totalSize += sumBlockLengths(updatedOlderKeyVersions); totalNamespace += 1; } } @@ -379,10 +386,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut dbOpenKey, trxnLogIndex); // Prevent hsync metadata from getting committed to the final key - omKeyInfo.getMetadata().remove(OzoneConsts.HSYNC_CLIENT_ID); - if (isRecovery) { - omKeyInfo.getMetadata().remove(OzoneConsts.LEASE_RECOVERY); - } + omKeyInfo = omKeyInfo.withMetadataMutations(metadata -> { + metadata.remove(OzoneConsts.HSYNC_CLIENT_ID); + if (isRecovery) { + metadata.remove(OzoneConsts.LEASE_RECOVERY); + } + }); } else if (newOpenKeyInfo != null) { // isHSync is true and newOpenKeyInfo is set, update OpenKeyTable omMetadataManager.getOpenKeyTable(getBucketLayout()).addCacheEntry( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java index a23716d40d1c..ffed1ebf5556 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java @@ -202,7 +202,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut Long.parseLong(keyToDelete.getMetadata().get(OzoneConsts.HSYNC_CLIENT_ID))); openKeyToDelete = OMFileRequest.getOmKeyInfoFromFileTable(true, omMetadataManager, dbOpenKeyToDeleteKey, keyName); - openKeyToDelete.getMetadata().put(OzoneConsts.OVERWRITTEN_HSYNC_KEY, "true"); + openKeyToDelete = openKeyToDelete.toBuilder() + .addMetadata(OzoneConsts.OVERWRITTEN_HSYNC_KEY, "true") + .build(); openKeyToDelete.setModificationTime(Time.now()); openKeyToDelete.setUpdateID(trxnLogIndex); OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, @@ -216,13 +218,15 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut if (isHSync) { if (!OmKeyHSyncUtil.isHSyncedPreviously(omKeyInfo, clientIdString, dbOpenFileKey)) { // Update open key as well if it is the first hsync of this key - omKeyInfo.getMetadata().put(OzoneConsts.HSYNC_CLIENT_ID, clientIdString); + omKeyInfo = omKeyInfo.withMetadataMutations( + metadata -> metadata.put(OzoneConsts.HSYNC_CLIENT_ID, clientIdString)); newOpenKeyInfo = omKeyInfo.copyObject(); } } - omKeyInfo.getMetadata().putAll(KeyValueUtil.getFromProtobuf( - commitKeyArgs.getMetadataList())); + omKeyInfo = omKeyInfo.withMetadataMutations(metadata -> + metadata.putAll(KeyValueUtil.getFromProtobuf( + commitKeyArgs.getMetadataList()))); omKeyInfo.setDataSize(commitKeyArgs.getDataSize()); List uncommitted = @@ -275,9 +279,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut long totalNamespace = 0; if (!oldVerKeyInfo.getOmKeyInfoList().isEmpty()) { oldKeyVersionsToDeleteMap.put(delKeyName, oldVerKeyInfo); - for (OmKeyInfo olderKeyVersions : oldVerKeyInfo.getOmKeyInfoList()) { - olderKeyVersions.setCommittedKeyDeletedFlag(true); - totalSize += sumBlockLengths(olderKeyVersions); + List oldKeys = oldVerKeyInfo.getOmKeyInfoList(); + for (int i = 0; i < oldKeys.size(); i++) { + OmKeyInfo updatedOlderKeyVersions = + oldKeys.get(i).withCommittedKeyDeletedFlag(true); + oldKeys.set(i, updatedOlderKeyVersions); + totalSize += sumBlockLengths(updatedOlderKeyVersions); totalNamespace += 1; } } @@ -322,9 +329,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut dbOpenFileKey, null, fileName, keyName, trxnLogIndex); // Prevent hsync metadata from getting committed to the final key - omKeyInfo.getMetadata().remove(OzoneConsts.HSYNC_CLIENT_ID); + omKeyInfo = omKeyInfo.withMetadataMutations( + metadata -> metadata.remove(OzoneConsts.HSYNC_CLIENT_ID)); if (isRecovery) { - omKeyInfo.getMetadata().remove(OzoneConsts.LEASE_RECOVERY); + omKeyInfo = omKeyInfo.withMetadataMutations( + metadata -> metadata.remove(OzoneConsts.LEASE_RECOVERY)); } } else if (newOpenKeyInfo != null) { // isHSync is true and newOpenKeyInfo is set, update OpenKeyTable diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 5c2065356c0c..fba766504e7c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -173,7 +173,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, hsyncClientId); OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); if (openKeyInfo != null) { - openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyInfo = openKeyInfo.withMetadataMutations( + metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); deletedOpenKeyInfo = openKeyInfo; } else { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java index 75b5966e005e..e2541ef051c8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java @@ -172,7 +172,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId, parentId, fileName, hsyncClientId); OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); if (openKeyInfo != null) { - openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyInfo = openKeyInfo.withMetadataMutations( + metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); deletedOpenKeyInfo = openKeyInfo; } else { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 2317a4815910..46cef44e1c57 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -980,10 +980,11 @@ protected OmKeyInfo prepareFileInfo( dbKeyInfo.setUpdateID(transactionLogIndex); dbKeyInfo.setReplicationConfig(replicationConfig); - // Construct a new metadata map from KeyArgs. - dbKeyInfo.getMetadata().clear(); - dbKeyInfo.getMetadata().putAll(KeyValueUtil.getFromProtobuf( - keyArgs.getMetadataList())); + // Construct a new metadata map from KeyArgs by rebuilding via toBuilder. + dbKeyInfo = dbKeyInfo.toBuilder() + .setMetadata(KeyValueUtil.getFromProtobuf( + keyArgs.getMetadataList())) + .build(); // Construct a new tags from KeyArgs // Clear the old one when the key is overwritten diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 427b2978f9c6..f0f447091ec7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -327,7 +327,8 @@ protected Pair markKeysAsDeletedInCache(OzoneManager ozoneManager String dbOpenKey = omMetadataManager.getOpenKey(volumeName, bucketName, keyName, hsyncClientId); OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); if (openKeyInfo != null) { - openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyInfo = openKeyInfo.withMetadataMutations( + metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); // Add to the map of open keys to be deleted. openKeyInfoMap.put(dbOpenKey, openKeyInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java index 0dbaa3c9d7bb..5d53ce7db09a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java @@ -116,7 +116,8 @@ protected Pair markKeysAsDeletedInCache( String dbOpenKey = omMetadataManager.getOpenFileName(volumeId, bucketId, parentId, fileName, hsyncClientId); OmKeyInfo openKeyInfo = openKeyTable.get(dbOpenKey); if (openKeyInfo != null) { - openKeyInfo.getMetadata().put(DELETED_HSYNC_KEY, "true"); + openKeyInfo = openKeyInfo.withMetadataMutations( + metadata -> metadata.put(DELETED_HSYNC_KEY, "true")); openKeyTable.addCacheEntry(dbOpenKey, openKeyInfo, trxnLogIndex); // Add to the map of open keys to be deleted. openKeyInfoMap.put(dbOpenKey, openKeyInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 5f715ded0b1b..d2c3d76e35c7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -161,8 +161,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut openKey + "entry is not found in the openKey table", KEY_NOT_FOUND); } - omKeyInfo.getMetadata().putAll(KeyValueUtil.getFromProtobuf( - keyArgs.getMetadataList())); + omKeyInfo = omKeyInfo.toBuilder() + .addAllMetadata(KeyValueUtil.getFromProtobuf( + keyArgs.getMetadataList())) + .build(); // set the data size and location info list omKeyInfo.setDataSize(keyArgs.getDataSize()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 6d41da38087c..d51055fdf2f2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -519,11 +519,14 @@ protected OmKeyInfo getOmKeyInfo(long trxnLogIndex, omKeyInfo.setModificationTime(keyArgs.getModificationTime()); omKeyInfo.setDataSize(dataSize); omKeyInfo.setReplicationConfig(dbOpenKeyInfo.getReplicationConfig()); + final String multipartHash = multipartUploadedKeyHash(partKeyInfoMap); + OmKeyInfo.Builder omKeyInfoBuilder = omKeyInfo.toBuilder(); if (dbOpenKeyInfo.getMetadata() != null) { - omKeyInfo.setMetadata(dbOpenKeyInfo.getMetadata()); + omKeyInfoBuilder.setMetadata(dbOpenKeyInfo.getMetadata()); } - omKeyInfo.getMetadata().put(OzoneConsts.ETAG, - multipartUploadedKeyHash(partKeyInfoMap)); + omKeyInfo = omKeyInfoBuilder + .addMetadata(OzoneConsts.ETAG, multipartHash) + .build(); if (dbOpenKeyInfo.getTags() != null) { omKeyInfo.setTags(dbOpenKeyInfo.getTags()); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/AbstractOMKeyDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/AbstractOMKeyDeleteResponse.java index e91bb403945e..3b5c457cb261 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/AbstractOMKeyDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/AbstractOMKeyDeleteResponse.java @@ -77,9 +77,9 @@ protected void addDeletionToBatch( // if RepeatedOMKeyInfo structure is null, we create a new instance, // if it is not null, then we simply add to the list and store this // instance in deletedTable. - omKeyInfo.setCommittedKeyDeletedFlag(isCommittedKey); - RepeatedOmKeyInfo repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(bucketId, - omKeyInfo, omKeyInfo.getUpdateID() + omKeyInfo = omKeyInfo.withCommittedKeyDeletedFlag(isCommittedKey); + RepeatedOmKeyInfo repeatedOmKeyInfo = OmUtils.prepareKeyForDelete( + bucketId, omKeyInfo, omKeyInfo.getUpdateID() ); String delKeyName = omMetadataManager.getOzoneDeletePathKey( omKeyInfo.getObjectID(), keyName); @@ -124,9 +124,9 @@ protected void addDeletionToBatch( // if RepeatedOMKeyInfo structure is null, we create a new instance, // if it is not null, then we simply add to the list and store this // instance in deletedTable. - omKeyInfo.setCommittedKeyDeletedFlag(isCommittedKey); - RepeatedOmKeyInfo repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(bucketId, - omKeyInfo, omKeyInfo.getUpdateID() + omKeyInfo = omKeyInfo.withCommittedKeyDeletedFlag(isCommittedKey); + RepeatedOmKeyInfo repeatedOmKeyInfo = OmUtils.prepareKeyForDelete( + bucketId, omKeyInfo, omKeyInfo.getUpdateID() ); omMetadataManager.getDeletedTable().putWithBatch( batchOperation, deleteKeyName, repeatedOmKeyInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 1beddd253130..4671dd8c3c90 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -157,8 +157,8 @@ public void processPaths( } for (OzoneManagerProtocolProtos.KeyInfo key : deletedSubFilesList) { - OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key); - keyInfo.setCommittedKeyDeletedFlag(true); + OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key) + .withCommittedKeyDeletedFlag(true); String ozoneDbKey = keySpaceOmMetadataManager.getOzonePathKey(volumeId, bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName()); keySpaceOmMetadataManager.getKeyTable(getBucketLayout()) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java index edfa51e7d02a..aeb33a06e6e2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMRecoverLeaseRequest.java @@ -560,15 +560,15 @@ private OMRequest doPreExecute(OMRequest originalOMRequest) throws Exception { String addToOpenFileTable(List locationList, boolean hsyncFlag) throws Exception { - OmKeyInfo omKeyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, + OmKeyInfo.Builder keyInfoBuilder = OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName, replicationConfig, new OmKeyLocationInfoGroup(version, new ArrayList<>(), false)) - .setParentObjectID(parentId) - .build(); - omKeyInfo.appendNewBlocks(locationList, false); + .setParentObjectID(parentId); if (hsyncFlag) { - omKeyInfo.getMetadata().put(OzoneConsts.HSYNC_CLIENT_ID, + keyInfoBuilder.addMetadata(OzoneConsts.HSYNC_CLIENT_ID, String.valueOf(clientID)); } + OmKeyInfo omKeyInfo = keyInfoBuilder.build(); + omKeyInfo.appendNewBlocks(locationList, false); OMRequestTestUtils.addFileToKeyTable( true, false, omKeyInfo.getFileName(), @@ -585,15 +585,15 @@ bucketName, keyName, replicationConfig, new OmKeyLocationInfoGroup(version, new String addToFileTable(List locationList, boolean hsyncFlag) throws Exception { - OmKeyInfo omKeyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, + OmKeyInfo.Builder keyInfoBuilder = OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName, replicationConfig, new OmKeyLocationInfoGroup(version, new ArrayList<>(), false)) - .setParentObjectID(parentId) - .build(); - omKeyInfo.appendNewBlocks(locationList, false); + .setParentObjectID(parentId); if (hsyncFlag) { - omKeyInfo.getMetadata().put(OzoneConsts.HSYNC_CLIENT_ID, + keyInfoBuilder.addMetadata(OzoneConsts.HSYNC_CLIENT_ID, String.valueOf(clientID)); } + OmKeyInfo omKeyInfo = keyInfoBuilder.build(); + omKeyInfo.appendNewBlocks(locationList, false); OMRequestTestUtils.addFileToKeyTable( false, false, omKeyInfo.getFileName(), diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 78d30550e82e..4b53233e98c8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@ -500,8 +500,7 @@ public void testOverwritingExistingMetadata( // We have to add the key to the key table, as validateAndUpdateCache only // updates the cache and not the DB. OmKeyInfo keyInfo = createOmKeyInfo(volumeName, bucketName, keyName, - replicationConfig).build(); - keyInfo.setMetadata(initialMetadata); + replicationConfig).setMetadata(initialMetadata).build(); omMetadataManager.getKeyTable(initialOmKeyCreateRequest.getBucketLayout()) .put(getOzoneKey(), keyInfo); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequestWithFSO.java index 20d142fdadb8..ac691020f37f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequestWithFSO.java @@ -80,8 +80,8 @@ public void createParentKey() throws Exception { @Test public void testRenameOpenFile() throws Exception { - fromKeyInfo.getMetadata().put(OzoneConsts.HSYNC_CLIENT_ID, - String.valueOf(1234)); + fromKeyInfo = fromKeyInfo.withMetadataMutations(metadata -> + metadata.put(OzoneConsts.HSYNC_CLIENT_ID, String.valueOf(1234))); addKeyToTable(fromKeyInfo); OMRequest modifiedOmRequest = doPreExecute(createRenameKeyRequest( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotUtils.java index b956a11237b6..aef6d97995e5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotUtils.java @@ -60,17 +60,16 @@ private OmKeyInfo createOmKeyInfo(boolean hsync, OmKeyLocationInfoGroup group, l } private OmKeyInfo createOmKeyInfo(boolean hsync, List group, long objectId) { - OmKeyInfo keyInfo = new OmKeyInfo.Builder() + OmKeyInfo.Builder builder = new OmKeyInfo.Builder() .setVolumeName("vol") .setBucketName("bucket") .setKeyName("key") .setOmKeyLocationInfos(group) - .setObjectID(objectId) - .build(); + .setObjectID(objectId); if (hsync) { - keyInfo.getMetadata().put(OzoneConsts.HSYNC_CLIENT_ID, "clientid"); + builder.addMetadata(OzoneConsts.HSYNC_CLIENT_ID, "clientid"); } - return keyInfo; + return builder.build(); } @Test