From 2eba8a70a9b558ae7071260e829dec88a9131f81 Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Wed, 9 Dec 2020 17:46:45 +0530 Subject: [PATCH 01/10] HDDS-4514. AllocateBlock : lookup and update open file table for the given path --- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 34 ++++++ .../fs/ozone/TestOzoneFileSystemV1.java | 3 + .../ratis/utils/OzoneManagerRatisUtils.java | 4 + .../request/file/OMFileCreateRequestV1.java | 3 + .../ozone/om/request/file/OMFileRequest.java | 77 +++++++++++- .../request/key/OMAllocateBlockRequest.java | 43 +++++-- .../request/key/OMAllocateBlockRequestV1.java | 91 ++++++++++++++ .../om/request/key/OMKeyCommitRequestV1.java | 76 +----------- .../response/key/OMAllocateBlockResponse.java | 16 +++ .../key/OMAllocateBlockResponseV1.java | 63 ++++++++++ .../request/file/TestOMFileCreateRequest.java | 22 ---- .../key/TestOMAllocateBlockRequest.java | 44 ++++--- .../key/TestOMAllocateBlockRequestV1.java | 114 ++++++++++++++++++ .../om/request/key/TestOMKeyRequest.java | 25 ++++ .../file/TestOMFileCreateResponseV1.java | 4 + .../key/TestOMAllocateBlockResponse.java | 39 ++++-- .../key/TestOMAllocateBlockResponseV1.java | 86 +++++++++++++ .../key/TestOMKeyCommitResponseV1.java | 4 + .../key/TestOMKeyDeleteResponseV1.java | 15 +++ 19 files changed, 628 insertions(+), 135 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index 016011078c8f..aed7dd660316 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -279,6 +280,7 @@ public void testFileSystem() throws Exception { testRenameDestinationParentDoesntExist(); testRenameToParentDir(); testSeekOnFileLength(); + testAllocateMoreThanOneBlock(); testDeleteRoot(); testRecursiveDelete(); @@ -662,6 +664,38 @@ public void testSeekOnFileLength() throws IOException { } } + public void testAllocateMoreThanOneBlock() throws IOException { + Path file = new Path("/file"); + String str = "TestOzoneFileSystemV1.testSeekOnFileLength"; + byte[] strBytes = str.getBytes(); + long numBlockAllocationsOrg = + cluster.getOzoneManager().getMetrics().getNumBlockAllocates(); + + try (FSDataOutputStream out1 = fs.create(file, FsPermission.getDefault(), + true, 8, (short) 3, 1, null)) { + for (int i = 0; i < 100000; i++) { + out1.write(strBytes); + } + } + + try (FSDataInputStream stream = fs.open(file)) { + FileStatus fileStatus = fs.getFileStatus(file); + long blkSize = fileStatus.getBlockSize(); + long fileLength = fileStatus.getLen(); + Assert.assertTrue("Block allocation should happen", + fileLength > blkSize); + + long newNumBlockAllocations = + cluster.getOzoneManager().getMetrics().getNumBlockAllocates(); + + Assert.assertTrue("Block allocation should happen", + (newNumBlockAllocations > numBlockAllocationsOrg)); + + stream.seek(fileLength); + assertEquals(-1, stream.read()); + } + } + public void testDeleteRoot() throws IOException { Path dir = new Path("/dir"); fs.mkdirs(dir); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java index d66ec4dbe9b4..ba6e47f261cf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java @@ -396,6 +396,9 @@ public void testFileSystem() throws Exception { testSeekOnFileLength(); deleteRootDir(); + testAllocateMoreThanOneBlock(); + deleteRootDir(); + testFileDelete(); deleteRootDir(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 7b5988c388ee..31120f13c5f7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.om.request.file.OMFileCreateRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeysDeleteRequest; import org.apache.hadoop.ozone.om.request.key.OMAllocateBlockRequest; +import org.apache.hadoop.ozone.om.request.key.OMAllocateBlockRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeyCommitRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyCommitRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequest; @@ -138,6 +139,9 @@ public static OMClientRequest createClientRequest(OMRequest omRequest) { case SetBucketProperty: return new OMBucketSetPropertyRequest(omRequest); case AllocateBlock: + if (omLayoutVersionV1) { + return new OMAllocateBlockRequestV1(omRequest); + } return new OMAllocateBlockRequest(omRequest); case CreateKey: return new OMKeyCreateRequest(omRequest); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java index a14fae5e8775..bff1d51ee4cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java @@ -141,6 +141,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, pathInfoV1.getLeafNodeName()); dbFileInfo = OMFileRequest.getOmKeyInfoFromFileTable(false, omMetadataManager, dbFileKey, keyName); + if (dbFileInfo != null) { + ozoneManager.getKeyManager().refresh(dbFileInfo); + } } // check if the file or directory already existed in OM diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index fc9bab0f2e83..dfd783109898 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -52,6 +53,8 @@ import javax.annotation.Nonnull; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; /** * Base class for file requests. @@ -545,6 +548,7 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, throws IOException { OmKeyInfo dbOmKeyInfo; + OmKeyInfo omKeyInfoCopy = null; if (openFileTable) { dbOmKeyInfo = omMetadataMgr.getOpenKeyTable().get(dbOpenFileKey); } else { @@ -556,9 +560,10 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, // For example, the user given key path is '/a/b/c/d/e/file1', then in DB // keyName field stores only the leaf node name, which is 'file1'. if (dbOmKeyInfo != null) { - dbOmKeyInfo.setKeyName(keyName); + omKeyInfoCopy = dbOmKeyInfo.copyObject(); + omKeyInfoCopy.setKeyName(keyName); } - return dbOmKeyInfo; + return omKeyInfoCopy; } /** @@ -851,4 +856,72 @@ private static boolean checkSubFileExists(OmKeyInfo omKeyInfo, public static boolean isImmediateChild(long parentId, long ancestorId) { return parentId == ancestorId; } + + + /** + * Check for directory exists with same name, if it exists throw error. + * + * @param keyName key name + * @param ozoneManager Ozone Manager + * @param reachedLastPathComponent true if the path component is a fileName + * @throws IOException if directory exists with same name + */ + private static void checkDirectoryAlreadyExists(String keyName, + OzoneManager ozoneManager, boolean reachedLastPathComponent) + throws IOException { + // Reached last component, which would be a file. Returns its parentID. + if (reachedLastPathComponent && ozoneManager.getEnableFileSystemPaths()) { + throw new OMException("Can not create file: " + keyName + + " as there is already directory in the given path", NOT_A_FILE); + } + } + + /** + * Get parent id for the user given path. + * + * @param bucketId bucket id + * @param pathComponents fie path elements + * @param keyName user given key name + * @param ozoneManager ozone manager + * @return lastKnownParentID + * @throws IOException DB failure or parent not exists in DirectoryTable + */ + public static long getParentID(long bucketId, Iterator pathComponents, + String keyName, OzoneManager ozoneManager) throws IOException { + + long lastKnownParentId = bucketId; + + // If no sub-dirs then bucketID is the root/parent. + if(!pathComponents.hasNext()){ + return bucketId; + } + + OmDirectoryInfo omDirectoryInfo; + OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + while (pathComponents.hasNext()) { + String nodeName = pathComponents.next().toString(); + boolean reachedLastPathComponent = !pathComponents.hasNext(); + String dbNodeName = + omMetadataManager.getOzonePathKey(lastKnownParentId, nodeName); + + omDirectoryInfo = omMetadataManager. + getDirectoryTable().get(dbNodeName); + if (omDirectoryInfo != null) { + checkDirectoryAlreadyExists(keyName, ozoneManager, + reachedLastPathComponent); + lastKnownParentId = omDirectoryInfo.getObjectID(); + } else { + // One of the sub-dir doesn't exists in DB. Immediate parent should + // exists for committing the key, otherwise will fail the operation. + if (!reachedLastPathComponent) { + throw new OMException("Failed to commit key, as parent directory of " + + keyName + " entry is not found in DirectoryTable", + KEY_NOT_FOUND); + } + break; + } + } + + return lastKnownParentId; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 1fd4b0754679..14180af2673c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -186,9 +186,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // Here we don't acquire bucket/volume lock because for a single client // allocateBlock is called in serial fashion. - openKeyName = omMetadataManager.getOpenKey(volumeName, bucketName, - keyName, clientID); - openKeyInfo = omMetadataManager.getOpenKeyTable().get(openKeyName); + openKeyName = getOpenKeyName(volumeName, bucketName, keyName, clientID, + ozoneManager); + openKeyInfo = getOpenKeyInfo(omMetadataManager, openKeyName, keyName); if (openKeyInfo == null) { throw new OMException("Open Key not found " + openKeyName, KEY_NOT_FOUND); @@ -216,17 +216,15 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, openKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); // Add to cache. - omMetadataManager.getOpenKeyTable().addCacheEntry( - new CacheKey<>(openKeyName), - new CacheValue<>(Optional.of(openKeyInfo), trxnLogIndex)); - + addOpenTableCacheEntry(trxnLogIndex, omMetadataManager, openKeyName, + openKeyInfo); omBucketInfo.incrUsedBytes(preAllocatedSpace); omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder() .setKeyLocation(blockLocation).build()); - omClientResponse = new OMAllocateBlockResponse(omResponse.build(), - openKeyInfo, clientID, omVolumeArgs, omBucketInfo.copyObject()); + omClientResponse = getOmClientResponse(clientID, omResponse, + openKeyInfo, omVolumeArgs, omBucketInfo.copyObject()); LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}", volumeName, bucketName, openKeyName); } catch (IOException ex) { @@ -250,4 +248,31 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, return omClientResponse; } + + protected OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager, + String openKeyName, String keyName) throws IOException { + return omMetadataManager.getOpenKeyTable().get(openKeyName); + } + + protected String getOpenKeyName(String volumeName, String bucketName, + String keyName, long clientID, OzoneManager ozoneManager) + throws IOException { + return ozoneManager.getMetadataManager().getOpenKey(volumeName, bucketName, + keyName, clientID); + } + + protected void addOpenTableCacheEntry(long trxnLogIndex, + OMMetadataManager omMetadataManager, String openKeyName, + OmKeyInfo openKeyInfo) { + omMetadataManager.getOpenKeyTable().addCacheEntry( + new CacheKey<>(openKeyName), + new CacheValue<>(Optional.of(openKeyInfo), trxnLogIndex)); + } + + protected OMClientResponse getOmClientResponse(long clientID, + OMResponse.Builder omResponse, OmKeyInfo openKeyInfo, + OmVolumeArgs omVolumeArgs, OmBucketInfo omBucketInfo) { + return new OMAllocateBlockResponse(omResponse.build(), + openKeyInfo, clientID, omVolumeArgs, omBucketInfo); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java new file mode 100644 index 000000000000..bcb91f47b2e0 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.key; + +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponseV1; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; + +/** + * Handles allocate block request layout version V1. + */ +public class OMAllocateBlockRequestV1 extends OMAllocateBlockRequest { + + private static final Logger LOG = + LoggerFactory.getLogger(OMAllocateBlockRequestV1.class); + + public OMAllocateBlockRequestV1(OMRequest omRequest) { + super(omRequest); + } + + protected OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager, + String openKeyName, String keyName) throws IOException { + String fileName = OzoneFSUtils.getFileName(keyName); + return OMFileRequest.getOmKeyInfoFromFileTable(true, + omMetadataManager, openKeyName, fileName); + } + + protected String getOpenKeyName(String volumeName, String bucketName, + String keyName, long clientID, OzoneManager ozoneManager) + throws IOException { + OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = + omMetadataManager.getBucketTable().get(bucketKey); + long bucketId = omBucketInfo.getObjectID(); + String fileName = OzoneFSUtils.getFileName(keyName); + Iterator pathComponents = Paths.get(keyName).iterator(); + long parentID = OMFileRequest.getParentID(bucketId, pathComponents, + keyName, ozoneManager); + return omMetadataManager.getOpenFileName(parentID, fileName, + clientID); + } + + protected void addOpenTableCacheEntry(long trxnLogIndex, + OMMetadataManager omMetadataManager, String openKeyName, + OmKeyInfo openKeyInfo) { + String fileName = openKeyInfo.getFileName(); + OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, openKeyName, + openKeyInfo, fileName, trxnLogIndex); + } + + @NotNull + protected OMClientResponse getOmClientResponse(long clientID, + OMResponse.Builder omResponse, OmKeyInfo openKeyInfo, + OmVolumeArgs omVolumeArgs, OmBucketInfo omBucketInfo) { + return new OMAllocateBlockResponseV1(omResponse.build(), + openKeyInfo, clientID, omVolumeArgs, omBucketInfo); + } +} \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java index ffabcd747262..1acfb1d0ec41 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java @@ -25,7 +25,6 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -53,7 +52,6 @@ import java.util.Map; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; /** @@ -130,8 +128,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, String fileName = OzoneFSUtils.getFileName(keyName); omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey); long bucketId = omBucketInfo.getObjectID(); - long parentID = getParentID(bucketId, pathComponents, keyName, - omMetadataManager, ozoneManager); + long parentID = OMFileRequest.getParentID(bucketId, pathComponents, + keyName, ozoneManager); String dbFileKey = omMetadataManager.getOzonePathKey(parentID, fileName); dbOpenFileKey = omMetadataManager.getOpenFileName(parentID, fileName, commitKeyRequest.getClientID()); @@ -199,74 +197,4 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, return omClientResponse; } - - - /** - * Check for directory exists with same name, if it exists throw error. - * - * @param keyName key name - * @param ozoneManager Ozone Manager - * @param reachedLastPathComponent true if the path component is a fileName - * @throws IOException if directory exists with same name - */ - private void checkDirectoryAlreadyExists(String keyName, - OzoneManager ozoneManager, - boolean reachedLastPathComponent) - throws IOException { - // Reached last component, which would be a file. Returns its parentID. - if (reachedLastPathComponent && ozoneManager.getEnableFileSystemPaths()) { - throw new OMException("Can not create file: " + keyName + - " as there is already directory in the given path", NOT_A_FILE); - } - } - - /** - * Get parent id for the user given path. - * - * @param bucketId bucket id - * @param pathComponents fie path elements - * @param keyName user given key name - * @param omMetadataManager metadata manager - * @return lastKnownParentID - * @throws IOException DB failure or parent not exists in DirectoryTable - */ - private long getParentID(long bucketId, Iterator pathComponents, - String keyName, OMMetadataManager omMetadataManager, - OzoneManager ozoneManager) - throws IOException { - - long lastKnownParentId = bucketId; - - // If no sub-dirs then bucketID is the root/parent. - if(!pathComponents.hasNext()){ - return bucketId; - } - - OmDirectoryInfo omDirectoryInfo; - while (pathComponents.hasNext()) { - String nodeName = pathComponents.next().toString(); - boolean reachedLastPathComponent = !pathComponents.hasNext(); - String dbNodeName = - omMetadataManager.getOzonePathKey(lastKnownParentId, nodeName); - - omDirectoryInfo = omMetadataManager. - getDirectoryTable().get(dbNodeName); - if (omDirectoryInfo != null) { - checkDirectoryAlreadyExists(keyName, ozoneManager, - reachedLastPathComponent); - lastKnownParentId = omDirectoryInfo.getObjectID(); - } else { - // One of the sub-dir doesn't exists in DB. Immediate parent should - // exists for committing the key, otherwise will fail the operation. - if (!reachedLastPathComponent) { - throw new OMException("Failed to commit key, as parent directory of " - + keyName + " entry is not found in DirectoryTable", - KEY_NOT_FOUND); - } - break; - } - } - - return lastKnownParentId; - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java index acc43eef8981..607a77925de9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java @@ -77,4 +77,20 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, omMetadataManager.getBucketKey(omVolumeArgs.getVolume(), omBucketInfo.getBucketName()), omBucketInfo); } + + protected OmKeyInfo getOmKeyInfo() { + return omKeyInfo; + } + + protected OmVolumeArgs getOmVolumeArgs() { + return omVolumeArgs; + } + + protected OmBucketInfo getOmBucketInfo() { + return omBucketInfo; + } + + protected long getClientID() { + return clientID; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java new file mode 100644 index 000000000000..d77d2e5546ba --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.key; + +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + +import javax.annotation.Nonnull; +import java.io.IOException; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE; + +/** + * Response for AllocateBlock request layout version V1. + */ +@CleanupTableInfo(cleanupTables = {OPEN_FILE_TABLE}) +public class OMAllocateBlockResponseV1 extends OMAllocateBlockResponse { + + public OMAllocateBlockResponseV1(@Nonnull OMResponse omResponse, + @Nonnull OmKeyInfo omKeyInfo, long clientID, + @Nonnull OmVolumeArgs omVolumeArgs, @Nonnull OmBucketInfo omBucketInfo) { + super(omResponse, omKeyInfo, clientID, omVolumeArgs, omBucketInfo); + } + + @Override + public void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + + OMFileRequest.addToOpenFileTable(omMetadataManager, batchOperation, + getOmKeyInfo(), getClientID()); + + // update volume usedBytes. + omMetadataManager.getVolumeTable().putWithBatch(batchOperation, + omMetadataManager.getVolumeKey(getOmVolumeArgs().getVolume()), + getOmVolumeArgs()); + // update bucket usedBytes. + omMetadataManager.getBucketTable().putWithBatch(batchOperation, + omMetadataManager.getBucketKey(getOmVolumeArgs().getVolume(), + getOmBucketInfo().getBucketName()), getOmBucketInfo()); + } +} \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java index 5010d0a7e3f3..43cbc85a15b9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java @@ -397,28 +397,6 @@ protected OMRequest createFileRequest( } - /** - * Verify path in open key table. Also, it returns OMKeyInfo for the given - * key path. - * - * @param key key name - * @param id client id - * @param doAssert if true then do assertion, otherwise it just skip. - * @return om key info for the given key path. - * @throws Exception DB failure - */ - protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, - boolean doAssert) - throws Exception { - String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, - key, id); - OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable().get(openKey); - if (doAssert) { - Assert.assertNotNull("Failed to find key in OpenKeyTable", omKeyInfo); - } - return omKeyInfo; - } - /** * Gets OMFileCreateRequest reference. * diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java index f309da041a71..ae992f82e725 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java @@ -24,6 +24,7 @@ import java.util.UUID; import org.apache.hadoop.ozone.OzoneConsts; +import org.jetbrains.annotations.NotNull; import org.junit.Assert; import org.junit.Test; @@ -57,21 +58,19 @@ public void testValidateAndUpdateCache() throws Exception { TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager); - TestOMRequestUtils.addKeyToTable(true, volumeName, bucketName, keyName, - clientID, replicationType, replicationFactor, omMetadataManager); + addKeyToOpenKeyTable(volumeName, bucketName); OMRequest modifiedOmRequest = doPreExecute(createAllocateBlockRequest()); OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(modifiedOmRequest); + getOmAllocateBlockRequest(modifiedOmRequest); // Check before calling validateAndUpdateCache. As adding DB entry has // not added any blocks, so size should be zero. - OmKeyInfo omKeyInfo = - omMetadataManager.getOpenKeyTable().get(omMetadataManager.getOpenKey( - volumeName, bucketName, keyName, clientID)); + OmKeyInfo omKeyInfo = verifyPathInOpenKeyTable(keyName, clientID, + true); List omKeyLocationInfo = omKeyInfo.getLatestVersionLocations().getLocationList(); @@ -87,10 +86,8 @@ public void testValidateAndUpdateCache() throws Exception { // Check open table whether new block is added or not. - omKeyInfo = - omMetadataManager.getOpenKeyTable().get(omMetadataManager.getOpenKey( - volumeName, bucketName, keyName, clientID)); - + omKeyInfo = verifyPathInOpenKeyTable(keyName, clientID, + true); // Check modification time Assert.assertEquals(modifiedOmRequest.getAllocateBlockRequest() @@ -119,6 +116,12 @@ public void testValidateAndUpdateCache() throws Exception { } + @NotNull + protected OMAllocateBlockRequest getOmAllocateBlockRequest( + OMRequest modifiedOmRequest) { + return new OMAllocateBlockRequest(modifiedOmRequest); + } + @Test public void testValidateAndUpdateCacheWithVolumeNotFound() throws Exception { @@ -126,7 +129,7 @@ public void testValidateAndUpdateCacheWithVolumeNotFound() throws Exception { doPreExecute(createAllocateBlockRequest()); OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(modifiedOmRequest); + getOmAllocateBlockRequest(modifiedOmRequest); OMClientResponse omAllocateBlockResponse = @@ -145,7 +148,7 @@ public void testValidateAndUpdateCacheWithBucketNotFound() throws Exception { doPreExecute(createAllocateBlockRequest()); OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(modifiedOmRequest); + getOmAllocateBlockRequest(modifiedOmRequest); // Added only volume to DB. @@ -168,7 +171,7 @@ public void testValidateAndUpdateCacheWithKeyNotFound() throws Exception { doPreExecute(createAllocateBlockRequest()); OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(modifiedOmRequest); + getOmAllocateBlockRequest(modifiedOmRequest); // Add volume, bucket entries to DB. TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, @@ -190,10 +193,11 @@ public void testValidateAndUpdateCacheWithKeyNotFound() throws Exception { * @return OMRequest - modified request returned from preExecute. * @throws Exception */ - private OMRequest doPreExecute(OMRequest originalOMRequest) throws Exception { + protected OMRequest doPreExecute(OMRequest originalOMRequest) + throws Exception { OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(originalOMRequest); + getOmAllocateBlockRequest(originalOMRequest); OMRequest modifiedOmRequest = omAllocateBlockRequest.preExecute(ozoneManager); @@ -228,7 +232,7 @@ private OMRequest doPreExecute(OMRequest originalOMRequest) throws Exception { } - private OMRequest createAllocateBlockRequest() { + protected OMRequest createAllocateBlockRequest() { KeyArgs keyArgs = KeyArgs.newBuilder() .setVolumeName(volumeName).setBucketName(bucketName) @@ -246,4 +250,12 @@ private OMRequest createAllocateBlockRequest() { .setAllocateBlockRequest(allocateBlockRequest).build(); } + + protected String addKeyToOpenKeyTable(String volumeName, String bucketName) + throws Exception { + TestOMRequestUtils.addKeyToTable(true, volumeName, bucketName, + keyName, clientID, replicationType, replicationFactor, + omMetadataManager); + return ""; + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java new file mode 100644 index 000000000000..84edf2439d55 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.ozone.om.request.key; + + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; +import org.jetbrains.annotations.NotNull; +import org.junit.Assert; + +/** + * Tests OMAllocateBlockRequest class layout version V1. + */ +public class TestOMAllocateBlockRequestV1 extends TestOMAllocateBlockRequest { + + @NotNull + @Override + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + return config; + } + + protected String addKeyToOpenKeyTable(String volumeName, String bucketName) + throws Exception { + // need to initialize parentID + String parentDir = keyName; + String fileName = "file1"; + keyName = parentDir + OzoneConsts.OM_KEY_PREFIX + fileName; + + // add parentDir to dirTable + long parentID = TestOMRequestUtils.addParentsToDirTable(volumeName, + bucketName, parentDir, omMetadataManager); + long txnId = 50; + long objectId = parentID + 1; + + OmKeyInfo omKeyInfoV1 = + TestOMRequestUtils.createOmKeyInfo(volumeName, bucketName, keyName, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, objectId, parentID, txnId, + Time.now()); + + // add key to openFileTable + TestOMRequestUtils.addFileToKeyTable(true, false, + fileName, omKeyInfoV1, clientID, txnLogId, omMetadataManager); + + return omMetadataManager.getOzonePathKey(parentID, fileName); + } + + @NotNull + protected OMAllocateBlockRequest getOmAllocateBlockRequest( + OzoneManagerProtocolProtos.OMRequest modifiedOmRequest) { + return new OMAllocateBlockRequestV1(modifiedOmRequest); + } + + @Override + protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, + boolean doAssert) throws Exception { + long bucketId = TestOMRequestUtils.getBucketId(volumeName, bucketName, + omMetadataManager); + String[] pathComponents = StringUtils.split(key, '/'); + long parentId = bucketId; + for (int indx = 0; indx < pathComponents.length; indx++) { + String pathElement = pathComponents[indx]; + // Reached last component, which is file name + if (indx == pathComponents.length - 1) { + String dbOpenFileName = omMetadataManager.getOpenFileName( + parentId, pathElement, id); + OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable() + .get(dbOpenFileName); + if (doAssert) { + Assert.assertNotNull("Invalid key!", omKeyInfo); + } + return omKeyInfo; + } else { + // directory + String dbKey = omMetadataManager.getOzonePathKey(parentId, + pathElement); + OmDirectoryInfo dirInfo = + omMetadataManager.getDirectoryTable().get(dbKey); + parentId = dirInfo.getObjectID(); + } + } + if (doAssert) { + Assert.fail("Invalid key!"); + } + return null; + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index 5f15c5047760..178a07205e64 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -26,11 +26,13 @@ import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.KeyManagerImpl; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.jetbrains.annotations.NotNull; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.rules.TemporaryFolder; @@ -176,6 +178,29 @@ protected OzoneConfiguration getOzoneConfiguration() { return new OzoneConfiguration(); } + + /** + * Verify path in open key table. Also, it returns OMKeyInfo for the given + * key path. + * + * @param key key name + * @param id client id + * @param doAssert if true then do assertion, otherwise it just skip. + * @return om key info for the given key path. + * @throws Exception DB failure + */ + protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, + boolean doAssert) + throws Exception { + String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, + key, id); + OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable().get(openKey); + if (doAssert) { + Assert.assertNotNull("Failed to find key in OpenKeyTable", omKeyInfo); + } + return omKeyInfo; + } + @After public void stop() { omMetrics.unRegister(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java index d2ab4658242d..bae1302eb992 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.om.response.key.TestOMKeyCreateResponse; @@ -69,6 +70,9 @@ protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // OzoneManager#start() is not invoked in this test cases. Hence need to + // explicitly set this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java index 494a308b28e1..1ac483d56bea 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java @@ -21,6 +21,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.util.Time; +import org.jetbrains.annotations.NotNull; import org.junit.Assert; import org.junit.Test; @@ -38,8 +39,7 @@ public class TestOMAllocateBlockResponse extends TestOMKeyResponse { @Test public void testAddToDBBatch() throws Exception { - OmKeyInfo omKeyInfo = TestOMRequestUtils.createOmKeyInfo(volumeName, - bucketName, keyName, replicationType, replicationFactor); + OmKeyInfo omKeyInfo = createOmKeyInfo(); OmVolumeArgs omVolumeArgs = OmVolumeArgs.newBuilder() .setOwnerName(keyName).setAdminName(keyName) .setVolume(volumeName).setCreationTime(Time.now()).build(); @@ -54,11 +54,10 @@ public void testAddToDBBatch() throws Exception { .setCmdType(OzoneManagerProtocolProtos.Type.AllocateBlock) .build(); OMAllocateBlockResponse omAllocateBlockResponse = - new OMAllocateBlockResponse(omResponse, omKeyInfo, clientID, - omVolumeArgs, omBucketInfo); + getOmAllocateBlockResponse(omKeyInfo, omVolumeArgs, + omBucketInfo, omResponse); - String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, - keyName, clientID); + String openKey = getOpenKey(); // Not adding key entry before to test whether commit is successful or not. Assert.assertFalse(omMetadataManager.getOpenKeyTable().isExist(openKey)); @@ -72,8 +71,7 @@ public void testAddToDBBatch() throws Exception { @Test public void testAddToDBBatchWithErrorResponse() throws Exception { - OmKeyInfo omKeyInfo = TestOMRequestUtils.createOmKeyInfo(volumeName, - bucketName, keyName, replicationType, replicationFactor); + OmKeyInfo omKeyInfo = createOmKeyInfo(); OmVolumeArgs omVolumeArgs = OmVolumeArgs.newBuilder() .setOwnerName(keyName).setAdminName(keyName) .setVolume(volumeName).setCreationTime(Time.now()).build(); @@ -88,12 +86,11 @@ public void testAddToDBBatchWithErrorResponse() throws Exception { .setCmdType(OzoneManagerProtocolProtos.Type.AllocateBlock) .build(); OMAllocateBlockResponse omAllocateBlockResponse = - new OMAllocateBlockResponse(omResponse, omKeyInfo, clientID, - omVolumeArgs, omBucketInfo); + getOmAllocateBlockResponse(omKeyInfo, omVolumeArgs, omBucketInfo, + omResponse); // Before calling addToDBBatch - String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, - keyName, clientID); + String openKey = getOpenKey(); Assert.assertFalse(omMetadataManager.getOpenKeyTable().isExist(openKey)); omAllocateBlockResponse.checkAndUpdateDB(omMetadataManager, batchOperation); @@ -105,4 +102,22 @@ public void testAddToDBBatchWithErrorResponse() throws Exception { Assert.assertFalse(omMetadataManager.getOpenKeyTable().isExist(openKey)); } + + protected OmKeyInfo createOmKeyInfo() throws Exception { + return TestOMRequestUtils.createOmKeyInfo(volumeName, + bucketName, keyName, replicationType, replicationFactor); + } + + protected String getOpenKey() throws Exception { + return omMetadataManager.getOpenKey(volumeName, bucketName, + keyName, clientID); + } + + @NotNull + protected OMAllocateBlockResponse getOmAllocateBlockResponse( + OmKeyInfo omKeyInfo, OmVolumeArgs omVolumeArgs, + OmBucketInfo omBucketInfo, OMResponse omResponse) { + return new OMAllocateBlockResponse(omResponse, omKeyInfo, clientID, + omVolumeArgs, omBucketInfo); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java new file mode 100644 index 000000000000..59519d40b50a --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file exgetOzoneConfigurationcept in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.key; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.util.Time; +import org.jetbrains.annotations.NotNull; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LAYOUT_VERSION; + +/** + * Tests OMAllocateBlockResponse layout version V1. + */ +public class TestOMAllocateBlockResponseV1 + extends TestOMAllocateBlockResponse { + + // logical ID, which really doesn't exist in dirTable + private long parentID = 10; + private String fileName = "file1"; + + protected OmKeyInfo createOmKeyInfo() throws Exception { + // need to initialize parentID + String parentDir = keyName; + keyName = parentDir + OzoneConsts.OM_KEY_PREFIX + fileName; + + long txnId = 50; + long objectId = parentID + 1; + + OmKeyInfo omKeyInfoV1 = + TestOMRequestUtils.createOmKeyInfo(volumeName, bucketName, keyName, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, objectId, parentID, txnId, + Time.now()); + return omKeyInfoV1; + } + + protected String getOpenKey() throws Exception { + return omMetadataManager.getOpenFileName( + parentID, fileName, clientID); + } + + @NotNull + protected OMAllocateBlockResponse getOmAllocateBlockResponse( + OmKeyInfo omKeyInfo, OmVolumeArgs omVolumeArgs, + OmBucketInfo omBucketInfo, OMResponse omResponse) { + return new OMAllocateBlockResponseV1(omResponse, omKeyInfo, clientID, + omVolumeArgs, omBucketInfo); + } + + @NotNull + @Override + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // OzoneManager#start() is not invoked in this test cases. Hence need to + // explicitly set this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + return config; + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java index 369faa96b0c1..b3fddce47598 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.util.Time; @@ -96,6 +97,9 @@ protected String getOzoneKey() { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // OzoneManager#start() is not invoked in this test cases. Hence need to + // explicitly set this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java index 37be730a24da..94e5a4b157a1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java @@ -18,12 +18,16 @@ package org.apache.hadoop.ozone.om.response.key; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.util.Time; +import org.jetbrains.annotations.NotNull; import org.junit.Assert; /** @@ -69,4 +73,15 @@ protected OmKeyInfo getOmKeyInfo() { getOmBucketInfo().getObjectID() + 1, getOmBucketInfo().getObjectID(), 100, Time.now()); } + + @NotNull + @Override + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // OzoneManager#start() is not invoked in this test cases. Hence need to + // explicitly set this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + return config; + } } From 8ab5ea12ad5ed7d2b6bd3d8780866d11ee645733 Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Wed, 9 Dec 2020 18:09:51 +0530 Subject: [PATCH 02/10] Updated RequestV1 test case to use new prefix tables --- .../om/request/key/OMAllocateBlockRequestV1.java | 2 +- .../om/request/file/TestOMFileCreateRequestV1.java | 4 ++++ .../request/key/TestOMAllocateBlockRequestV1.java | 4 ++++ .../om/request/key/TestOMKeyCommitRequestV1.java | 4 ++++ .../om/request/key/TestOMKeyDeleteRequestV1.java | 13 +++++++++++++ .../response/key/TestOMAllocateBlockResponseV1.java | 4 +--- 6 files changed, 27 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java index bcb91f47b2e0..f485bd7a5721 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java @@ -58,7 +58,7 @@ protected OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager, } protected String getOpenKeyName(String volumeName, String bucketName, - String keyName, long clientID, OzoneManager ozoneManager) + String keyName, long clientID, OzoneManager ozoneManager) throws IOException { OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java index 7ded386fdd8c..f790e8f86578 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.util.StringUtils; @@ -183,6 +184,9 @@ private OmDirectoryInfo getDirInfo(String key) protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // OzoneManager#start() is not invoked in this test cases. Hence need to + // explicitly set this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java index 84edf2439d55..0589e2f205a6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java @@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.util.StringUtils; @@ -43,6 +44,9 @@ public class TestOMAllocateBlockRequestV1 extends TestOMAllocateBlockRequest { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // OzoneManager#start() is not invoked in this test cases. Hence need to + // explicitly set this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java index f5168e1a3f04..47e7f81edce4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.util.Time; @@ -87,6 +88,9 @@ protected String addKeyToOpenKeyTable() throws Exception { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // OzoneManager#start() is not invoked in this test cases. Hence need to + // explicitly set this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java index dbba1434323e..821c11d7677f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java @@ -18,8 +18,11 @@ package org.apache.hadoop.ozone.om.request.key; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.util.Time; @@ -54,4 +57,14 @@ protected String addKeyToTable() throws Exception { fileName, omKeyInfo, -1, 50, omMetadataManager); return omKeyInfo.getPath(); } + + @Override + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // OzoneManager#start() is not invoked in this test cases. Hence need to + // explicitly set this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + return config; + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java index 59519d40b50a..fc2bfe157234 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java @@ -4,7 +4,7 @@ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file exgetOzoneConfigurationcept in compliance + * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 @@ -31,8 +31,6 @@ import org.apache.hadoop.util.Time; import org.jetbrains.annotations.NotNull; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LAYOUT_VERSION; - /** * Tests OMAllocateBlockResponse layout version V1. */ From dc40da0723b2318f155ec15be77978708983b640 Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Wed, 9 Dec 2020 23:07:59 +0530 Subject: [PATCH 03/10] Fixed review comment - removed volume usedbytes and added logs to check UT failure --- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 20 ++++++++++++++++++- .../key/OMAllocateBlockResponseV1.java | 4 ---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index aed7dd660316..958a0502029c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -587,6 +587,9 @@ protected void testListStatusOnRoot() throws Exception { */ protected void testListStatusOnLargeDirectory() throws Exception { Path root = new Path("/"); + FileStatus[] fileStatuses = o3fs.listStatus(root); + assertEquals("Root shouldn't have any child!", 0, + fileStatuses.length); Set paths = new TreeSet<>(); int numDirs = 5111; for(int i = 0; i < numDirs; i++) { @@ -596,7 +599,22 @@ protected void testListStatusOnLargeDirectory() throws Exception { rootItemCount++; } - FileStatus[] fileStatuses = o3fs.listStatus(root); + fileStatuses = o3fs.listStatus(root); + // Added logs for debugging failures, to check any sub-path mismatches. + Set actualPaths = new TreeSet<>(); + ArrayList actualPathList = new ArrayList<>(); + if (rootItemCount != fileStatuses.length) { + for (int i = 0; i < fileStatuses.length; i++) { + actualPaths.add(fileStatuses[i].getPath().getName()); + actualPathList.add(fileStatuses[i].getPath().getName()); + } + if (rootItemCount != actualPathList.size()) { + actualPaths.removeAll(paths); + actualPathList.removeAll(paths); + LOG.info("actualPaths:{}" + actualPaths); + LOG.info("actualPathList:{}" + actualPathList); + } + } assertEquals( "Total directories listed do not match the existing directories", rootItemCount, fileStatuses.length); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java index d77d2e5546ba..8f2f4ed70dbc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java @@ -51,10 +51,6 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, OMFileRequest.addToOpenFileTable(omMetadataManager, batchOperation, getOmKeyInfo(), getClientID()); - // update volume usedBytes. - omMetadataManager.getVolumeTable().putWithBatch(batchOperation, - omMetadataManager.getVolumeKey(getOmVolumeArgs().getVolume()), - getOmVolumeArgs()); // update bucket usedBytes. omMetadataManager.getBucketTable().putWithBatch(batchOperation, omMetadataManager.getBucketKey(getOmVolumeArgs().getVolume(), From 37d20395cdcaa9799bc26b675ee0b2bea0a4f2ef Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Thu, 10 Dec 2020 08:27:20 +0530 Subject: [PATCH 04/10] Added rootDirCleanup on TestOzoneFileSystem#testListStatusOnLargeDirectory to fix failure --- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 31 ++++++++++++++++--- .../fs/ozone/TestOzoneFileSystemV1.java | 26 ---------------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index 958a0502029c..fbdd45c00ed6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -587,9 +587,7 @@ protected void testListStatusOnRoot() throws Exception { */ protected void testListStatusOnLargeDirectory() throws Exception { Path root = new Path("/"); - FileStatus[] fileStatuses = o3fs.listStatus(root); - assertEquals("Root shouldn't have any child!", 0, - fileStatuses.length); + deleteRootDir(); // cleanup Set paths = new TreeSet<>(); int numDirs = 5111; for(int i = 0; i < numDirs; i++) { @@ -599,7 +597,7 @@ protected void testListStatusOnLargeDirectory() throws Exception { rootItemCount++; } - fileStatuses = o3fs.listStatus(root); + FileStatus[] fileStatuses = o3fs.listStatus(root); // Added logs for debugging failures, to check any sub-path mismatches. Set actualPaths = new TreeSet<>(); ArrayList actualPathList = new ArrayList<>(); @@ -624,6 +622,31 @@ protected void testListStatusOnLargeDirectory() throws Exception { } } + /** + * Cleanup files and directories. + * + * @throws IOException DB failure + */ + protected void deleteRootDir() throws IOException { + Path root = new Path("/"); + FileStatus[] fileStatuses = fs.listStatus(root); + + rootItemCount = 0; // reset to zero + + if (fileStatuses == null) { + return; + } + + for (FileStatus fStatus : fileStatuses) { + fs.delete(fStatus.getPath(), true); + } + + fileStatuses = fs.listStatus(root); + if (fileStatuses != null) { + Assert.assertEquals("Delete root failed!", 0, fileStatuses.length); + } + } + /** * Tests listStatus on a path with subdirs. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java index ba6e47f261cf..7ea66a3375cb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java @@ -409,32 +409,6 @@ public void testFileSystem() throws Exception { deleteRootDir(); } - /** - * Cleanup files and directories. - * - * @throws IOException DB failure - */ - protected void deleteRootDir() throws IOException { - Path root = new Path("/"); - FileStatus[] fileStatuses = fs.listStatus(root); - - if (fileStatuses == null) { - return; - } - - for (FileStatus fStatus : fileStatuses) { - fs.delete(fStatus.getPath(), true); - } - - fileStatuses = fs.listStatus(root); - if (fileStatuses != null) { - Assert.assertEquals("Delete root failed!", 0, fileStatuses.length); - rootItemCount = 0; - return; - } - rootItemCount = 0; - } - @NotNull @Override protected OzoneConfiguration getOzoneConfig() { From d6130b4dae2c92a2eeb58b490d6c08f40bb4b32a Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Thu, 10 Dec 2020 13:25:20 +0530 Subject: [PATCH 05/10] Fixed checkstyle warning --- .../java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java index 7ea66a3375cb..dfbd72000592 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java @@ -38,7 +38,6 @@ import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; -import java.io.IOException; import java.util.ArrayList; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; From 4b0c26b7b947d63ff182e17219cedd9ba030326b Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Mon, 21 Dec 2020 11:07:09 +0530 Subject: [PATCH 06/10] Fixed Bharat's review comments --- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 4 +- .../ozone/om/request/file/OMFileRequest.java | 43 ++--- .../request/key/OMAllocateBlockRequest.java | 42 +---- .../request/key/OMAllocateBlockRequestV1.java | 149 +++++++++++++++++- .../file/TestOMFileCreateRequestV1.java | 5 +- .../key/TestOMAllocateBlockRequestV1.java | 5 +- .../request/key/TestOMKeyCommitRequestV1.java | 5 +- .../request/key/TestOMKeyDeleteRequestV1.java | 5 +- .../file/TestOMFileCreateResponseV1.java | 5 +- .../key/TestOMAllocateBlockResponseV1.java | 5 +- .../key/TestOMKeyCommitResponseV1.java | 5 +- .../key/TestOMKeyDeleteResponseV1.java | 5 +- 12 files changed, 187 insertions(+), 91 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index fbdd45c00ed6..3d21c31be743 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -609,8 +609,8 @@ protected void testListStatusOnLargeDirectory() throws Exception { if (rootItemCount != actualPathList.size()) { actualPaths.removeAll(paths); actualPathList.removeAll(paths); - LOG.info("actualPaths:{}" + actualPaths); - LOG.info("actualPathList:{}" + actualPathList); + LOG.info("actualPaths: {}", actualPaths); + LOG.info("actualPathList: {}", actualPathList); } } assertEquals( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index dfd783109898..aadc1266562f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -548,7 +547,6 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, throws IOException { OmKeyInfo dbOmKeyInfo; - OmKeyInfo omKeyInfoCopy = null; if (openFileTable) { dbOmKeyInfo = omMetadataMgr.getOpenKeyTable().get(dbOpenFileKey); } else { @@ -560,10 +558,9 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, // For example, the user given key path is '/a/b/c/d/e/file1', then in DB // keyName field stores only the leaf node name, which is 'file1'. if (dbOmKeyInfo != null) { - omKeyInfoCopy = dbOmKeyInfo.copyObject(); - omKeyInfoCopy.setKeyName(keyName); + dbOmKeyInfo.setKeyName(keyName); } - return omKeyInfoCopy; + return dbOmKeyInfo; } /** @@ -857,37 +854,18 @@ public static boolean isImmediateChild(long parentId, long ancestorId) { return parentId == ancestorId; } - - /** - * Check for directory exists with same name, if it exists throw error. - * - * @param keyName key name - * @param ozoneManager Ozone Manager - * @param reachedLastPathComponent true if the path component is a fileName - * @throws IOException if directory exists with same name - */ - private static void checkDirectoryAlreadyExists(String keyName, - OzoneManager ozoneManager, boolean reachedLastPathComponent) - throws IOException { - // Reached last component, which would be a file. Returns its parentID. - if (reachedLastPathComponent && ozoneManager.getEnableFileSystemPaths()) { - throw new OMException("Can not create file: " + keyName + - " as there is already directory in the given path", NOT_A_FILE); - } - } - /** * Get parent id for the user given path. * * @param bucketId bucket id * @param pathComponents fie path elements * @param keyName user given key name - * @param ozoneManager ozone manager + * @param omMetadataManager om metadata manager * @return lastKnownParentID * @throws IOException DB failure or parent not exists in DirectoryTable */ public static long getParentID(long bucketId, Iterator pathComponents, - String keyName, OzoneManager ozoneManager) throws IOException { + String keyName, OMMetadataManager omMetadataManager) throws IOException { long lastKnownParentId = bucketId; @@ -897,7 +875,6 @@ public static long getParentID(long bucketId, Iterator pathComponents, } OmDirectoryInfo omDirectoryInfo; - OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); while (pathComponents.hasNext()) { String nodeName = pathComponents.next().toString(); boolean reachedLastPathComponent = !pathComponents.hasNext(); @@ -907,16 +884,18 @@ public static long getParentID(long bucketId, Iterator pathComponents, omDirectoryInfo = omMetadataManager. getDirectoryTable().get(dbNodeName); if (omDirectoryInfo != null) { - checkDirectoryAlreadyExists(keyName, ozoneManager, - reachedLastPathComponent); + if (reachedLastPathComponent) { + throw new OMException("Can not create file: " + keyName + + " as there is already directory in the given path", + NOT_A_FILE); + } lastKnownParentId = omDirectoryInfo.getObjectID(); } else { // One of the sub-dir doesn't exists in DB. Immediate parent should // exists for committing the key, otherwise will fail the operation. if (!reachedLastPathComponent) { - throw new OMException("Failed to commit key, as parent directory of " - + keyName + " entry is not found in DirectoryTable", - KEY_NOT_FOUND); + throw new OMException("Failed to find parent directory of " + + keyName + " in DirectoryTable", KEY_NOT_FOUND); } break; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 14180af2673c..46b97942a691 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -186,9 +186,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // Here we don't acquire bucket/volume lock because for a single client // allocateBlock is called in serial fashion. - openKeyName = getOpenKeyName(volumeName, bucketName, keyName, clientID, - ozoneManager); - openKeyInfo = getOpenKeyInfo(omMetadataManager, openKeyName, keyName); + openKeyName = ozoneManager.getMetadataManager().getOpenKey(volumeName, + bucketName, keyName, clientID); + openKeyInfo = omMetadataManager.getOpenKeyTable().get(openKeyName); if (openKeyInfo == null) { throw new OMException("Open Key not found " + openKeyName, KEY_NOT_FOUND); @@ -216,15 +216,16 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, openKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); // Add to cache. - addOpenTableCacheEntry(trxnLogIndex, omMetadataManager, openKeyName, - openKeyInfo); + omMetadataManager.getOpenKeyTable().addCacheEntry( + new CacheKey<>(openKeyName), + new CacheValue<>(Optional.of(openKeyInfo), trxnLogIndex)); omBucketInfo.incrUsedBytes(preAllocatedSpace); omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder() .setKeyLocation(blockLocation).build()); - omClientResponse = getOmClientResponse(clientID, omResponse, - openKeyInfo, omVolumeArgs, omBucketInfo.copyObject()); + omClientResponse = new OMAllocateBlockResponse(omResponse.build(), + openKeyInfo, clientID, omVolumeArgs, omBucketInfo); LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}", volumeName, bucketName, openKeyName); } catch (IOException ex) { @@ -248,31 +249,4 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, return omClientResponse; } - - protected OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager, - String openKeyName, String keyName) throws IOException { - return omMetadataManager.getOpenKeyTable().get(openKeyName); - } - - protected String getOpenKeyName(String volumeName, String bucketName, - String keyName, long clientID, OzoneManager ozoneManager) - throws IOException { - return ozoneManager.getMetadataManager().getOpenKey(volumeName, bucketName, - keyName, clientID); - } - - protected void addOpenTableCacheEntry(long trxnLogIndex, - OMMetadataManager omMetadataManager, String openKeyName, - OmKeyInfo openKeyInfo) { - omMetadataManager.getOpenKeyTable().addCacheEntry( - new CacheKey<>(openKeyName), - new CacheValue<>(Optional.of(openKeyInfo), trxnLogIndex)); - } - - protected OMClientResponse getOmClientResponse(long clientID, - OMResponse.Builder omResponse, OmKeyInfo openKeyInfo, - OmVolumeArgs omVolumeArgs, OmBucketInfo omBucketInfo) { - return new OMAllocateBlockResponse(omResponse.build(), - openKeyInfo, clientID, omVolumeArgs, omBucketInfo); - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java index f485bd7a5721..e95a781a985a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java @@ -18,17 +18,28 @@ package org.apache.hadoop.ozone.om.request.key; +import com.google.common.base.Preconditions; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponse; import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponseV1; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.*; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +47,13 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collections; import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; +import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; /** * Handles allocate block request layout version V1. @@ -50,14 +67,132 @@ public OMAllocateBlockRequestV1(OMRequest omRequest) { super(omRequest); } - protected OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager, + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { + + AllocateBlockRequest allocateBlockRequest = + getOmRequest().getAllocateBlockRequest(); + + KeyArgs keyArgs = + allocateBlockRequest.getKeyArgs(); + + OzoneManagerProtocolProtos.KeyLocation blockLocation = + allocateBlockRequest.getKeyLocation(); + Preconditions.checkNotNull(blockLocation); + + String volumeName = keyArgs.getVolumeName(); + String bucketName = keyArgs.getBucketName(); + String keyName = keyArgs.getKeyName(); + long clientID = allocateBlockRequest.getClientID(); + + OMMetrics omMetrics = ozoneManager.getMetrics(); + omMetrics.incNumBlockAllocateCalls(); + + AuditLogger auditLogger = ozoneManager.getAuditLogger(); + + Map auditMap = buildKeyArgsAuditMap(keyArgs); + auditMap.put(OzoneConsts.CLIENT_ID, String.valueOf(clientID)); + + OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + String openKeyName = null; + + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( + getOmRequest()); + OMClientResponse omClientResponse = null; + + OmKeyInfo openKeyInfo = null; + IOException exception = null; + OmVolumeArgs omVolumeArgs = null; + OmBucketInfo omBucketInfo = null; + boolean acquiredLock = false; + + try { + keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap); + volumeName = keyArgs.getVolumeName(); + bucketName = keyArgs.getBucketName(); + + // check Acl + checkKeyAclsInOpenKeyTable(ozoneManager, volumeName, bucketName, keyName, + IAccessAuthorizer.ACLType.WRITE, allocateBlockRequest.getClientID()); + + validateBucketAndVolume(omMetadataManager, volumeName, + bucketName); + + // Here we don't acquire bucket/volume lock because for a single client + // allocateBlock is called in serial fashion. With this approach, it + // won't make 'fail-fast' during race condition case on delete/rename op, + // assuming that later it will fail at the key commit operation. + openKeyName = getOpenKeyName(volumeName, bucketName, keyName, clientID, + ozoneManager); + openKeyInfo = getOpenKeyInfo(omMetadataManager, openKeyName, keyName); + if (openKeyInfo == null) { + throw new OMException("Open Key not found " + openKeyName, + KEY_NOT_FOUND); + } + + List newLocationList = Collections.singletonList( + OmKeyLocationInfo.getFromProtobuf(blockLocation)); + omVolumeArgs = getVolumeInfo(omMetadataManager, volumeName); + + acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, + volumeName, bucketName); + omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); + // check bucket and volume quota + long preAllocatedSpace = newLocationList.size() + * ozoneManager.getScmBlockSize() + * openKeyInfo.getFactor().getNumber(); + checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace); + // Append new block + openKeyInfo.appendNewBlocks(newLocationList, false); + + // Set modification time. + openKeyInfo.setModificationTime(keyArgs.getModificationTime()); + + // Set the UpdateID to current transactionLogIndex + openKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); + + // Add to cache. + addOpenTableCacheEntry(trxnLogIndex, omMetadataManager, openKeyName, + openKeyInfo); + omBucketInfo.incrUsedBytes(preAllocatedSpace); + + omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder() + .setKeyLocation(blockLocation).build()); + omClientResponse = getOmClientResponse(clientID, omResponse, + openKeyInfo, omVolumeArgs, omBucketInfo.copyObject()); + LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}", + volumeName, bucketName, openKeyName); + } catch (IOException ex) { + omMetrics.incNumBlockAllocateCallFails(); + exception = ex; + omClientResponse = new OMAllocateBlockResponse(createErrorOMResponse( + omResponse, exception)); + LOG.error("Allocate Block failed. Volume:{}, Bucket:{}, OpenKey:{}. " + + "Exception:{}", volumeName, bucketName, openKeyName, exception); + } finally { + addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, + omDoubleBufferHelper); + if (acquiredLock) { + omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, + bucketName); + } + } + + auditLog(auditLogger, buildAuditMessage(OMAction.ALLOCATE_BLOCK, auditMap, + exception, getOmRequest().getUserInfo())); + + return omClientResponse; + } + + private OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager, String openKeyName, String keyName) throws IOException { String fileName = OzoneFSUtils.getFileName(keyName); return OMFileRequest.getOmKeyInfoFromFileTable(true, omMetadataManager, openKeyName, fileName); } - protected String getOpenKeyName(String volumeName, String bucketName, + private String getOpenKeyName(String volumeName, String bucketName, String keyName, long clientID, OzoneManager ozoneManager) throws IOException { OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); @@ -68,12 +203,12 @@ protected String getOpenKeyName(String volumeName, String bucketName, String fileName = OzoneFSUtils.getFileName(keyName); Iterator pathComponents = Paths.get(keyName).iterator(); long parentID = OMFileRequest.getParentID(bucketId, pathComponents, - keyName, ozoneManager); + keyName, omMetadataManager); return omMetadataManager.getOpenFileName(parentID, fileName, clientID); } - protected void addOpenTableCacheEntry(long trxnLogIndex, + private void addOpenTableCacheEntry(long trxnLogIndex, OMMetadataManager omMetadataManager, String openKeyName, OmKeyInfo openKeyInfo) { String fileName = openKeyInfo.getFileName(); @@ -82,7 +217,7 @@ protected void addOpenTableCacheEntry(long trxnLogIndex, } @NotNull - protected OMClientResponse getOmClientResponse(long clientID, + private OMClientResponse getOmClientResponse(long clientID, OMResponse.Builder omResponse, OmKeyInfo openKeyInfo, OmVolumeArgs omVolumeArgs, OmBucketInfo omBucketInfo) { return new OMAllocateBlockResponseV1(omResponse.build(), diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java index f790e8f86578..046ac90e9a63 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java @@ -184,8 +184,9 @@ private OmDirectoryInfo getDirInfo(String key) protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - // OzoneManager#start() is not invoked in this test cases. Hence need to - // explicitly set this configuration to populate prefix tables. + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java index 0589e2f205a6..4e74979de2e3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java @@ -44,8 +44,9 @@ public class TestOMAllocateBlockRequestV1 extends TestOMAllocateBlockRequest { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - // OzoneManager#start() is not invoked in this test cases. Hence need to - // explicitly set this configuration to populate prefix tables. + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java index 47e7f81edce4..ed1e2bd3b771 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java @@ -88,8 +88,9 @@ protected String addKeyToOpenKeyTable() throws Exception { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - // OzoneManager#start() is not invoked in this test cases. Hence need to - // explicitly set this configuration to populate prefix tables. + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java index 821c11d7677f..7527e78e71a2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java @@ -62,8 +62,9 @@ protected String addKeyToTable() throws Exception { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - // OzoneManager#start() is not invoked in this test cases. Hence need to - // explicitly set this configuration to populate prefix tables. + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java index bae1302eb992..2f8bb743da4c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java @@ -70,8 +70,9 @@ protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - // OzoneManager#start() is not invoked in this test cases. Hence need to - // explicitly set this configuration to populate prefix tables. + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java index fc2bfe157234..773edaa0afad 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java @@ -75,8 +75,9 @@ protected OMAllocateBlockResponse getOmAllocateBlockResponse( protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - // OzoneManager#start() is not invoked in this test cases. Hence need to - // explicitly set this configuration to populate prefix tables. + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java index b3fddce47598..1e59ce8eaa7a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java @@ -97,8 +97,9 @@ protected String getOzoneKey() { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - // OzoneManager#start() is not invoked in this test cases. Hence need to - // explicitly set this configuration to populate prefix tables. + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java index 94e5a4b157a1..967fa102b914 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java @@ -79,8 +79,9 @@ protected OmKeyInfo getOmKeyInfo() { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - // OzoneManager#start() is not invoked in this test cases. Hence need to - // explicitly set this configuration to populate prefix tables. + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } From 3c46fb1807d3bbeb22e8fcca466acb8dbf6fdac0 Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Mon, 21 Dec 2020 11:44:55 +0530 Subject: [PATCH 07/10] Fixed compilation error --- .../hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java index 1acfb1d0ec41..34417cae00e9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java @@ -129,7 +129,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey); long bucketId = omBucketInfo.getObjectID(); long parentID = OMFileRequest.getParentID(bucketId, pathComponents, - keyName, ozoneManager); + keyName, omMetadataManager); String dbFileKey = omMetadataManager.getOzonePathKey(parentID, fileName); dbOpenFileKey = omMetadataManager.getOpenFileName(parentID, fileName, commitKeyRequest.getClientID()); From 702e038a39f255e5e3d0a91ab53c3ef019e0ef1d Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Mon, 21 Dec 2020 11:47:35 +0530 Subject: [PATCH 08/10] Fixed checkstyle warnings --- .../ozone/om/request/key/OMAllocateBlockRequestV1.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java index e95a781a985a..55d3108ee097 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java @@ -38,7 +38,11 @@ import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponse; import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponseV1; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.*; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; @@ -114,10 +118,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // check Acl checkKeyAclsInOpenKeyTable(ozoneManager, volumeName, bucketName, keyName, - IAccessAuthorizer.ACLType.WRITE, allocateBlockRequest.getClientID()); + IAccessAuthorizer.ACLType.WRITE, allocateBlockRequest.getClientID()); validateBucketAndVolume(omMetadataManager, volumeName, - bucketName); + bucketName); // Here we don't acquire bucket/volume lock because for a single client // allocateBlock is called in serial fashion. With this approach, it From 82cec8d1873f6d5cd1a8210076eba623996f665e Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Mon, 21 Dec 2020 19:22:17 +0530 Subject: [PATCH 09/10] Added omBucketInfo.copyObject() in OMAllocateBlockRequest --- .../ozone/om/request/key/OMAllocateBlockRequest.java | 10 +++++----- .../om/response/key/OMAllocateBlockResponseV1.java | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 46b97942a691..4ba989364c6a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -186,8 +186,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // Here we don't acquire bucket/volume lock because for a single client // allocateBlock is called in serial fashion. - openKeyName = ozoneManager.getMetadataManager().getOpenKey(volumeName, - bucketName, keyName, clientID); + openKeyName = omMetadataManager.getOpenKey(volumeName, bucketName, + keyName, clientID); openKeyInfo = omMetadataManager.getOpenKeyTable().get(openKeyName); if (openKeyInfo == null) { throw new OMException("Open Key not found " + openKeyName, @@ -217,15 +217,15 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // Add to cache. omMetadataManager.getOpenKeyTable().addCacheEntry( - new CacheKey<>(openKeyName), - new CacheValue<>(Optional.of(openKeyInfo), trxnLogIndex)); + new CacheKey<>(openKeyName), + new CacheValue<>(Optional.of(openKeyInfo), trxnLogIndex)); omBucketInfo.incrUsedBytes(preAllocatedSpace); omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder() .setKeyLocation(blockLocation).build()); omClientResponse = new OMAllocateBlockResponse(omResponse.build(), - openKeyInfo, clientID, omVolumeArgs, omBucketInfo); + openKeyInfo, clientID, omVolumeArgs, omBucketInfo.copyObject()); LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}", volumeName, bucketName, openKeyName); } catch (IOException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java index 8f2f4ed70dbc..cf92e2b43d40 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java @@ -56,4 +56,5 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, omMetadataManager.getBucketKey(getOmVolumeArgs().getVolume(), getOmBucketInfo().getBucketName()), getOmBucketInfo()); } -} \ No newline at end of file +} + From aaf77f4494b2213a5369deace4e9d700671a08bc Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Tue, 22 Dec 2020 07:37:28 +0530 Subject: [PATCH 10/10] Removed unnecessary spaces in OMAllocateBlockRequest --- .../hadoop/ozone/om/request/key/OMAllocateBlockRequest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 4ba989364c6a..1fd4b0754679 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -219,13 +219,14 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, omMetadataManager.getOpenKeyTable().addCacheEntry( new CacheKey<>(openKeyName), new CacheValue<>(Optional.of(openKeyInfo), trxnLogIndex)); + omBucketInfo.incrUsedBytes(preAllocatedSpace); omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder() .setKeyLocation(blockLocation).build()); - omClientResponse = new OMAllocateBlockResponse(omResponse.build(), openKeyInfo, clientID, omVolumeArgs, omBucketInfo.copyObject()); + LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}", volumeName, bucketName, openKeyName); } catch (IOException ex) {