From e803b1ec2155e7a93abc62fc9de2ebdcf94cb776 Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Thu, 5 Sep 2024 12:33:26 +0530 Subject: [PATCH 1/5] HDDS-11414. Key listing for FSO buckets fails with forward client --- .../hadoop/ozone/OzoneManagerVersion.java | 2 + .../hadoop/ozone/client/OzoneBucket.java | 27 ++++++-------- .../ozone/client/protocol/ClientProtocol.java | 20 ---------- .../hadoop/ozone/client/rpc/RpcClient.java | 24 +++++------- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 37 +++++++++++++++++++ .../ozone/om/helpers/OzoneFileStatus.java | 6 +++ .../hadoop/ozone/shell/TestOzoneShellHA.java | 2 +- .../src/main/proto/OmClientProtocol.proto | 1 + .../ozone/client/ClientProtocolStub.java | 8 ---- .../ozone/freon/OmBucketReadWriteKeyOps.java | 2 +- .../ozone/shell/keys/DeleteKeyHandler.java | 2 +- 11 files changed, 71 insertions(+), 60 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index eec2ceeb5e8d..982b559c7a58 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -44,6 +44,8 @@ public enum OzoneManagerVersion implements ComponentVersion { ATOMIC_REWRITE_KEY(6, "OzoneManager version that supports rewriting key as atomic operation"), HBASE_SUPPORT(7, "OzoneManager version that supports HBase integration"), + LIGHTWEIGHT_LIST_STATUS(8, "OzoneManager version that supports lightweight" + + " listStatus API."), FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 216b51b8e866..dd54c1f61a35 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -38,13 +38,11 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; -import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.WithMetadata; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -1357,10 +1355,9 @@ List getNextShallowListOfKeys(String prevKey) } } - // 2. Get immediate children by listStatusLight method - List statuses = - proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false, - startKey, listCacheSize, false); + // 2. Get immediate children by listStatus method + List statuses = proxy.listStatus(volumeName, name, + delimiterKeyPrefix, false, startKey, listCacheSize, false); if (addedKeyPrefix && statuses.size() > 0) { // previous round already include the startKey, so remove it @@ -1383,7 +1380,7 @@ protected void initDelimiterKeyPrefix() { } protected List buildKeysWithKeyPrefix( - List statuses) { + List statuses) { return statuses.stream() .map(OzoneBucket::toOzoneKey) .filter(key -> StringUtils.startsWith(key.getName(), getKeyPrefix())) @@ -1392,8 +1389,8 @@ protected List buildKeysWithKeyPrefix( } - private static OzoneKey toOzoneKey(OzoneFileStatusLight status) { - BasicOmKeyInfo keyInfo = status.getKeyInfo(); + private static OzoneKey toOzoneKey(OzoneFileStatus status) { + OmKeyInfo keyInfo = status.getKeyInfo(); String keyName = keyInfo.getKeyName(); final Map metadata; if (status.isDirectory()) { @@ -1600,8 +1597,8 @@ List getNextShallowListOfKeys(String prevKey) } // 2. Get immediate children by listStatus method. - List statuses = - proxy.listStatusLight(volumeName, name, getDelimiterKeyPrefix(), + List statuses = + proxy.listStatus(volumeName, name, getDelimiterKeyPrefix(), false, startKey, listCacheSize, false); if (!statuses.isEmpty()) { @@ -1760,7 +1757,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, startKey = startKey == null ? "" : startKey; // 1. Get immediate children of keyPrefix, starting with startKey - List statuses = proxy.listStatusLight(volumeName, + List statuses = proxy.listStatus(volumeName, name, keyPrefix, false, startKey, listCacheSize, true); boolean reachedLimitCacheSize = statuses.size() == listCacheSize; @@ -1779,8 +1776,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, // 4. Iterating over the resultStatuses list and add each key to the // resultList. for (int indx = 0; indx < statuses.size(); indx++) { - OzoneFileStatusLight status = statuses.get(indx); - BasicOmKeyInfo keyInfo = status.getKeyInfo(); + OzoneFileStatus status = statuses.get(indx); + OmKeyInfo keyInfo = status.getKeyInfo(); OzoneKey ozoneKey = toOzoneKey(status); keysResultList.add(ozoneKey); @@ -1808,7 +1805,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, } private void removeStartKeyIfExistsInStatusList(String startKey, - List statuses) { + List statuses) { if (!statuses.isEmpty()) { String firstElement = statuses.get(0).getKeyInfo().getKeyName(); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 8d9614b554a8..4d65cf9bc0d5 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -58,7 +58,6 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; -import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.TenantStateList; @@ -989,25 +988,6 @@ List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException; - /** - * Lightweight listStatus API. - * - * @param volumeName Volume name - * @param bucketName Bucket name - * @param keyName Absolute path of the entry to be listed - * @param recursive For a directory if true all the descendants of a - * particular directory are listed - * @param startKey Key from which listing needs to start. If startKey exists - * its status is included in the final list. - * @param numEntries Number of entries to list from the start key - * @param allowPartialPrefixes if partial prefixes should be allowed, - * this is needed in context of ListKeys - * @return list of file status - */ - List listStatusLight(String volumeName, - String bucketName, String keyName, boolean recursive, String startKey, - long numEntries, boolean allowPartialPrefixes) throws IOException; - /** * Add acl for Ozone object. Return true if acl is added successfully else * false. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 35db51b3e4de..b8f826a0a0ad 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -122,7 +122,6 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; -import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; @@ -2305,19 +2304,16 @@ public List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException { OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName); - return ozoneManagerClient - .listStatus(keyArgs, recursive, startKey, numEntries, - allowPartialPrefixes); - } - - @Override - public List listStatusLight(String volumeName, - String bucketName, String keyName, boolean recursive, String startKey, - long numEntries, boolean allowPartialPrefixes) throws IOException { - OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName); - return ozoneManagerClient - .listStatusLight(keyArgs, recursive, startKey, numEntries, - allowPartialPrefixes); + if (omVersion.compareTo(OzoneManagerVersion.LIGHTWEIGHT_LIST_STATUS) >= 0) { + return ozoneManagerClient.listStatusLight(keyArgs, recursive, startKey, + numEntries, allowPartialPrefixes) + .stream() + .map(OzoneFileStatus::new) + .collect(Collectors.toList()); + } else { + return ozoneManagerClient.listStatus(keyArgs, recursive, startKey, + numEntries, allowPartialPrefixes); + } } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index f52a142239b4..7dfedacb249a 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -88,6 +88,7 @@ public static Codec getCodec(boolean ignorePipeline) { * Support OFS use-case to identify if the key is a file or a directory. */ private boolean isFile; + private final String eTag; /** * Represents leaf node name. This also will be used when the keyName is @@ -133,6 +134,22 @@ private OmKeyInfo(Builder b) { this.ownerName = b.ownerName; this.tags = b.tags; this.expectedDataGeneration = b.expectedDataGeneration; + this.eTag = b.eTag; + } + + public OmKeyInfo(BasicOmKeyInfo b) { + this.volumeName = b.getVolumeName(); + this.bucketName = b.getBucketName(); + this.keyName = b.getKeyName(); + this.dataSize = b.getDataSize(); + this.creationTime = b.getCreationTime(); + this.modificationTime = b.getModificationTime(); + this.replicationConfig = b.getReplicationConfig(); + this.isFile = b.isFile(); + this.eTag = b.getETag(); + this.ownerName = b.getOwnerName(); + this.fileChecksum = null; + this.acls = null; } public String getVolumeName() { @@ -163,6 +180,10 @@ public long getReplicatedSize() { return QuotaUtil.getReplicatedSize(getDataSize(), replicationConfig); } + public String getETag() { + return eTag; + } + public void setDataSize(long size) { this.dataSize = size; } @@ -451,6 +472,7 @@ public String toString() { ", fileChecksum=" + fileChecksum + ", isFile=" + isFile + ", fileName='" + fileName + '\'' + + ", eTag=" + eTag + '\'' + ", acls=" + acls + '}'; } @@ -476,6 +498,7 @@ public static class Builder extends WithParentObjectId.Builder { private FileChecksum fileChecksum; private boolean isFile; + private String eTag; private final Map tags = new HashMap<>(); private Long expectedDataGeneration = null; @@ -606,6 +629,11 @@ public Builder setFile(boolean isAFile) { return this; } + public Builder setETag(String etag) { + this.eTag = etag; + return this; + } + public Builder addTag(String key, String value) { tags.put(key, value); return this; @@ -702,6 +730,9 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, } else { kb.setFactor(ReplicationConfig.getLegacyFactor(replicationConfig)); } + if (eTag != null) { + kb.setETag(eTag); + } kb.setLatestVersion(latestVersion) .addAllKeyLocationList(keyLocations) .setCreationTime(creationTime) @@ -779,6 +810,9 @@ public static OmKeyInfo getFromProtobuf(KeyInfo keyInfo) throws IOException { if (keyInfo.hasIsFile()) { builder.setFile(keyInfo.getIsFile()); } + if (keyInfo.hasETag()) { + builder.setETag(keyInfo.getETag()); + } if (keyInfo.hasExpectedDataGeneration()) { builder.setExpectedDataGeneration(keyInfo.getExpectedDataGeneration()); } @@ -903,6 +937,9 @@ public OmKeyInfo copyObject() { if (expectedDataGeneration != null) { builder.setExpectedDataGeneration(expectedDataGeneration); } + if (eTag != null) { + builder.setETag(eTag); + } return builder.build(); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java index dcb7e2a2c1e3..a23afff88be6 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java @@ -53,6 +53,12 @@ public OzoneFileStatus(OmKeyInfo keyInfo, this.blockSize = blockSize; } + public OzoneFileStatus(OzoneFileStatusLight b) { + this.keyInfo = new OmKeyInfo(b.getKeyInfo()); + this.isDirectory = b.isDirectory(); + this.blockSize = b.getBlockSize(); + } + public OmKeyInfo getKeyInfo() { return keyInfo; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 89f068cdedfa..4f86d5497138 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -2459,7 +2459,7 @@ public void testKeyDeleteLegacyWithEnableFileSystemPath() throws IOException { // Check number of keys OzoneVolume volume = client.getObjectStore().getVolume(volumeName); OzoneBucket bucket = volume.getBucket(bucketName); - List files = bucket.listStatus("", true, "", 5); + List files = bucket.listStatus("", true, "", 5, false); // Two keys should still exist, dirPath and keyPath assertEquals(2, files.size()); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index eeddc5500527..05c4f9ce9728 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1152,6 +1152,7 @@ message KeyInfo { // This allows a key to be created an committed atomically if the original has not // been modified. optional uint64 expectedDataGeneration = 22; + optional string eTag = 23; } message BasicKeyInfo { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index e3e3537b1c3b..f4a919733e2a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -42,7 +42,6 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; -import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.TenantStateList; @@ -564,13 +563,6 @@ public List listStatus(String volumeName, String bucketName, return null; } - @Override - public List listStatusLight(String volumeName, - String bucketName, String keyName, boolean recursive, String startKey, - long numEntries, boolean allowPartialPrefixes) throws IOException { - return null; - } - @Override public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException { return false; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java index abc7ad5002fe..6eb508a93894 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java @@ -130,7 +130,7 @@ protected int getReadCount(int readCount, String readPath) throws IOException { List ozoneFileStatusList = bucket.listStatus( OzoneConsts.OM_KEY_PREFIX + readPath + OzoneConsts.OM_KEY_PREFIX, true, - "/", keyCountForRead); + "/", keyCountForRead, false); readCount += ozoneFileStatusList.size(); return readCount; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java index 4c795f1e82b4..3022dbf5c720 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java @@ -106,7 +106,7 @@ private void deleteFSOKey(OzoneBucket bucket, String keyName) if (bucket.getFileStatus(keyName).isDirectory()) { List ozoneFileStatusList = - bucket.listStatus(keyName, false, "", 1); + bucket.listStatus(keyName, false, "", 1, false); if (ozoneFileStatusList != null && !ozoneFileStatusList.isEmpty()) { out().printf("Directory is not empty %n"); return; From 895c25ac8ee35c3341d95c19b921edd78ab28aef Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Fri, 6 Sep 2024 14:50:28 +0530 Subject: [PATCH 2/5] Add omVersion checks to listStatusLight instead of listStatus --- .../hadoop/ozone/client/OzoneBucket.java | 27 ++++++++------ .../ozone/client/protocol/ClientProtocol.java | 20 ++++++++++ .../hadoop/ozone/client/rpc/RpcClient.java | 19 ++++++++-- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 37 ------------------- .../ozone/om/helpers/OzoneFileStatus.java | 6 --- .../hadoop/ozone/shell/TestOzoneShellHA.java | 2 +- .../src/main/proto/OmClientProtocol.proto | 1 - .../ozone/client/ClientProtocolStub.java | 8 ++++ .../ozone/freon/OmBucketReadWriteKeyOps.java | 2 +- .../ozone/shell/keys/DeleteKeyHandler.java | 2 +- 10 files changed, 61 insertions(+), 63 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index dd54c1f61a35..216b51b8e866 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -38,11 +38,13 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.WithMetadata; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -1355,9 +1357,10 @@ List getNextShallowListOfKeys(String prevKey) } } - // 2. Get immediate children by listStatus method - List statuses = proxy.listStatus(volumeName, name, - delimiterKeyPrefix, false, startKey, listCacheSize, false); + // 2. Get immediate children by listStatusLight method + List statuses = + proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false, + startKey, listCacheSize, false); if (addedKeyPrefix && statuses.size() > 0) { // previous round already include the startKey, so remove it @@ -1380,7 +1383,7 @@ protected void initDelimiterKeyPrefix() { } protected List buildKeysWithKeyPrefix( - List statuses) { + List statuses) { return statuses.stream() .map(OzoneBucket::toOzoneKey) .filter(key -> StringUtils.startsWith(key.getName(), getKeyPrefix())) @@ -1389,8 +1392,8 @@ protected List buildKeysWithKeyPrefix( } - private static OzoneKey toOzoneKey(OzoneFileStatus status) { - OmKeyInfo keyInfo = status.getKeyInfo(); + private static OzoneKey toOzoneKey(OzoneFileStatusLight status) { + BasicOmKeyInfo keyInfo = status.getKeyInfo(); String keyName = keyInfo.getKeyName(); final Map metadata; if (status.isDirectory()) { @@ -1597,8 +1600,8 @@ List getNextShallowListOfKeys(String prevKey) } // 2. Get immediate children by listStatus method. - List statuses = - proxy.listStatus(volumeName, name, getDelimiterKeyPrefix(), + List statuses = + proxy.listStatusLight(volumeName, name, getDelimiterKeyPrefix(), false, startKey, listCacheSize, false); if (!statuses.isEmpty()) { @@ -1757,7 +1760,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, startKey = startKey == null ? "" : startKey; // 1. Get immediate children of keyPrefix, starting with startKey - List statuses = proxy.listStatus(volumeName, + List statuses = proxy.listStatusLight(volumeName, name, keyPrefix, false, startKey, listCacheSize, true); boolean reachedLimitCacheSize = statuses.size() == listCacheSize; @@ -1776,8 +1779,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, // 4. Iterating over the resultStatuses list and add each key to the // resultList. for (int indx = 0; indx < statuses.size(); indx++) { - OzoneFileStatus status = statuses.get(indx); - OmKeyInfo keyInfo = status.getKeyInfo(); + OzoneFileStatusLight status = statuses.get(indx); + BasicOmKeyInfo keyInfo = status.getKeyInfo(); OzoneKey ozoneKey = toOzoneKey(status); keysResultList.add(ozoneKey); @@ -1805,7 +1808,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, } private void removeStartKeyIfExistsInStatusList(String startKey, - List statuses) { + List statuses) { if (!statuses.isEmpty()) { String firstElement = statuses.get(0).getKeyInfo().getKeyName(); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 4d65cf9bc0d5..8d9614b554a8 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -58,6 +58,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.TenantStateList; @@ -988,6 +989,25 @@ List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException; + /** + * Lightweight listStatus API. + * + * @param volumeName Volume name + * @param bucketName Bucket name + * @param keyName Absolute path of the entry to be listed + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @param allowPartialPrefixes if partial prefixes should be allowed, + * this is needed in context of ListKeys + * @return list of file status + */ + List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, + long numEntries, boolean allowPartialPrefixes) throws IOException; + /** * Add acl for Ozone object. Return true if acl is added successfully else * false. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index b8f826a0a0ad..c5d0b3718f31 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -122,6 +122,7 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; @@ -2304,15 +2305,25 @@ public List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException { OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName); + return ozoneManagerClient + .listStatus(keyArgs, recursive, startKey, numEntries, + allowPartialPrefixes); + } + + @Override + public List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, + long numEntries, boolean allowPartialPrefixes) throws IOException { + OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName); if (omVersion.compareTo(OzoneManagerVersion.LIGHTWEIGHT_LIST_STATUS) >= 0) { return ozoneManagerClient.listStatusLight(keyArgs, recursive, startKey, + numEntries, allowPartialPrefixes); + } else { + return ozoneManagerClient.listStatus(keyArgs, recursive, startKey, numEntries, allowPartialPrefixes) .stream() - .map(OzoneFileStatus::new) + .map(OzoneFileStatusLight::fromOzoneFileStatus) .collect(Collectors.toList()); - } else { - return ozoneManagerClient.listStatus(keyArgs, recursive, startKey, - numEntries, allowPartialPrefixes); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index 7dfedacb249a..f52a142239b4 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -88,7 +88,6 @@ public static Codec getCodec(boolean ignorePipeline) { * Support OFS use-case to identify if the key is a file or a directory. */ private boolean isFile; - private final String eTag; /** * Represents leaf node name. This also will be used when the keyName is @@ -134,22 +133,6 @@ private OmKeyInfo(Builder b) { this.ownerName = b.ownerName; this.tags = b.tags; this.expectedDataGeneration = b.expectedDataGeneration; - this.eTag = b.eTag; - } - - public OmKeyInfo(BasicOmKeyInfo b) { - this.volumeName = b.getVolumeName(); - this.bucketName = b.getBucketName(); - this.keyName = b.getKeyName(); - this.dataSize = b.getDataSize(); - this.creationTime = b.getCreationTime(); - this.modificationTime = b.getModificationTime(); - this.replicationConfig = b.getReplicationConfig(); - this.isFile = b.isFile(); - this.eTag = b.getETag(); - this.ownerName = b.getOwnerName(); - this.fileChecksum = null; - this.acls = null; } public String getVolumeName() { @@ -180,10 +163,6 @@ public long getReplicatedSize() { return QuotaUtil.getReplicatedSize(getDataSize(), replicationConfig); } - public String getETag() { - return eTag; - } - public void setDataSize(long size) { this.dataSize = size; } @@ -472,7 +451,6 @@ public String toString() { ", fileChecksum=" + fileChecksum + ", isFile=" + isFile + ", fileName='" + fileName + '\'' + - ", eTag=" + eTag + '\'' + ", acls=" + acls + '}'; } @@ -498,7 +476,6 @@ public static class Builder extends WithParentObjectId.Builder { private FileChecksum fileChecksum; private boolean isFile; - private String eTag; private final Map tags = new HashMap<>(); private Long expectedDataGeneration = null; @@ -629,11 +606,6 @@ public Builder setFile(boolean isAFile) { return this; } - public Builder setETag(String etag) { - this.eTag = etag; - return this; - } - public Builder addTag(String key, String value) { tags.put(key, value); return this; @@ -730,9 +702,6 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, } else { kb.setFactor(ReplicationConfig.getLegacyFactor(replicationConfig)); } - if (eTag != null) { - kb.setETag(eTag); - } kb.setLatestVersion(latestVersion) .addAllKeyLocationList(keyLocations) .setCreationTime(creationTime) @@ -810,9 +779,6 @@ public static OmKeyInfo getFromProtobuf(KeyInfo keyInfo) throws IOException { if (keyInfo.hasIsFile()) { builder.setFile(keyInfo.getIsFile()); } - if (keyInfo.hasETag()) { - builder.setETag(keyInfo.getETag()); - } if (keyInfo.hasExpectedDataGeneration()) { builder.setExpectedDataGeneration(keyInfo.getExpectedDataGeneration()); } @@ -937,9 +903,6 @@ public OmKeyInfo copyObject() { if (expectedDataGeneration != null) { builder.setExpectedDataGeneration(expectedDataGeneration); } - if (eTag != null) { - builder.setETag(eTag); - } return builder.build(); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java index a23afff88be6..dcb7e2a2c1e3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java @@ -53,12 +53,6 @@ public OzoneFileStatus(OmKeyInfo keyInfo, this.blockSize = blockSize; } - public OzoneFileStatus(OzoneFileStatusLight b) { - this.keyInfo = new OmKeyInfo(b.getKeyInfo()); - this.isDirectory = b.isDirectory(); - this.blockSize = b.getBlockSize(); - } - public OmKeyInfo getKeyInfo() { return keyInfo; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 4f86d5497138..89f068cdedfa 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -2459,7 +2459,7 @@ public void testKeyDeleteLegacyWithEnableFileSystemPath() throws IOException { // Check number of keys OzoneVolume volume = client.getObjectStore().getVolume(volumeName); OzoneBucket bucket = volume.getBucket(bucketName); - List files = bucket.listStatus("", true, "", 5, false); + List files = bucket.listStatus("", true, "", 5); // Two keys should still exist, dirPath and keyPath assertEquals(2, files.size()); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 05c4f9ce9728..eeddc5500527 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1152,7 +1152,6 @@ message KeyInfo { // This allows a key to be created an committed atomically if the original has not // been modified. optional uint64 expectedDataGeneration = 22; - optional string eTag = 23; } message BasicKeyInfo { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index f4a919733e2a..e3e3537b1c3b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.TenantStateList; @@ -563,6 +564,13 @@ public List listStatus(String volumeName, String bucketName, return null; } + @Override + public List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, + long numEntries, boolean allowPartialPrefixes) throws IOException { + return null; + } + @Override public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException { return false; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java index 6eb508a93894..abc7ad5002fe 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java @@ -130,7 +130,7 @@ protected int getReadCount(int readCount, String readPath) throws IOException { List ozoneFileStatusList = bucket.listStatus( OzoneConsts.OM_KEY_PREFIX + readPath + OzoneConsts.OM_KEY_PREFIX, true, - "/", keyCountForRead, false); + "/", keyCountForRead); readCount += ozoneFileStatusList.size(); return readCount; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java index 3022dbf5c720..4c795f1e82b4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java @@ -106,7 +106,7 @@ private void deleteFSOKey(OzoneBucket bucket, String keyName) if (bucket.getFileStatus(keyName).isDirectory()) { List ozoneFileStatusList = - bucket.listStatus(keyName, false, "", 1, false); + bucket.listStatus(keyName, false, "", 1); if (ozoneFileStatusList != null && !ozoneFileStatusList.isEmpty()) { out().printf("Directory is not empty %n"); return; From d8e2629615ff701437c6fb5cb92c96355fe2f4d5 Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Fri, 6 Sep 2024 17:33:14 +0530 Subject: [PATCH 3/5] Add acceptance-compat tests for list --- hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot index 511679c56f4d..ef1740bd7844 100644 --- a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot +++ b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot @@ -30,6 +30,10 @@ Key Can Be Read Dir Can Be Listed Execute ozone fs -ls o3fs://bucket1.vol1/dir-${SUFFIX} +Dir Can Be Listed Using Shell + Execute ozone sh key list vol1/bucket1 + Execute ozone sh key list vol1/bucket1/dir-${SUFFIX} + File Can Be Get Execute ozone fs -get o3fs://bucket1.vol1/dir-${SUFFIX}/file-${SUFFIX} /tmp/ Execute diff -q ${TESTFILE} /tmp/file-${SUFFIX} From 10bc76ad8a409e61adac44e2dd614bf2ade38dfa Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Fri, 6 Sep 2024 20:02:28 +0530 Subject: [PATCH 4/5] Fix acceptance-compat test --- hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot index ef1740bd7844..f89c3a949a46 100644 --- a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot +++ b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot @@ -31,8 +31,8 @@ Dir Can Be Listed Execute ozone fs -ls o3fs://bucket1.vol1/dir-${SUFFIX} Dir Can Be Listed Using Shell - Execute ozone sh key list vol1/bucket1 - Execute ozone sh key list vol1/bucket1/dir-${SUFFIX} + Execute ozone sh key list /vol1/bucket1 + Should Contain ${result} key-${SUFFIX} File Can Be Get Execute ozone fs -get o3fs://bucket1.vol1/dir-${SUFFIX}/file-${SUFFIX} /tmp/ From 71be2d508b8ac0df972816f6878a9615924760a8 Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Fri, 6 Sep 2024 23:58:55 +0530 Subject: [PATCH 5/5] Fix acceptance-compat test --- hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot index f89c3a949a46..57715cda95f8 100644 --- a/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot +++ b/hadoop-ozone/dist/src/main/smoketest/compatibility/read.robot @@ -31,8 +31,8 @@ Dir Can Be Listed Execute ozone fs -ls o3fs://bucket1.vol1/dir-${SUFFIX} Dir Can Be Listed Using Shell - Execute ozone sh key list /vol1/bucket1 - Should Contain ${result} key-${SUFFIX} + ${result} = Execute ozone sh key list /vol1/bucket1 + Should Contain ${result} key-${SUFFIX} File Can Be Get Execute ozone fs -get o3fs://bucket1.vol1/dir-${SUFFIX}/file-${SUFFIX} /tmp/