From f40e7b47e0eb29906e01da29128758fd2143f985 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Mon, 2 May 2022 14:38:20 -0700 Subject: [PATCH 1/5] HDDS-6474. Fixed File System Optimized(FSO) bucket list status. --- .../apache/hadoop/ozone/om/KeyManagerImpl.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index cd04acbef076..d470eae7544c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1862,7 +1862,8 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, } - return buildFinalStatusList(cacheFileMap, cacheDirMap, args, clientAddress); + return buildFinalStatusList(cacheFileMap, cacheDirMap, args, + clientAddress, numEntries); } @@ -1873,18 +1874,27 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, private List buildFinalStatusList( Map cacheFileMap, Map cacheDirMap, OmKeyArgs omKeyArgs, - String clientAddress) + String clientAddress, long numEntries) throws IOException { List keyInfoList = new ArrayList<>(); List fileStatusFinalList = new ArrayList<>(); + long countEntries = 0; for (OzoneFileStatus fileStatus : cacheFileMap.values()) { fileStatusFinalList.add(fileStatus); keyInfoList.add(fileStatus.getKeyInfo()); + countEntries++; + if (countEntries >= numEntries) { + break; + } } for (OzoneFileStatus fileStatus : cacheDirMap.values()) { + if (countEntries >= numEntries) { + break; + } fileStatusFinalList.add(fileStatus); + countEntries++; } if (omKeyArgs.getLatestVersionLocation()) { @@ -1947,7 +1957,7 @@ protected int getDirectories( iterator = dirTable.iterator(); iterator.seek(seekDirInDB); - + countEntries = 0; while (iterator.hasNext() && numEntries - countEntries > 0) { Table.KeyValue entry = iterator.next(); OmDirectoryInfo dirInfo = entry.getValue(); @@ -1985,6 +1995,7 @@ private int getFilesFromDirectory( iterator) throws IOException { iterator.seek(seekKeyInDB); + countEntries = 0; while (iterator.hasNext() && numEntries - countEntries > 0) { Table.KeyValue entry = iterator.next(); OmKeyInfo keyInfo = entry.getValue(); From 4993d8c55c1b8b9334ef885c8724eed65753b15e Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Thu, 5 May 2022 10:17:44 -0700 Subject: [PATCH 2/5] HDDS-6474. Added test and updated parameters. --- .../TestRootedOzoneFileSystemWithFSO.java | 16 ++++++++++++++ .../hadoop/ozone/om/KeyManagerImpl.java | 22 ++++++++----------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java index d824352d5f6d..dbde9a89a0e7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.ozone; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.junit.Assert; @@ -224,4 +225,19 @@ public void testDeleteVolumeAndBucket() throws IOException { Assert.assertTrue(deletes == prevDeletes + 1); } + @Test + public void testListStatusFSO() throws Exception { + Path parent = new Path(getBucketPath(), "testListStatusFSO"); + for (int i = 0; i < 1300; i++) { + Path key = new Path(parent, "tempKey" + i); + ContractTestUtils.touch(getFs(), key); + // To add keys to the cache + getFs().rename(key, new Path(parent, "key" + i)); + } + + FileStatus[] fileStatuses = getFs().listStatus( + new Path(getBucketPath() + "/testListStatusFSO")); + Assert.assertEquals(1300, fileStatuses.length); + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index d470eae7544c..33ce9dc14c46 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1740,8 +1740,7 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, bucketName); } countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB, - prefixPath, prefixKeyInDB, countEntries, numEntries, deletedKeySet, - iterator); + prefixPath, prefixKeyInDB, numEntries, deletedKeySet, iterator); } else { /* @@ -1831,8 +1830,7 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, // 1. Seek the given key in key table. countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB, - prefixPath, prefixKeyInDB, countEntries, numEntries, - deletedKeySet, iterator); + prefixPath, prefixKeyInDB, numEntries, deletedKeySet, iterator); } } else { // TODO: HDDS-4364: startKey can be a non-existed key @@ -1857,7 +1855,7 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, // 2. Seek the given key in dir table. if (countEntries < numEntries) { getDirectories(cacheDirMap, seekDirInDB, prefixPath, - prefixKeyInDB, countEntries, numEntries, recursive, + prefixKeyInDB, numEntries, recursive, volumeName, bucketName, deletedKeySet); } @@ -1948,16 +1946,15 @@ private int getFilesAndDirsFromCacheWithBucket(String volumeName, protected int getDirectories( TreeMap cacheKeyMap, String seekDirInDB, String prefixPath, long prefixKeyInDB, - int countEntries, long numEntries, boolean recursive, - String volumeName, String bucketName, Set deletedKeySet) - throws IOException { + long numEntries, boolean recursive, String volumeName, + String bucketName, Set deletedKeySet) throws IOException { Table dirTable = metadataManager.getDirectoryTable(); TableIterator> iterator = dirTable.iterator(); iterator.seek(seekDirInDB); - countEntries = 0; + int countEntries = 0; while (iterator.hasNext() && numEntries - countEntries > 0) { Table.KeyValue entry = iterator.next(); OmDirectoryInfo dirInfo = entry.getValue(); @@ -1990,12 +1987,11 @@ protected int getDirectories( private int getFilesFromDirectory( TreeMap cacheKeyMap, String seekKeyInDB, String prefixKeyPath, long prefixKeyInDB, - int countEntries, long numEntries, Set deletedKeySet, + long numEntries, Set deletedKeySet, TableIterator> - iterator) - throws IOException { + iterator) throws IOException { iterator.seek(seekKeyInDB); - countEntries = 0; + int countEntries = 0; while (iterator.hasNext() && numEntries - countEntries > 0) { Table.KeyValue entry = iterator.next(); OmKeyInfo keyInfo = entry.getValue(); From 016e8890bfbf21ee5a69de84408a13fb36a829e5 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Wed, 11 May 2022 10:19:59 -0700 Subject: [PATCH 3/5] Added JavaDoc and updated comments. --- .../ozone/TestRootedOzoneFileSystemWithFSO.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java index dbde9a89a0e7..7846a0184227 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystemWithFSO.java @@ -225,19 +225,29 @@ public void testDeleteVolumeAndBucket() throws IOException { Assert.assertTrue(deletes == prevDeletes + 1); } + /** + * Test the consistency of listStatusFSO with TableCache present. + */ @Test public void testListStatusFSO() throws Exception { + // list keys batch size is 1024. Creating keys greater than the + // batch size to test batch listing of the keys. + int valueGreaterBatchSize = 1200; Path parent = new Path(getBucketPath(), "testListStatusFSO"); - for (int i = 0; i < 1300; i++) { + for (int i = 0; i < valueGreaterBatchSize; i++) { Path key = new Path(parent, "tempKey" + i); ContractTestUtils.touch(getFs(), key); - // To add keys to the cache + /* + To add keys to the cache. listStatusFSO goes through the cache first. + The cache is not continuous and may be greater than the batch size. + This may cause inconsistency in the listing of keys. + */ getFs().rename(key, new Path(parent, "key" + i)); } FileStatus[] fileStatuses = getFs().listStatus( new Path(getBucketPath() + "/testListStatusFSO")); - Assert.assertEquals(1300, fileStatuses.length); + Assert.assertEquals(valueGreaterBatchSize, fileStatuses.length); } } From 311b32b77121eb160998ee39f668a10f6bfc15f4 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Fri, 13 May 2022 12:44:28 -0700 Subject: [PATCH 4/5] Updated TableCache iteration condition. --- .../hadoop/ozone/om/KeyManagerImpl.java | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 33ce9dc14c46..bd5cf1b96d85 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1733,7 +1733,7 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, countEntries = getFilesAndDirsFromCacheWithBucket(volumeName, bucketName, cacheFileMap, tempCacheDirMap, deletedKeySet, prefixKeyInDB, seekFileInDB, seekDirInDB, prefixPath, startKey, - countEntries, numEntries); + countEntries); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, @@ -1796,9 +1796,9 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, bucketName); try { listStatusFindDirsInTableCache(tempCacheDirMap, - metadataManager.getDirectoryTable(), - prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName, - bucketName, countEntries, numEntries, deletedKeySet); + metadataManager.getDirectoryTable(), prefixKeyInDB, + seekDirInDB, prefixPath, startKey, volumeName, + bucketName, countEntries, deletedKeySet); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); @@ -1822,7 +1822,7 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, countEntries = getFilesAndDirsFromCacheWithBucket(volumeName, bucketName, cacheFileMap, tempCacheDirMap, deletedKeySet, prefixKeyInDB, seekFileInDB, seekDirInDB, prefixPath, startKey, - countEntries, numEntries); + countEntries); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); @@ -1853,11 +1853,9 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, } // 2. Seek the given key in dir table. - if (countEntries < numEntries) { - getDirectories(cacheDirMap, seekDirInDB, prefixPath, - prefixKeyInDB, numEntries, recursive, - volumeName, bucketName, deletedKeySet); - } + getDirectories(cacheDirMap, seekDirInDB, prefixPath, + prefixKeyInDB, numEntries, recursive, + volumeName, bucketName, deletedKeySet); return buildFinalStatusList(cacheFileMap, cacheDirMap, args, @@ -1916,11 +1914,11 @@ private List buildFinalStatusList( */ @SuppressWarnings("parameternumber") private int getFilesAndDirsFromCacheWithBucket(String volumeName, - String bucketName, Map cacheFileMap, - Map tempCacheDirMap, - Set deletedKeySet, long prefixKeyInDB, - String seekFileInDB, String seekDirInDB, String prefixPath, - String startKey, int countEntries, long numEntries) throws IOException { + String bucketName, Map cacheFileMap, + Map tempCacheDirMap, + Set deletedKeySet, long prefixKeyInDB, + String seekFileInDB, String seekDirInDB, String prefixPath, + String startKey, int countEntries) throws IOException { // First under lock obtain both entries from dir/file cache and generate @@ -1928,16 +1926,15 @@ private int getFilesAndDirsFromCacheWithBucket(String volumeName, BucketLayout bucketLayout = getBucketLayout(metadataManager, volumeName, bucketName); countEntries = listStatusFindFilesInTableCache(cacheFileMap, metadataManager - .getKeyTable(bucketLayout), - prefixKeyInDB, seekFileInDB, prefixPath, startKey, countEntries, - numEntries, deletedKeySet); + .getKeyTable(bucketLayout), prefixKeyInDB, seekFileInDB, prefixPath, + startKey, countEntries, deletedKeySet); // Don't count entries from dir cache, as first we need to return all // files and then directories. listStatusFindDirsInTableCache(tempCacheDirMap, metadataManager.getDirectoryTable(), prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName, - bucketName, countEntries, numEntries, deletedKeySet); + bucketName, countEntries, deletedKeySet); return countEntries; } @@ -2021,16 +2018,16 @@ private int getFilesFromDirectory( */ @SuppressWarnings("parameternumber") private int listStatusFindFilesInTableCache( - Map cacheKeyMap, Table keyTable, long prefixKeyInDB, String seekKeyInDB, - String prefixKeyPath, String startKey, int countEntries, - long numEntries, Set deletedKeySet) { + Map cacheKeyMap, Table keyTable, long prefixKeyInDB, String seekKeyInDB, + String prefixKeyPath, String startKey, int countEntries, + Set deletedKeySet) { Iterator, CacheValue>> cacheIter = keyTable.cacheIterator(); // TODO: recursive list will be handled in HDDS-4360 jira. - while (cacheIter.hasNext() && numEntries - countEntries > 0) { + while (cacheIter.hasNext()) { Map.Entry, CacheValue> entry = cacheIter.next(); String cacheKey = entry.getKey().getCacheKey(); @@ -2063,11 +2060,10 @@ private int listStatusFindFilesInTableCache( */ @SuppressWarnings("parameternumber") private int listStatusFindDirsInTableCache( - Map cacheKeyMap, Table dirTable, long prefixKeyInDB, String seekKeyInDB, - String prefixKeyPath, String startKey, String volumeName, - String bucketName, int countEntries, long numEntries, - Set deletedKeySet) { + Map cacheKeyMap, Table dirTable, long prefixKeyInDB, String seekKeyInDB, + String prefixKeyPath, String startKey, String volumeName, + String bucketName, int countEntries, Set deletedKeySet) { Iterator, CacheValue>> cacheIter = dirTable.cacheIterator(); @@ -2075,7 +2071,7 @@ private int listStatusFindDirsInTableCache( // 1. "1024/" -> startKey is null or empty // 2. "1024/b" -> startKey exists // TODO: recursive list will be handled in HDDS-4360 jira. - while (cacheIter.hasNext() && numEntries - countEntries > 0) { + while (cacheIter.hasNext()) { Map.Entry, CacheValue> entry = cacheIter.next(); String cacheKey = entry.getKey().getCacheKey(); From f640469a5e00a593655943938877a7fd511c3936 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Tue, 31 May 2022 13:09:55 -0700 Subject: [PATCH 5/5] Undo KeyManagerImpl changes --- .../hadoop/ozone/om/KeyManagerImpl.java | 89 +++++++++---------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index bd5cf1b96d85..cd04acbef076 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1733,14 +1733,15 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, countEntries = getFilesAndDirsFromCacheWithBucket(volumeName, bucketName, cacheFileMap, tempCacheDirMap, deletedKeySet, prefixKeyInDB, seekFileInDB, seekDirInDB, prefixPath, startKey, - countEntries); + countEntries, numEntries); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); } countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB, - prefixPath, prefixKeyInDB, numEntries, deletedKeySet, iterator); + prefixPath, prefixKeyInDB, countEntries, numEntries, deletedKeySet, + iterator); } else { /* @@ -1796,9 +1797,9 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, bucketName); try { listStatusFindDirsInTableCache(tempCacheDirMap, - metadataManager.getDirectoryTable(), prefixKeyInDB, - seekDirInDB, prefixPath, startKey, volumeName, - bucketName, countEntries, deletedKeySet); + metadataManager.getDirectoryTable(), + prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName, + bucketName, countEntries, numEntries, deletedKeySet); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); @@ -1822,7 +1823,7 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, countEntries = getFilesAndDirsFromCacheWithBucket(volumeName, bucketName, cacheFileMap, tempCacheDirMap, deletedKeySet, prefixKeyInDB, seekFileInDB, seekDirInDB, prefixPath, startKey, - countEntries); + countEntries, numEntries); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); @@ -1830,7 +1831,8 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, // 1. Seek the given key in key table. countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB, - prefixPath, prefixKeyInDB, numEntries, deletedKeySet, iterator); + prefixPath, prefixKeyInDB, countEntries, numEntries, + deletedKeySet, iterator); } } else { // TODO: HDDS-4364: startKey can be a non-existed key @@ -1853,13 +1855,14 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, } // 2. Seek the given key in dir table. - getDirectories(cacheDirMap, seekDirInDB, prefixPath, - prefixKeyInDB, numEntries, recursive, - volumeName, bucketName, deletedKeySet); + if (countEntries < numEntries) { + getDirectories(cacheDirMap, seekDirInDB, prefixPath, + prefixKeyInDB, countEntries, numEntries, recursive, + volumeName, bucketName, deletedKeySet); + } - return buildFinalStatusList(cacheFileMap, cacheDirMap, args, - clientAddress, numEntries); + return buildFinalStatusList(cacheFileMap, cacheDirMap, args, clientAddress); } @@ -1870,27 +1873,18 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, private List buildFinalStatusList( Map cacheFileMap, Map cacheDirMap, OmKeyArgs omKeyArgs, - String clientAddress, long numEntries) + String clientAddress) throws IOException { List keyInfoList = new ArrayList<>(); List fileStatusFinalList = new ArrayList<>(); - long countEntries = 0; for (OzoneFileStatus fileStatus : cacheFileMap.values()) { fileStatusFinalList.add(fileStatus); keyInfoList.add(fileStatus.getKeyInfo()); - countEntries++; - if (countEntries >= numEntries) { - break; - } } for (OzoneFileStatus fileStatus : cacheDirMap.values()) { - if (countEntries >= numEntries) { - break; - } fileStatusFinalList.add(fileStatus); - countEntries++; } if (omKeyArgs.getLatestVersionLocation()) { @@ -1914,11 +1908,11 @@ private List buildFinalStatusList( */ @SuppressWarnings("parameternumber") private int getFilesAndDirsFromCacheWithBucket(String volumeName, - String bucketName, Map cacheFileMap, - Map tempCacheDirMap, - Set deletedKeySet, long prefixKeyInDB, - String seekFileInDB, String seekDirInDB, String prefixPath, - String startKey, int countEntries) throws IOException { + String bucketName, Map cacheFileMap, + Map tempCacheDirMap, + Set deletedKeySet, long prefixKeyInDB, + String seekFileInDB, String seekDirInDB, String prefixPath, + String startKey, int countEntries, long numEntries) throws IOException { // First under lock obtain both entries from dir/file cache and generate @@ -1926,15 +1920,16 @@ private int getFilesAndDirsFromCacheWithBucket(String volumeName, BucketLayout bucketLayout = getBucketLayout(metadataManager, volumeName, bucketName); countEntries = listStatusFindFilesInTableCache(cacheFileMap, metadataManager - .getKeyTable(bucketLayout), prefixKeyInDB, seekFileInDB, prefixPath, - startKey, countEntries, deletedKeySet); + .getKeyTable(bucketLayout), + prefixKeyInDB, seekFileInDB, prefixPath, startKey, countEntries, + numEntries, deletedKeySet); // Don't count entries from dir cache, as first we need to return all // files and then directories. listStatusFindDirsInTableCache(tempCacheDirMap, metadataManager.getDirectoryTable(), prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName, - bucketName, countEntries, deletedKeySet); + bucketName, countEntries, numEntries, deletedKeySet); return countEntries; } @@ -1943,15 +1938,16 @@ private int getFilesAndDirsFromCacheWithBucket(String volumeName, protected int getDirectories( TreeMap cacheKeyMap, String seekDirInDB, String prefixPath, long prefixKeyInDB, - long numEntries, boolean recursive, String volumeName, - String bucketName, Set deletedKeySet) throws IOException { + int countEntries, long numEntries, boolean recursive, + String volumeName, String bucketName, Set deletedKeySet) + throws IOException { Table dirTable = metadataManager.getDirectoryTable(); TableIterator> iterator = dirTable.iterator(); iterator.seek(seekDirInDB); - int countEntries = 0; + while (iterator.hasNext() && numEntries - countEntries > 0) { Table.KeyValue entry = iterator.next(); OmDirectoryInfo dirInfo = entry.getValue(); @@ -1984,11 +1980,11 @@ protected int getDirectories( private int getFilesFromDirectory( TreeMap cacheKeyMap, String seekKeyInDB, String prefixKeyPath, long prefixKeyInDB, - long numEntries, Set deletedKeySet, + int countEntries, long numEntries, Set deletedKeySet, TableIterator> - iterator) throws IOException { + iterator) + throws IOException { iterator.seek(seekKeyInDB); - int countEntries = 0; while (iterator.hasNext() && numEntries - countEntries > 0) { Table.KeyValue entry = iterator.next(); OmKeyInfo keyInfo = entry.getValue(); @@ -2018,16 +2014,16 @@ private int getFilesFromDirectory( */ @SuppressWarnings("parameternumber") private int listStatusFindFilesInTableCache( - Map cacheKeyMap, Table keyTable, long prefixKeyInDB, String seekKeyInDB, - String prefixKeyPath, String startKey, int countEntries, - Set deletedKeySet) { + Map cacheKeyMap, Table keyTable, long prefixKeyInDB, String seekKeyInDB, + String prefixKeyPath, String startKey, int countEntries, + long numEntries, Set deletedKeySet) { Iterator, CacheValue>> cacheIter = keyTable.cacheIterator(); // TODO: recursive list will be handled in HDDS-4360 jira. - while (cacheIter.hasNext()) { + while (cacheIter.hasNext() && numEntries - countEntries > 0) { Map.Entry, CacheValue> entry = cacheIter.next(); String cacheKey = entry.getKey().getCacheKey(); @@ -2060,10 +2056,11 @@ private int listStatusFindFilesInTableCache( */ @SuppressWarnings("parameternumber") private int listStatusFindDirsInTableCache( - Map cacheKeyMap, Table dirTable, long prefixKeyInDB, String seekKeyInDB, - String prefixKeyPath, String startKey, String volumeName, - String bucketName, int countEntries, Set deletedKeySet) { + Map cacheKeyMap, Table dirTable, long prefixKeyInDB, String seekKeyInDB, + String prefixKeyPath, String startKey, String volumeName, + String bucketName, int countEntries, long numEntries, + Set deletedKeySet) { Iterator, CacheValue>> cacheIter = dirTable.cacheIterator(); @@ -2071,7 +2068,7 @@ private int listStatusFindDirsInTableCache( // 1. "1024/" -> startKey is null or empty // 2. "1024/b" -> startKey exists // TODO: recursive list will be handled in HDDS-4360 jira. - while (cacheIter.hasNext()) { + while (cacheIter.hasNext() && numEntries - countEntries > 0) { Map.Entry, CacheValue> entry = cacheIter.next(); String cacheKey = entry.getKey().getCacheKey();