From b905e2802f4a3458daba6e4cb26f96c725ec8ee0 Mon Sep 17 00:00:00 2001 From: ashishk Date: Tue, 18 Apr 2023 11:27:07 +0530 Subject: [PATCH 1/3] HDDS-8440. Ozone Manager crashed when failed to delete FSO bucket. --- .../ozone/om/OmMetadataManagerImpl.java | 83 ++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 7e7dbc277722..98a417f6cf69 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -959,11 +959,11 @@ public boolean isBucketEmpty(String volume, String bucket) // Check dirTable as well in case of FSO bucket. if (bucketLayout.isFileSystemOptimized()) { // First check in dirTable cache - if (isKeyPresentInTableCache(keyPrefix, dirTable)) { + if (isKeyPresentInDirTableCache(keyPrefix, dirTable)) { return false; } // Check in the table - return !isKeyPresentInTable(keyPrefix, dirTable); + return !isKeyPresentInDirectoryTable(keyPrefix, dirTable); } return true; } @@ -1041,6 +1041,85 @@ private boolean isKeyPresentInTable(String keyPrefix, } return false; } + + + /** + * Checks if a key starting with a given keyPrefix exists + * in the directory table cache. + * + * @param keyPrefix - key prefix to be searched. + * @param table - table to be searched. + * @return true if the key is present in the cache. + */ + + private boolean isKeyPresentInDirTableCache(String keyPrefix, + Table table) { + Iterator, + CacheValue>> iterator = table.cacheIterator(); + while (iterator.hasNext()) { + Map.Entry, CacheValue> entry = + iterator.next(); + String key = entry.getKey().getCacheKey(); + OmDirectoryInfo omDirectoryInfo = entry.getValue().getCacheValue(); + // Making sure that entry is not for delete key request. + if (key.startsWith(keyPrefix) && omDirectoryInfo != null) { + return true; + } + } + return false; + } + + + /** + * Checks if a key starts with the given prefix is present + * in the directory table. + * @param keyPrefix - Prefix to check for + * @param table - Table to check in + * @return true if the key is present in the table + * @throws IOException + */ + private boolean isKeyPresentInDirectoryTable(String keyPrefix, + Table table) + throws IOException { + try (TableIterator> + keyIter = table.iterator()) { + KeyValue kv = keyIter.seek(keyPrefix); + + // Iterate through all the entries in the table which start with + // the current bucket's prefix. + while (kv != null && kv.getKey().startsWith(keyPrefix)) { + // Check the entry in db is not marked for delete. This can happen + // while entry is marked for delete, but it is not flushed to DB. + CacheValue cacheValue = + table.getCacheValue(new CacheKey(kv.getKey())); + + // Case 1: We found an entry, but no cache entry. + if (cacheValue == null) { + // we found at least one key with this prefix. + return true; + } + + // Case 2a: + // We found a cache entry and cache value is not null. + if (cacheValue.getCacheValue() != null) { + return true; + } + + // Case 2b: + // Cache entry is present but cache value is null, hence this key is + // marked for deletion. + // However, we still need to iterate through the rest of the prefix + // range to check for other keys with the same prefix that might still + // be present. + if (!keyIter.hasNext()) { + break; + } + kv = keyIter.next(); + } + } + return false; + } + /** * {@inheritDoc} From 7abadbbcc96bb5844cf128ad6e8f328e142aaa00 Mon Sep 17 00:00:00 2001 From: ashishk Date: Tue, 18 Apr 2023 15:32:36 +0530 Subject: [PATCH 2/3] HDDS-8440. Ozone Manager crashed when failed to delete FSO bucket. --- .../ozone/om/OmMetadataManagerImpl.java | 102 +++--------------- 1 file changed, 12 insertions(+), 90 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 98a417f6cf69..bbdbe2c92882 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -959,16 +959,15 @@ public boolean isBucketEmpty(String volume, String bucket) // Check dirTable as well in case of FSO bucket. if (bucketLayout.isFileSystemOptimized()) { // First check in dirTable cache - if (isKeyPresentInDirTableCache(keyPrefix, dirTable)) { + if (isKeyPresentInTableCache(keyPrefix, dirTable)) { return false; } // Check in the table - return !isKeyPresentInDirectoryTable(keyPrefix, dirTable); + return !isKeyPresentInTable(keyPrefix, dirTable); } return true; } - /** * Checks if a key starting with a given keyPrefix exists in the table cache. * @@ -978,20 +977,22 @@ public boolean isBucketEmpty(String volume, String bucket) */ private boolean isKeyPresentInTableCache(String keyPrefix, Table table) { - Iterator, CacheValue>> iterator = + Iterator, CacheValue>> iterator = table.cacheIterator(); while (iterator.hasNext()) { - Map.Entry, CacheValue> entry = + Map.Entry, CacheValue> entry = iterator.next(); String key = entry.getKey().getCacheKey(); - OmKeyInfo omKeyInfo = entry.getValue().getCacheValue(); + Object value = entry.getValue().getCacheValue(); + // Making sure that entry is not for delete key request. - if (key.startsWith(keyPrefix) && omKeyInfo != null) { + if (key.startsWith(keyPrefix) && value != null) { return true; } } return false; } + /** * Checks if a key starts with the given prefix is present in the table. * @@ -1001,18 +1002,18 @@ private boolean isKeyPresentInTableCache(String keyPrefix, * @throws IOException */ private boolean isKeyPresentInTable(String keyPrefix, - Table table) + Table table) throws IOException { - try (TableIterator> + try (TableIterator> keyIter = table.iterator()) { - KeyValue kv = keyIter.seek(keyPrefix); + KeyValue kv = keyIter.seek(keyPrefix); // Iterate through all the entries in the table which start with // the current bucket's prefix. while (kv != null && kv.getKey().startsWith(keyPrefix)) { // Check the entry in db is not marked for delete. This can happen // while entry is marked for delete, but it is not flushed to DB. - CacheValue cacheValue = + CacheValue cacheValue = table.getCacheValue(new CacheKey(kv.getKey())); // Case 1: We found an entry, but no cache entry. @@ -1041,85 +1042,6 @@ private boolean isKeyPresentInTable(String keyPrefix, } return false; } - - - /** - * Checks if a key starting with a given keyPrefix exists - * in the directory table cache. - * - * @param keyPrefix - key prefix to be searched. - * @param table - table to be searched. - * @return true if the key is present in the cache. - */ - - private boolean isKeyPresentInDirTableCache(String keyPrefix, - Table table) { - Iterator, - CacheValue>> iterator = table.cacheIterator(); - while (iterator.hasNext()) { - Map.Entry, CacheValue> entry = - iterator.next(); - String key = entry.getKey().getCacheKey(); - OmDirectoryInfo omDirectoryInfo = entry.getValue().getCacheValue(); - // Making sure that entry is not for delete key request. - if (key.startsWith(keyPrefix) && omDirectoryInfo != null) { - return true; - } - } - return false; - } - - - /** - * Checks if a key starts with the given prefix is present - * in the directory table. - * @param keyPrefix - Prefix to check for - * @param table - Table to check in - * @return true if the key is present in the table - * @throws IOException - */ - private boolean isKeyPresentInDirectoryTable(String keyPrefix, - Table table) - throws IOException { - try (TableIterator> - keyIter = table.iterator()) { - KeyValue kv = keyIter.seek(keyPrefix); - - // Iterate through all the entries in the table which start with - // the current bucket's prefix. - while (kv != null && kv.getKey().startsWith(keyPrefix)) { - // Check the entry in db is not marked for delete. This can happen - // while entry is marked for delete, but it is not flushed to DB. - CacheValue cacheValue = - table.getCacheValue(new CacheKey(kv.getKey())); - - // Case 1: We found an entry, but no cache entry. - if (cacheValue == null) { - // we found at least one key with this prefix. - return true; - } - - // Case 2a: - // We found a cache entry and cache value is not null. - if (cacheValue.getCacheValue() != null) { - return true; - } - - // Case 2b: - // Cache entry is present but cache value is null, hence this key is - // marked for deletion. - // However, we still need to iterate through the rest of the prefix - // range to check for other keys with the same prefix that might still - // be present. - if (!keyIter.hasNext()) { - break; - } - kv = keyIter.next(); - } - } - return false; - } - /** * {@inheritDoc} From b4acad7df6769a225f7f2ce005e39f674e9b8b14 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 18 Apr 2023 17:01:28 +0200 Subject: [PATCH 3/3] Add generic types for dir/key/file tables --- .../ozone/om/OmMetadataManagerImpl.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index bbdbe2c92882..fc6282d46d42 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -258,15 +258,15 @@ public class OmMetadataManagerImpl implements OMMetadataManager, private Table userTable; private Table volumeTable; private Table bucketTable; - private Table keyTable; + private Table keyTable; private Table deletedTable; private Table openKeyTable; private Table multipartInfoTable; private Table s3SecretTable; private Table dTokenTable; private Table prefixTable; - private Table dirTable; - private Table fileTable; + private Table dirTable; + private Table fileTable; private Table openFileTable; private Table transactionInfoTable; private Table metaTable; @@ -975,12 +975,12 @@ public boolean isBucketEmpty(String volume, String bucket) * @param table - table to be searched. * @return true if the key is present in the cache. */ - private boolean isKeyPresentInTableCache(String keyPrefix, - Table table) { - Iterator, CacheValue>> iterator = + private boolean isKeyPresentInTableCache(String keyPrefix, + Table table) { + Iterator, CacheValue>> iterator = table.cacheIterator(); while (iterator.hasNext()) { - Map.Entry, CacheValue> entry = + Map.Entry, CacheValue> entry = iterator.next(); String key = entry.getKey().getCacheKey(); Object value = entry.getValue().getCacheValue(); @@ -1001,20 +1001,20 @@ private boolean isKeyPresentInTableCache(String keyPrefix, * @return true if the key is present in the table * @throws IOException */ - private boolean isKeyPresentInTable(String keyPrefix, - Table table) + private boolean isKeyPresentInTable(String keyPrefix, + Table table) throws IOException { - try (TableIterator> + try (TableIterator> keyIter = table.iterator()) { - KeyValue kv = keyIter.seek(keyPrefix); + KeyValue kv = keyIter.seek(keyPrefix); // Iterate through all the entries in the table which start with // the current bucket's prefix. while (kv != null && kv.getKey().startsWith(keyPrefix)) { // Check the entry in db is not marked for delete. This can happen // while entry is marked for delete, but it is not flushed to DB. - CacheValue cacheValue = - table.getCacheValue(new CacheKey(kv.getKey())); + CacheValue cacheValue = + table.getCacheValue(new CacheKey<>(kv.getKey())); // Case 1: We found an entry, but no cache entry. if (cacheValue == null) {