From 1c98dda75d2f5a90ed6a7c45c6b0408ec0b663b1 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 3 Feb 2025 14:26:43 +0530 Subject: [PATCH 1/2] HDDS-12186. Avoid array allocation for table iterator. --- .../apache/hadoop/hdds/utils/db/Table.java | 19 +++++-------------- .../hadoop/hdds/utils/db/TypedTable.java | 8 +++----- .../hadoop/ozone/om/KeyManagerImpl.java | 4 ++-- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java index 0c435066b8e2..d0cf42e4be9a 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java @@ -337,12 +337,8 @@ interface KeyValue { VALUE getValue() throws IOException; - default byte[] getRawKey() throws IOException { - return null; - } - - default byte[] getRawValue() throws IOException { - return null; + default int getReadableBytes() throws IOException { + return -1; } } @@ -383,7 +379,7 @@ public int hashCode() { }; } - static KeyValue newKeyValue(K key, V value, byte[] rawKey, byte[] rawValue) { + static KeyValue newKeyValue(K key, V value, int readableBytes) { return new KeyValue() { @Override public K getKey() { @@ -396,13 +392,8 @@ public V getValue() { } @Override - public byte[] getRawKey() throws IOException { - return rawKey; - } - - @Override - public byte[] getRawValue() throws IOException { - return rawValue; + public int getReadableBytes() throws IOException { + return readableBytes; } @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java index 9609b5bfc283..206ce45daa02 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java @@ -605,11 +605,9 @@ public CodecBuffer get() { @Override KeyValue convert(KeyValue raw) throws IOException { - CodecBuffer keyCodecBuffer = raw.getKey(); - final KEY key = keyCodec.fromCodecBuffer(keyCodecBuffer); - CodecBuffer valueCodecBuffer = raw.getValue(); - final VALUE value = valueCodec.fromCodecBuffer(valueCodecBuffer); - return Table.newKeyValue(key, value, keyCodecBuffer.getArray(), valueCodecBuffer.getArray()); + final KEY key = keyCodec.fromCodecBuffer(raw.getKey()); + final VALUE value = valueCodec.fromCodecBuffer(raw.getValue()); + return Table.newKeyValue(key, value, raw.getReadableBytes()); } }; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 4fe509d7e9cd..19f7fc0c8fe2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -2102,7 +2102,7 @@ private DeleteKeysResult gatherSubDirsWithIterator(OmKeyInfo parentInfo, while (iterator.hasNext() && remainingBufLimit > 0) { Table.KeyValue entry = iterator.next(); OmDirectoryInfo dirInfo = entry.getValue(); - long objectSerializedSize = entry.getRawValue().length; + long objectSerializedSize = entry.getReadableBytes(); if (!OMFileRequest.isImmediateChild(dirInfo.getParentObjectID(), parentInfo.getObjectID())) { processedSubDirs = true; @@ -2149,7 +2149,7 @@ public DeleteKeysResult getPendingDeletionSubFiles(long volumeId, while (iterator.hasNext() && remainingBufLimit > 0) { Table.KeyValue entry = iterator.next(); OmKeyInfo fileInfo = entry.getValue(); - long objectSerializedSize = entry.getRawValue().length; + long objectSerializedSize = entry.getReadableBytes(); if (!OMFileRequest.isImmediateChild(fileInfo.getParentObjectID(), parentInfo.getObjectID())) { processedSubFiles = true; From 55b0b29536ed0938dcc7a77447d3e1000b2f4b49 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 3 Feb 2025 14:45:22 +0530 Subject: [PATCH 2/2] address comments --- .../java/org/apache/hadoop/hdds/utils/db/Table.java | 10 +++++----- .../org/apache/hadoop/hdds/utils/db/TypedTable.java | 2 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java index d0cf42e4be9a..f519bca5b107 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java @@ -337,8 +337,8 @@ interface KeyValue { VALUE getValue() throws IOException; - default int getReadableBytes() throws IOException { - return -1; + default int getRawSize() throws IOException { + return 0; } } @@ -379,7 +379,7 @@ public int hashCode() { }; } - static KeyValue newKeyValue(K key, V value, int readableBytes) { + static KeyValue newKeyValue(K key, V value, int rawSize) { return new KeyValue() { @Override public K getKey() { @@ -392,8 +392,8 @@ public V getValue() { } @Override - public int getReadableBytes() throws IOException { - return readableBytes; + public int getRawSize() throws IOException { + return rawSize; } @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java index 206ce45daa02..d7bd554a0cd0 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java @@ -607,7 +607,7 @@ KeyValue convert(KeyValue raw) throws IOException { final KEY key = keyCodec.fromCodecBuffer(raw.getKey()); final VALUE value = valueCodec.fromCodecBuffer(raw.getValue()); - return Table.newKeyValue(key, value, raw.getReadableBytes()); + return Table.newKeyValue(key, value, raw.getRawSize()); } }; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 19f7fc0c8fe2..6bad74b6d42e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -2102,7 +2102,7 @@ private DeleteKeysResult gatherSubDirsWithIterator(OmKeyInfo parentInfo, while (iterator.hasNext() && remainingBufLimit > 0) { Table.KeyValue entry = iterator.next(); OmDirectoryInfo dirInfo = entry.getValue(); - long objectSerializedSize = entry.getReadableBytes(); + long objectSerializedSize = entry.getRawSize(); if (!OMFileRequest.isImmediateChild(dirInfo.getParentObjectID(), parentInfo.getObjectID())) { processedSubDirs = true; @@ -2149,7 +2149,7 @@ public DeleteKeysResult getPendingDeletionSubFiles(long volumeId, while (iterator.hasNext() && remainingBufLimit > 0) { Table.KeyValue entry = iterator.next(); OmKeyInfo fileInfo = entry.getValue(); - long objectSerializedSize = entry.getReadableBytes(); + long objectSerializedSize = entry.getRawSize(); if (!OMFileRequest.isImmediateChild(fileInfo.getParentObjectID(), parentInfo.getObjectID())) { processedSubFiles = true;