From e68f6d543099bc805deed638644f0eb9abda4dbd Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Mon, 23 Jun 2025 18:20:58 +0530 Subject: [PATCH 1/2] HDDS-13324. Ozone Recon - Optimize memory footprint for listKeys API. --- .../apache/hadoop/ozone/recon/ReconUtils.java | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 745da0c7279c..55c897e6d34c 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -245,10 +245,11 @@ public static String constructFullPath(String keyName, long initialParentId, Str public static StringBuilder constructFullPathPrefix(long initialParentId, String volumeName, String bucketName, ReconNamespaceSummaryManager reconNamespaceSummaryManager, ReconOMMetadataManager omMetadataManager) throws IOException { - StringBuilder fullPath = new StringBuilder(); + long parentId = initialParentId; boolean isDirectoryPresent = false; + List pathSegments = new ArrayList<>(); while (parentId != 0) { NSSummary nsSummary = reconNamespaceSummaryManager.getNSSummary(parentId); if (nsSummary == null) { @@ -265,7 +266,7 @@ public static StringBuilder constructFullPathPrefix(long initialParentId, String } // On the last pass, dir-name will be empty and parent will be zero, indicating the loop should end. if (!nsSummary.getDirName().isEmpty()) { - fullPath.insert(0, nsSummary.getDirName() + OM_KEY_PREFIX); + pathSegments.add(nsSummary.getDirName()); } // Move to the parent ID of the current directory @@ -273,8 +274,15 @@ public static StringBuilder constructFullPathPrefix(long initialParentId, String isDirectoryPresent = true; } - // Prepend the volume and bucket to the constructed path - fullPath.insert(0, volumeName + OM_KEY_PREFIX + bucketName + OM_KEY_PREFIX); + Collections.reverse(pathSegments); + StringBuilder fullPath = new StringBuilder(); + + // Build the components in a list, then reverse and join once + fullPath.append(volumeName).append(OM_KEY_PREFIX) + .append(bucketName).append(OM_KEY_PREFIX); + for (String segment : pathSegments) { + fullPath.append(segment).append(OM_KEY_PREFIX); + } // TODO - why is this needed? It seems lke it should handle double slashes in the path name, // but its not clear how they get there. This normalize call is quite expensive as it // creates several objects (URI, PATH, back to string). There was a bug fixed above @@ -282,9 +290,11 @@ public static StringBuilder constructFullPathPrefix(long initialParentId, String // bucket name, but with that fixed, it seems like this should not be needed. All tests // pass without it for key listing. if (isDirectoryPresent) { - String path = fullPath.toString(); - fullPath.setLength(0); - fullPath.append(OmUtils.normalizeKey(path, true)); + if (fullPath.indexOf("//") >= 0) { + String path = fullPath.toString(); + fullPath.setLength(0); + fullPath.append(OmUtils.normalizeKey(path, true)); + } } return fullPath; } From b17499efdae4d73853c19811e01f9b6320e32522 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Singh Date: Mon, 23 Jun 2025 20:47:33 +0530 Subject: [PATCH 2/2] HDDS-13324. Ozone Recon - Optimize memory footprint for listKeys API. --- .../java/org/apache/hadoop/ozone/recon/ReconUtils.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 55c897e6d34c..10014ae741cb 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -274,15 +274,15 @@ public static StringBuilder constructFullPathPrefix(long initialParentId, String isDirectoryPresent = true; } - Collections.reverse(pathSegments); StringBuilder fullPath = new StringBuilder(); - - // Build the components in a list, then reverse and join once fullPath.append(volumeName).append(OM_KEY_PREFIX) .append(bucketName).append(OM_KEY_PREFIX); - for (String segment : pathSegments) { - fullPath.append(segment).append(OM_KEY_PREFIX); + + // Build the components in a list, then reverse and join once + for (int i = pathSegments.size() - 1; i >= 0; i--) { + fullPath.append(pathSegments.get(i)).append(OM_KEY_PREFIX); } + // TODO - why is this needed? It seems lke it should handle double slashes in the path name, // but its not clear how they get there. This normalize call is quite expensive as it // creates several objects (URI, PATH, back to string). There was a bug fixed above