From f6f1c13699bae100e0c12c8eb1cfdc1e3ecc8c84 Mon Sep 17 00:00:00 2001 From: Penumudy Tanvi Date: Mon, 9 Jun 2025 13:13:59 +0530 Subject: [PATCH 1/5] HDDS-13188. Track total pending deletion size for FSO directories in OM --- .../apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 048ab0ebc4ef..e48b05eee9c6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -1352,6 +1352,13 @@ private void createSummaryForDeletedDirectories( OmTableInsightTask.getTableCountKeyFromTable(DELETED_DIR_TABLE))); // Calculate the total number of deleted directories dirSummary.put("totalDeletedDirectories", deletedDirCount); + + // Get all deleted directories without pagination + KeyInsightInfoResponse deletedDirInsightInfo = new KeyInsightInfoResponse(); + getPendingForDeletionDirInfo(Integer.MAX_VALUE, "", deletedDirInsightInfo); + // Calculate the total replicated and unreplicated data size for deleted directories + dirSummary.put("totalReplicatedDataSize", deletedDirInsightInfo.getReplicatedDataSize()); + dirSummary.put("totalUnreplicatedDataSize", deletedDirInsightInfo.getUnreplicatedDataSize()); } private boolean validateStartPrefix(String startPrefix) { From d1f60c80283c7a4a33fbdf2e01f2890feecc0376 Mon Sep 17 00:00:00 2001 From: Penumudy Tanvi Date: Fri, 20 Jun 2025 14:06:42 +0530 Subject: [PATCH 2/5] Address review comments --- .../ozone/recon/api/OMDBInsightEndpoint.java | 51 ++++++++++++++++--- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index e48b05eee9c6..044412c97cd9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -673,6 +673,28 @@ private void getPendingForDeletionDirInfo( } } + private void calculateTotalPendingDeletedDirSizes(Map dirSummary) { + long totalUnreplicatedSize = 0L; + long totalReplicatedSize = 0L; + + Table deletedDirTable = omMetadataManager.getDeletedDirTable(); + try (TableIterator> iterator = deletedDirTable.iterator()) { + while (iterator.hasNext()) { + Table.KeyValue kv = iterator.next(); + OmKeyInfo omKeyInfo = kv.getValue(); + if (omKeyInfo != null) { + totalUnreplicatedSize += fetchSizeForDeletedDirectory(omKeyInfo.getObjectID()); + totalReplicatedSize += omKeyInfo.getReplicatedSize(); + } + } + } catch (IOException ex) { + throw new WebApplicationException(ex, Response.Status.INTERNAL_SERVER_ERROR); + } + + dirSummary.put("totalUnreplicatedDataSize", totalUnreplicatedSize); + dirSummary.put("totalReplicatedDataSize", totalReplicatedSize); + } + /** * Given an object ID, return total data size (no replication) * under this object. Note:- This method is RECURSIVE. @@ -779,6 +801,28 @@ public Response getDeletedDirectorySummary() { return Response.ok(dirSummary).build(); } + /** + * Retrieves the summary of the total delete pending directory size (unreplicated and replicated). + * + * @return The HTTP response body includes a map with the following entries: + * - "totalUnreplicatedDataSize": the total replicated size of delete pending directories. + * - "totalReplicatedDataSize": the total unreplicated size of delete pending directories. + * + * Example response: + * { + * "totalUnreplicatedDataSize": 30000, + * "totalReplicatedDataSize": 90000 + * } + */ + @GET + @Path("/deletePending/dirs/size") + public Response getTotalDeletedDirectorySizeSummary() { + Map dirSummary = new HashMap<>(); + // Create a keys summary for deleted directories + calculateTotalPendingDeletedDirSizes(dirSummary); + return Response.ok(dirSummary).build(); + } + /** * This API will list out limited 'count' number of keys after applying below filters in API parameters: * Default Values of API param filters: @@ -1352,13 +1396,6 @@ private void createSummaryForDeletedDirectories( OmTableInsightTask.getTableCountKeyFromTable(DELETED_DIR_TABLE))); // Calculate the total number of deleted directories dirSummary.put("totalDeletedDirectories", deletedDirCount); - - // Get all deleted directories without pagination - KeyInsightInfoResponse deletedDirInsightInfo = new KeyInsightInfoResponse(); - getPendingForDeletionDirInfo(Integer.MAX_VALUE, "", deletedDirInsightInfo); - // Calculate the total replicated and unreplicated data size for deleted directories - dirSummary.put("totalReplicatedDataSize", deletedDirInsightInfo.getReplicatedDataSize()); - dirSummary.put("totalUnreplicatedDataSize", deletedDirInsightInfo.getUnreplicatedDataSize()); } private boolean validateStartPrefix(String startPrefix) { From c05b31e685f13d401e16cba1e779defdf2153c46 Mon Sep 17 00:00:00 2001 From: Penumudy Tanvi Date: Mon, 14 Jul 2025 13:37:46 +0530 Subject: [PATCH 3/5] Update TODO for calculateTotalPendingDeletedDirSizes --- .../org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 044412c97cd9..06dbb6eb03fd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -684,6 +684,7 @@ private void calculateTotalPendingDeletedDirSizes(Map dirSummary) OmKeyInfo omKeyInfo = kv.getValue(); if (omKeyInfo != null) { totalUnreplicatedSize += fetchSizeForDeletedDirectory(omKeyInfo.getObjectID()); + // TODO: Based on HDDS-13180 merge, update totalReplicatedSize calculation using fetchSizeForDeletedDirectory. totalReplicatedSize += omKeyInfo.getReplicatedSize(); } } From 664413502e138108c3c2e97e127e642cda87d490 Mon Sep 17 00:00:00 2001 From: Penumudy Tanvi Date: Tue, 22 Jul 2025 17:38:37 +0530 Subject: [PATCH 4/5] Rename endpoint name and JSON output fields --- .../ozone/recon/api/OMDBInsightEndpoint.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index 06dbb6eb03fd..b4e2195b1438 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -674,8 +674,8 @@ private void getPendingForDeletionDirInfo( } private void calculateTotalPendingDeletedDirSizes(Map dirSummary) { - long totalUnreplicatedSize = 0L; - long totalReplicatedSize = 0L; + long totalDataSize = 0L; + long totalReplicatedDataSize = 0L; Table deletedDirTable = omMetadataManager.getDeletedDirTable(); try (TableIterator> iterator = deletedDirTable.iterator()) { @@ -683,17 +683,18 @@ private void calculateTotalPendingDeletedDirSizes(Map dirSummary) Table.KeyValue kv = iterator.next(); OmKeyInfo omKeyInfo = kv.getValue(); if (omKeyInfo != null) { - totalUnreplicatedSize += fetchSizeForDeletedDirectory(omKeyInfo.getObjectID()); - // TODO: Based on HDDS-13180 merge, update totalReplicatedSize calculation using fetchSizeForDeletedDirectory. - totalReplicatedSize += omKeyInfo.getReplicatedSize(); + totalDataSize += fetchSizeForDeletedDirectory(omKeyInfo.getObjectID()); + // TODO: Based on HDDS-13180 merge, update totalReplicatedDataSize calculation using + // fetchSizeForDeletedDirectory. + totalReplicatedDataSize += omKeyInfo.getReplicatedSize(); } } } catch (IOException ex) { throw new WebApplicationException(ex, Response.Status.INTERNAL_SERVER_ERROR); } - dirSummary.put("totalUnreplicatedDataSize", totalUnreplicatedSize); - dirSummary.put("totalReplicatedDataSize", totalReplicatedSize); + dirSummary.put("totalDataSize", totalDataSize); + dirSummary.put("totalReplicatedDataSize", totalReplicatedDataSize); } /** @@ -806,17 +807,17 @@ public Response getDeletedDirectorySummary() { * Retrieves the summary of the total delete pending directory size (unreplicated and replicated). * * @return The HTTP response body includes a map with the following entries: - * - "totalUnreplicatedDataSize": the total replicated size of delete pending directories. + * - "totalDataSize": the total replicated size of delete pending directories. * - "totalReplicatedDataSize": the total unreplicated size of delete pending directories. * * Example response: * { - * "totalUnreplicatedDataSize": 30000, + * "totalDataSize": 30000, * "totalReplicatedDataSize": 90000 * } */ @GET - @Path("/deletePending/dirs/size") + @Path("/deletePending/dirs/size-summary") public Response getTotalDeletedDirectorySizeSummary() { Map dirSummary = new HashMap<>(); // Create a keys summary for deleted directories From ce42be8efab254736e5b6625444007a5fb7163f1 Mon Sep 17 00:00:00 2001 From: Penumudy Tanvi Date: Wed, 23 Jul 2025 11:49:00 +0530 Subject: [PATCH 5/5] Fix TODO --- .../apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java index bd77c16bd09f..16c47bd92c70 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java @@ -686,10 +686,9 @@ private void calculateTotalPendingDeletedDirSizes(Map dirSummary) Table.KeyValue kv = iterator.next(); OmKeyInfo omKeyInfo = kv.getValue(); if (omKeyInfo != null) { - totalDataSize += fetchSizeForDeletedDirectory(omKeyInfo.getObjectID()); - // TODO: Based on HDDS-13180 merge, update totalReplicatedDataSize calculation using - // fetchSizeForDeletedDirectory. - totalReplicatedDataSize += omKeyInfo.getReplicatedSize(); + Pair sizeInfo = fetchSizeForDeletedDirectory(omKeyInfo.getObjectID()); + totalDataSize += sizeInfo.getLeft(); + totalReplicatedDataSize += sizeInfo.getRight(); } } } catch (IOException ex) {