From 14bfc8a5bba8672b88b979a1d611651eb7749c8e Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Sat, 29 Mar 2025 10:30:48 +0800 Subject: [PATCH 1/8] HDDS-12711. Change log level to debug to print excluded SST Files during bootstrap --- .../apache/hadoop/hdds/utils/DBCheckpointServlet.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 597a2415a1ef..07c4dbbcc7a7 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -195,7 +195,9 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, .filter(s -> s.endsWith(ROCKSDB_SST_SUFFIX)) .distinct() .collect(Collectors.toList())); - LOG.info("Received excluding SST {}", receivedSstList); + if (sstParam.length % 10 == 0) { + LOG.debug("Received excluding SST {}", receivedSstList); + } } Path tmpdir = null; @@ -229,9 +231,10 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, long duration = Duration.between(start, end).toMillis(); LOG.info("Time taken to write the checkpoint to response output " + "stream: {} milliseconds", duration); - - LOG.info("Excluded SST {} from the latest checkpoint.", - excludedSstList); + if (excludedSstList.size() % 10 == 0) { + LOG.debug("Excluded SST {} from the latest checkpoint.", + excludedSstList); + } if (!excludedSstList.isEmpty()) { dbMetrics.incNumIncrementalCheckpoint(); } From 035038429da2c8a3971271d2dc4b77989b95520f Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Sat, 29 Mar 2025 13:09:07 +0800 Subject: [PATCH 2/8] Appied comments --- .../apache/hadoop/hdds/utils/DBCheckpointServlet.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 07c4dbbcc7a7..9a25d1f884e7 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -195,8 +195,9 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, .filter(s -> s.endsWith(ROCKSDB_SST_SUFFIX)) .distinct() .collect(Collectors.toList())); - if (sstParam.length % 10 == 0) { - LOG.debug("Received excluding SST {}", receivedSstList); + if (LOG.isDebugEnabled()) { + LOG.debug("Received excluding SST {}. The total excluded sst files count is {}" + , receivedSstList.subList(0, Math.min(5, receivedSstList.size())), receivedSstList.size()); } } @@ -231,9 +232,9 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, long duration = Duration.between(start, end).toMillis(); LOG.info("Time taken to write the checkpoint to response output " + "stream: {} milliseconds", duration); - if (excludedSstList.size() % 10 == 0) { - LOG.debug("Excluded SST {} from the latest checkpoint.", - excludedSstList); + if (LOG.isDebugEnabled()) { + LOG.debug("Excluded SST {} from the latest checkpoint. The total excluded sst files count is {}", + excludedSstList.subList(0, Math.min(5, excludedSstList.size())), excludedSstList.size()); } if (!excludedSstList.isEmpty()) { dbMetrics.incNumIncrementalCheckpoint(); From 1ffec5bdd783793f1d11fa596f72178f14292e18 Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Sat, 29 Mar 2025 13:23:24 +0800 Subject: [PATCH 3/8] Fixed styles --- .../org/apache/hadoop/hdds/utils/DBCheckpointServlet.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 9a25d1f884e7..7e8defb7fe66 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -196,8 +196,8 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, .distinct() .collect(Collectors.toList())); if (LOG.isDebugEnabled()) { - LOG.debug("Received excluding SST {}. The total excluded sst files count is {}" - , receivedSstList.subList(0, Math.min(5, receivedSstList.size())), receivedSstList.size()); + LOG.debug("Received excluding SST {}. The total excluded sst files count is {}", + receivedSstList.subList(0, Math.min(5, receivedSstList.size())), receivedSstList.size()); } } From af54dc513dae798ee9a2c5b9f801bce397c8bb3c Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Sat, 29 Mar 2025 13:49:36 +0800 Subject: [PATCH 4/8] Applied comments --- .../apache/hadoop/hdds/utils/DBCheckpointServlet.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 7e8defb7fe66..eb961eb47b4e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -195,9 +195,10 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, .filter(s -> s.endsWith(ROCKSDB_SST_SUFFIX)) .distinct() .collect(Collectors.toList())); + LOG.info("Received excluding SST {}. The total excluded sst files count is {}", + receivedSstList.subList(0, Math.min(5, receivedSstList.size())), receivedSstList.size()); if (LOG.isDebugEnabled()) { - LOG.debug("Received excluding SST {}. The total excluded sst files count is {}", - receivedSstList.subList(0, Math.min(5, receivedSstList.size())), receivedSstList.size()); + LOG.debug("Received excluding SST {}", receivedSstList); } } @@ -232,9 +233,10 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, long duration = Duration.between(start, end).toMillis(); LOG.info("Time taken to write the checkpoint to response output " + "stream: {} milliseconds", duration); + LOG.info("Excluded SST {} from the latest checkpoint. The total excluded sst files count is {}", + excludedSstList.subList(0, Math.min(5, excludedSstList.size())), excludedSstList.size()); if (LOG.isDebugEnabled()) { - LOG.debug("Excluded SST {} from the latest checkpoint. The total excluded sst files count is {}", - excludedSstList.subList(0, Math.min(5, excludedSstList.size())), excludedSstList.size()); + LOG.debug("Excluded SST {} from the latest checkpoint.", excludedSstList); } if (!excludedSstList.isEmpty()) { dbMetrics.incNumIncrementalCheckpoint(); From 776c5d2bae4bca94c31205488926735ce80539cb Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Thu, 3 Apr 2025 15:06:03 +0800 Subject: [PATCH 5/8] Applied comments --- .../hadoop/hdds/utils/DBCheckpointServlet.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index eb961eb47b4e..f6c660d867f9 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -130,6 +130,10 @@ private boolean hasPermission(UserGroupInformation user) { } } + private static List extractSubList(Listinput, int index) { + return input.subList(0, Math.min(index, input.size())); + } + /** * Generates Snapshot checkpoint as tar ball. * @param request the HTTP servlet request @@ -195,11 +199,8 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, .filter(s -> s.endsWith(ROCKSDB_SST_SUFFIX)) .distinct() .collect(Collectors.toList())); - LOG.info("Received excluding SST {}. The total excluded sst files count is {}", - receivedSstList.subList(0, Math.min(5, receivedSstList.size())), receivedSstList.size()); - if (LOG.isDebugEnabled()) { - LOG.debug("Received excluding SST {}", receivedSstList); - } + LOG.info("Received list of {} SST files to be excluded, sample: {}", + receivedSstList.size(), extractSubList(receivedSstList, 5)); } Path tmpdir = null; @@ -234,10 +235,7 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, LOG.info("Time taken to write the checkpoint to response output " + "stream: {} milliseconds", duration); LOG.info("Excluded SST {} from the latest checkpoint. The total excluded sst files count is {}", - excludedSstList.subList(0, Math.min(5, excludedSstList.size())), excludedSstList.size()); - if (LOG.isDebugEnabled()) { - LOG.debug("Excluded SST {} from the latest checkpoint.", excludedSstList); - } + extractSubList(excludedSstList, 5), excludedSstList.size()); if (!excludedSstList.isEmpty()) { dbMetrics.incNumIncrementalCheckpoint(); } From ae2f7f208e4405f6f2822dab7923566ec5d73803 Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Thu, 3 Apr 2025 17:22:49 +0800 Subject: [PATCH 6/8] Applied comments --- .../hdds/utils/DBCheckpointServlet.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index f6c660d867f9..c68be4ffbe53 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -130,8 +130,17 @@ private boolean hasPermission(UserGroupInformation user) { } } - private static List extractSubList(Listinput, int index) { - return input.subList(0, Math.min(index, input.size())); + private static void logSstFileList(ListsstList, int sampleSize) { + String msg = "Received list of {} SST files to be excluded{}: {}"; + int count = sstList.size(); + if (LOG.isDebugEnabled()) { + LOG.debug(msg, count, "", sstList); + } else if (count > sampleSize) { + List sample = sstList.subList(0, sampleSize); + LOG.info(msg, count, ", sample", sample); + } else { + LOG.info(msg, count, "", sstList); + } } /** @@ -199,8 +208,7 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, .filter(s -> s.endsWith(ROCKSDB_SST_SUFFIX)) .distinct() .collect(Collectors.toList())); - LOG.info("Received list of {} SST files to be excluded, sample: {}", - receivedSstList.size(), extractSubList(receivedSstList, 5)); + logSstFileList(receivedSstList, 5); } Path tmpdir = null; @@ -234,8 +242,7 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, long duration = Duration.between(start, end).toMillis(); LOG.info("Time taken to write the checkpoint to response output " + "stream: {} milliseconds", duration); - LOG.info("Excluded SST {} from the latest checkpoint. The total excluded sst files count is {}", - extractSubList(excludedSstList, 5), excludedSstList.size()); + logSstFileList(excludedSstList, 5); if (!excludedSstList.isEmpty()) { dbMetrics.incNumIncrementalCheckpoint(); } From 97c2ed35eec118a2722ebf76e7f07759eaf58046 Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Thu, 3 Apr 2025 17:56:59 +0800 Subject: [PATCH 7/8] Applied comments --- .../org/apache/hadoop/hdds/utils/DBCheckpointServlet.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index c68be4ffbe53..63e4f811105e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -130,8 +130,7 @@ private boolean hasPermission(UserGroupInformation user) { } } - private static void logSstFileList(ListsstList, int sampleSize) { - String msg = "Received list of {} SST files to be excluded{}: {}"; + private static void logSstFileList(ListsstList, String msg, int sampleSize) { int count = sstList.size(); if (LOG.isDebugEnabled()) { LOG.debug(msg, count, "", sstList); @@ -151,6 +150,7 @@ private static void logSstFileList(ListsstList, int sampleSize) { */ private void generateSnapshotCheckpoint(HttpServletRequest request, HttpServletResponse response, boolean isFormData) { + String msg = "Received list of {} SST files to be excluded{}: {}"; if (dbStore == null) { LOG.error( "Unable to process metadata snapshot request. DB Store is null"); @@ -208,7 +208,7 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, .filter(s -> s.endsWith(ROCKSDB_SST_SUFFIX)) .distinct() .collect(Collectors.toList())); - logSstFileList(receivedSstList, 5); + logSstFileList(receivedSstList, msg, 5); } Path tmpdir = null; @@ -242,7 +242,7 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, long duration = Duration.between(start, end).toMillis(); LOG.info("Time taken to write the checkpoint to response output " + "stream: {} milliseconds", duration); - logSstFileList(excludedSstList, 5); + logSstFileList(excludedSstList, msg, 5); if (!excludedSstList.isEmpty()) { dbMetrics.incNumIncrementalCheckpoint(); } From cc2127652230374bb28772421a47bf9d3c361bb8 Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Thu, 3 Apr 2025 20:58:23 +0800 Subject: [PATCH 8/8] Changed the sample msg content --- .../org/apache/hadoop/hdds/utils/DBCheckpointServlet.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 63e4f811105e..4f528a83c201 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -150,7 +150,6 @@ private static void logSstFileList(ListsstList, String msg, int sampleSi */ private void generateSnapshotCheckpoint(HttpServletRequest request, HttpServletResponse response, boolean isFormData) { - String msg = "Received list of {} SST files to be excluded{}: {}"; if (dbStore == null) { LOG.error( "Unable to process metadata snapshot request. DB Store is null"); @@ -208,7 +207,8 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, .filter(s -> s.endsWith(ROCKSDB_SST_SUFFIX)) .distinct() .collect(Collectors.toList())); - logSstFileList(receivedSstList, msg, 5); + logSstFileList(receivedSstList, + "Received list of {} SST files to be excluded{}: {}", 5); } Path tmpdir = null; @@ -242,7 +242,8 @@ private void generateSnapshotCheckpoint(HttpServletRequest request, long duration = Duration.between(start, end).toMillis(); LOG.info("Time taken to write the checkpoint to response output " + "stream: {} milliseconds", duration); - logSstFileList(excludedSstList, msg, 5); + logSstFileList(excludedSstList, + "Excluded {} SST files from the latest checkpoint{}: {}", 5); if (!excludedSstList.isEmpty()) { dbMetrics.incNumIncrementalCheckpoint(); }