From c7012f3bc91e92af08f2d89ee6ac1cbb103475d2 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Fri, 26 Apr 2024 22:02:37 +0800 Subject: [PATCH 1/5] HDDS-10568. When the ldb command is executed, it is output by line --- .../apache/hadoop/ozone/debug/TestLDBCli.java | 49 +++++++++++++++++++ .../apache/hadoop/ozone/debug/DBScanner.java | 34 +++++++++++-- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index e94f46a398b3..4db4ee9bd864 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -263,6 +263,55 @@ void testScanOfPipelinesWhenNoData() throws IOException { assertEquals("", stderr.toString()); } + @Test + void testScanWithRecordsPerFile() throws IOException { + // Prepare dummy table + prepareTable(KEY_TABLE, false); + + String scanDir1 = tempDir.getAbsolutePath() + "/scandir1"; + // Prepare scan args + List completeScanArgs1 = new ArrayList<>(Arrays.asList( + "--db", dbStore.getDbLocation().getAbsolutePath(), + "scan", + "--column-family", KEY_TABLE, "--out", scanDir1 + "/key", + "--max-records-per-file", "2")); + + int exitCode1 = cmd.execute(completeScanArgs1.toArray(new String[0])); + // Check exit code. Print stderr if not expected + assertEquals(0, exitCode1); + + // Number of files generated by traversal + int count1 = 0; + File tmpDir1 = new File(scanDir1); + for (File tmpFile : tmpDir1.listFiles()) { + if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { + count1++; + } + } + assertEquals(count1, 3); + + // Used with parameter '-l' + String scanDir2 = tempDir.getAbsolutePath() + "/scandir2"; + // Prepare scan args + List completeScanArgs2 = new ArrayList<>(Arrays.asList( + "--db", dbStore.getDbLocation().getAbsolutePath(), + "scan", + "--column-family", KEY_TABLE, "--out", scanDir2 + "/key", + "--max-records-per-file", "3", "-l", "2")); + + int exitCode2 = cmd.execute(completeScanArgs2.toArray(new String[0])); + // Check exit code. Print stderr if not expected + assertEquals(0, exitCode2); + int count2 = 0; + File tmpDir2 = new File(scanDir2); + for (File tmpFile : tmpDir2.listFiles()) { + if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { + count2++; + } + } + assertEquals(count2, 2); + } + /** * Converts String input to a Map and compares to the given Map input. * @param expected expected result Map diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index 91cf38dada6d..9240477b98c4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -53,6 +53,7 @@ import picocli.CommandLine; import java.io.BufferedWriter; +import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Paths; @@ -113,6 +114,8 @@ public class DBScanner implements Callable, SubcommandWithParent { description = "File to dump table scan data") private String fileName; + private int fileSuffix = 0; + @CommandLine.Option(names = {"--startkey", "--sk", "-s"}, description = "Key from which to iterate the DB") private String startKey; @@ -148,6 +151,11 @@ public class DBScanner implements Callable, SubcommandWithParent { defaultValue = "10") private int threadCount; + @CommandLine.Option(names = {"--max-records-per-file"}, + description = "The number of records to print per file.", + defaultValue = "0") + private long recordsPerFile; + private static final String KEY_SEPARATOR_SCHEMA_V3 = new OzoneConfiguration().getObject(DatanodeConfiguration.class) .getContainerSchemaV3KeySeparator(); @@ -217,11 +225,26 @@ private boolean displayTable(ManagedRocksIterator iterator, return displayTable(iterator, dbColumnFamilyDef, out(), schemaV3); } + // If there are no parent directories, create them + File file = new File(fileName); + File parentFile = file.getParentFile(); + if (!parentFile.exists()) { + parentFile.mkdirs(); + } + // Write to file output - try (PrintWriter out = new PrintWriter(new BufferedWriter( - new PrintWriter(fileName, UTF_8.name())))) { - return displayTable(iterator, dbColumnFamilyDef, out, schemaV3); + while (iterator.get().isValid()) { + String fileNameTarget = recordsPerFile > 0 ? fileName + fileSuffix++ : + fileName; + try (PrintWriter out = new PrintWriter(new BufferedWriter( + new PrintWriter(fileNameTarget, UTF_8.name())))) { + if (!displayTable(iterator, dbColumnFamilyDef, out, schemaV3)) { + return false; + } + } } + + return true; } private boolean displayTable(ManagedRocksIterator iterator, @@ -288,6 +311,9 @@ private void processRecords(ManagedRocksIterator iterator, batch = new ArrayList<>(batchSize); sequenceId++; } + if ((recordsPerFile > 0) && (count >= recordsPerFile)) { + break; + } } if (!batch.isEmpty()) { Future future = threadPool.submit(new Task(dbColumnFamilyDef, @@ -305,7 +331,7 @@ private void processRecords(ManagedRocksIterator iterator, } private boolean withinLimit(long i) { - return limit == -1L || i < limit; + return recordsPerFile > 0 || limit == -1L || i < limit; } private ColumnFamilyHandle getColumnFamilyHandle( From 70f342fae33c51125cf65857d3fded566b3889e8 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Mon, 29 Apr 2024 15:42:16 +0800 Subject: [PATCH 2/5] Fixed some exceptions related to CI. --- .../apache/hadoop/ozone/debug/TestLDBCli.java | 26 +++++++++---------- .../apache/hadoop/ozone/debug/DBScanner.java | 6 ++++- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index 4db4ee9bd864..b158f5b73d69 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -52,19 +52,14 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.TreeMap; +import java.util.*; import java.util.stream.Stream; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * This class tests `ozone debug ldb` CLI that reads from a RocksDB directory. @@ -283,11 +278,14 @@ void testScanWithRecordsPerFile() throws IOException { // Number of files generated by traversal int count1 = 0; File tmpDir1 = new File(scanDir1); - for (File tmpFile : tmpDir1.listFiles()) { - if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { - count1++; + if (tmpDir1.isDirectory()) { + for (File tmpFile : Objects.requireNonNull(tmpDir1.listFiles())) { + if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { + count1++; + } } } + assertEquals(count1, 3); // Used with parameter '-l' @@ -304,9 +302,11 @@ void testScanWithRecordsPerFile() throws IOException { assertEquals(0, exitCode2); int count2 = 0; File tmpDir2 = new File(scanDir2); - for (File tmpFile : tmpDir2.listFiles()) { - if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { - count2++; + if (tmpDir2.isDirectory()) { + for (File tmpFile : Objects.requireNonNull(tmpDir2.listFiles())) { + if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { + count2++; + } } } assertEquals(count2, 2); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index 9240477b98c4..4b36be5a1b7c 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -229,7 +229,11 @@ private boolean displayTable(ManagedRocksIterator iterator, File file = new File(fileName); File parentFile = file.getParentFile(); if (!parentFile.exists()) { - parentFile.mkdirs(); + boolean flg = parentFile.mkdirs(); + if (!flg) { + throw new IOException("An exception occurred while creating " + + "the directory. Directorys: " + parentFile.getAbsolutePath()); + } } // Write to file output From 805e7dd919c1fb0ba37eff2eaaf9decd8740433a Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Mon, 29 Apr 2024 16:02:21 +0800 Subject: [PATCH 3/5] Fix some checkstyle. --- .../java/org/apache/hadoop/ozone/debug/TestLDBCli.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index b158f5b73d69..f03d05835b9f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -52,14 +52,20 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; import java.util.stream.Stream; +import java.util.Objects; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; /** * This class tests `ozone debug ldb` CLI that reads from a RocksDB directory. From 59e1451bb1f3cbac36b9fb761cba84e464c60d2f Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Mon, 29 Apr 2024 16:50:38 +0800 Subject: [PATCH 4/5] Fix some checkstyle. --- .../java/org/apache/hadoop/ozone/debug/TestLDBCli.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index f03d05835b9f..65f0e5208b85 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -60,7 +60,6 @@ import java.util.NavigableMap; import java.util.TreeMap; import java.util.stream.Stream; -import java.util.Objects; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE; @@ -284,8 +283,8 @@ void testScanWithRecordsPerFile() throws IOException { // Number of files generated by traversal int count1 = 0; File tmpDir1 = new File(scanDir1); - if (tmpDir1.isDirectory()) { - for (File tmpFile : Objects.requireNonNull(tmpDir1.listFiles())) { + if (tmpDir1.isDirectory() && tmpDir1.listFiles() != null) { + for (File tmpFile : tmpDir1.listFiles()) { if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { count1++; } @@ -308,8 +307,8 @@ void testScanWithRecordsPerFile() throws IOException { assertEquals(0, exitCode2); int count2 = 0; File tmpDir2 = new File(scanDir2); - if (tmpDir2.isDirectory()) { - for (File tmpFile : Objects.requireNonNull(tmpDir2.listFiles())) { + if (tmpDir2.isDirectory() && tmpDir2.listFiles() != null) { + for (File tmpFile : tmpDir2.listFiles()) { if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { count2++; } From 15efe88fd986edf70336baaecaec9e4688e28376 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Mon, 29 Apr 2024 17:53:28 +0800 Subject: [PATCH 5/5] Fix some checkstyle. --- .../apache/hadoop/ozone/debug/TestLDBCli.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index 65f0e5208b85..7aac5a8d40b3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -283,10 +283,13 @@ void testScanWithRecordsPerFile() throws IOException { // Number of files generated by traversal int count1 = 0; File tmpDir1 = new File(scanDir1); - if (tmpDir1.isDirectory() && tmpDir1.listFiles() != null) { - for (File tmpFile : tmpDir1.listFiles()) { - if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { - count1++; + if (tmpDir1.isDirectory()) { + File[] files = tmpDir1.listFiles(); + if (files != null) { + for (File tmpFile : files) { + if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { + count1++; + } } } } @@ -307,10 +310,13 @@ void testScanWithRecordsPerFile() throws IOException { assertEquals(0, exitCode2); int count2 = 0; File tmpDir2 = new File(scanDir2); - if (tmpDir2.isDirectory() && tmpDir2.listFiles() != null) { - for (File tmpFile : tmpDir2.listFiles()) { - if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { - count2++; + if (tmpDir2.isDirectory()) { + File[] files = tmpDir2.listFiles(); + if (files != null) { + for (File tmpFile : files) { + if (tmpFile.isFile() && tmpFile.getName().startsWith("key")) { + count2++; + } } } }