From 67b876454d4603a8642e74f658cb7387bec5750a Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Thu, 21 Nov 2024 22:26:50 +0800 Subject: [PATCH 1/8] HDDS-10568. When the ldb command is executed, it is output by line --- .../apache/hadoop/ozone/debug/TestLDBCli.java | 37 +++++++++++++++++ .../apache/hadoop/ozone/debug/DBScanner.java | 41 ++++++++++++++++--- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index 7e16c0a29e35..5eef625db05c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -341,6 +341,43 @@ void testScanOfPipelinesWhenNoData() throws IOException { assertEquals("", stderr.toString()); } + @Test + void testScanWithRecordsPerFile() throws IOException { + // Prepare dummy table + prepareTable(KEY_TABLE, false); + + String scanDir1 = tempDir.getAbsolutePath() + "/scandir1"; + // Prepare scan args + List completeScanArgs1 = new ArrayList<>(Arrays.asList( + "--db", dbStore.getDbLocation().getAbsolutePath(), + "scan", + "--column-family", KEY_TABLE, "--out", scanDir1, + "--max-records-per-file", "2")); + + File tmpDir1 = new File(scanDir1); + tmpDir1.deleteOnExit(); + + int exitCode1 = cmd.execute(completeScanArgs1.toArray(new String[0])); + assertEquals(0, exitCode1); + assertTrue(tmpDir1.isDirectory()); + assertEquals(3, tmpDir1.listFiles().length); + + String scanDir2 = tempDir.getAbsolutePath() + "/scandir2"; + // Used with parameter '-l' + List completeScanArgs2 = new ArrayList<>(Arrays.asList( + "--db", dbStore.getDbLocation().getAbsolutePath(), + "scan", + "--column-family", KEY_TABLE, "--out", scanDir2, + "--max-records-per-file", "3", "-l", "2")); + File tmpDir2 = new File(scanDir2); + tmpDir2.deleteOnExit(); + + int exitCode2 = cmd.execute(completeScanArgs2.toArray(new String[0])); + assertEquals(0, exitCode2); + assertTrue(tmpDir2.isDirectory()); + assertEquals(1, tmpDir2.listFiles().length); + } + @Test void testSchemaCommand() throws IOException { // Prepare dummy table diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index 8aa52166cb1c..b9c2de546f05 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -54,6 +54,7 @@ import picocli.CommandLine; import java.io.BufferedWriter; +import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Field; @@ -172,6 +173,14 @@ public class DBScanner implements Callable, SubcommandWithParent { defaultValue = "10") private int threadCount; + @CommandLine.Option(names = {"--max-records-per-file"}, + description = "The number of records to print per file.", + defaultValue = "0") + private long recordsPerFile; + + private int fileSuffix = 0; + private long globalCount = 0; + private static final String KEY_SEPARATOR_SCHEMA_V3 = new OzoneConfiguration().getObject(DatanodeConfiguration.class) .getContainerSchemaV3KeySeparator(); @@ -180,7 +189,8 @@ public class DBScanner implements Callable, SubcommandWithParent { @Override public Void call() throws Exception { - + fileSuffix = 0; + globalCount = 0; List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(parent.getDbPath()); final List cfHandleList = new ArrayList<>(); @@ -240,11 +250,28 @@ private boolean displayTable(ManagedRocksIterator iterator, return displayTable(iterator, dbColumnFamilyDef, out(), schemaV3); } + // If there are no parent directories, create them + File dirFile = new File(fileName); + if (!dirFile.exists()) { + boolean flg = dirFile.mkdirs(); + if (!flg) { + throw new IOException("An exception occurred while creating " + + "the directory. Directorys: " + dirFile.getAbsolutePath()); + } + } + // Write to file output - try (PrintWriter out = new PrintWriter(new BufferedWriter( - new PrintWriter(fileName, UTF_8.name())))) { - return displayTable(iterator, dbColumnFamilyDef, out, schemaV3); + while (iterator.get().isValid() && withinLimit(globalCount)) { + String fileNameTarget = recordsPerFile > 0 ? fileName + File.separator + fileSuffix++ : + fileName; + try (PrintWriter out = new PrintWriter(new BufferedWriter( + new PrintWriter(fileNameTarget, UTF_8.name())))) { + if (!displayTable(iterator, dbColumnFamilyDef, out, schemaV3)) { + return false; + } + } } + return true; } private boolean displayTable(ManagedRocksIterator iterator, @@ -314,7 +341,7 @@ private void processRecords(ManagedRocksIterator iterator, } } - while (withinLimit(count) && iterator.get().isValid() && !exception && !reachedEnd) { + while (withinLimit(globalCount) && iterator.get().isValid() && !exception && !reachedEnd) { // if invalid endKey is given, it is ignored if (null != endKey && Arrays.equals(iterator.get().key(), getValueObject(dbColumnFamilyDef, endKey))) { reachedEnd = true; @@ -326,6 +353,7 @@ private void processRecords(ManagedRocksIterator iterator, // the record passes the filter batch.add(new ByteArrayKeyValue( iterator.get().key(), iterator.get().value())); + globalCount++; count++; if (batch.size() >= batchSize) { while (logWriter.getInflightLogCount() > threadCount * 10L @@ -343,6 +371,9 @@ private void processRecords(ManagedRocksIterator iterator, } } iterator.get().next(); + if ((recordsPerFile > 0) && (count >= recordsPerFile)) { + break; + } } if (!batch.isEmpty()) { Future future = threadPool.submit(new Task(dbColumnFamilyDef, From 1b2b8d89babb0e02f3aeb4686c2cac07f15ffa53 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 24 Nov 2024 17:29:54 +0800 Subject: [PATCH 2/8] Update some unit tests. --- .../apache/hadoop/ozone/debug/TestLDBCli.java | 29 ++++++++++++------- .../apache/hadoop/ozone/debug/DBScanner.java | 17 ++++++----- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index 5eef625db05c..2bdb5e617b4d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.client.BlockID; @@ -52,6 +54,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; +import java.io.FileReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -344,31 +347,37 @@ void testScanOfPipelinesWhenNoData() throws IOException { @Test void testScanWithRecordsPerFile() throws IOException { // Prepare dummy table - prepareTable(KEY_TABLE, false); + int recordsCount = 5; + prepareTable(KEY_TABLE, false, recordsCount); String scanDir1 = tempDir.getAbsolutePath() + "/scandir1"; // Prepare scan args + int maxRecordsPerFile = 2; List completeScanArgs1 = new ArrayList<>(Arrays.asList( "--db", dbStore.getDbLocation().getAbsolutePath(), "scan", - "--column-family", KEY_TABLE, "--out", scanDir1, - "--max-records-per-file", "2")); - + "--column-family", KEY_TABLE, "--out", scanDir1 + File.separator + "keytable", + "--max-records-per-file", String.valueOf(maxRecordsPerFile))); File tmpDir1 = new File(scanDir1); tmpDir1.deleteOnExit(); int exitCode1 = cmd.execute(completeScanArgs1.toArray(new String[0])); assertEquals(0, exitCode1); assertTrue(tmpDir1.isDirectory()); - assertEquals(3, tmpDir1.listFiles().length); + File[] subFiles = tmpDir1.listFiles(); + assertEquals(Math.ceil(recordsCount / (maxRecordsPerFile * 1.0)), subFiles.length); + for (File subFile : subFiles) { + JsonElement jsonElement = JsonParser.parseReader(new FileReader(subFile)); + assertTrue(jsonElement.isJsonArray() || jsonElement.isJsonObject()); + } String scanDir2 = tempDir.getAbsolutePath() + "/scandir2"; // Used with parameter '-l' List completeScanArgs2 = new ArrayList<>(Arrays.asList( "--db", dbStore.getDbLocation().getAbsolutePath(), "scan", - "--column-family", KEY_TABLE, "--out", scanDir2, - "--max-records-per-file", "3", "-l", "2")); + "--column-family", KEY_TABLE, "--out", scanDir2 + File.separator + "keytable", + "--max-records-per-file", String.valueOf(maxRecordsPerFile), "-l", "2")); File tmpDir2 = new File(scanDir2); tmpDir2.deleteOnExit(); @@ -421,7 +430,7 @@ private void assertContents(Map expected, String actualStr) * @param tableName table name * @param schemaV3 set to true for SchemaV3. applicable to block_data table */ - private void prepareTable(String tableName, boolean schemaV3) + private void prepareTable(String tableName, boolean schemaV3, int... recordsCount) throws IOException { switch (tableName) { @@ -431,8 +440,8 @@ private void prepareTable(String tableName, boolean schemaV3) .setPath(tempDir.toPath()).addTable(KEY_TABLE).build(); Table keyTable = dbStore.getTable(KEY_TABLE); - // Insert 5 keys - for (int i = 1; i <= 5; i++) { + // Insert the number of keys specified by recordsCount[0] + for (int i = 1; i <= recordsCount[0]; i++) { String key = "key" + i; OmKeyInfo value = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", key, ReplicationConfig.fromProtoTypeAndFactor(STAND_ALONE, HddsProtos.ReplicationFactor.ONE)).build(); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index b9c2de546f05..89486b7e3aa5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -251,18 +251,21 @@ private boolean displayTable(ManagedRocksIterator iterator, } // If there are no parent directories, create them - File dirFile = new File(fileName); - if (!dirFile.exists()) { - boolean flg = dirFile.mkdirs(); - if (!flg) { - throw new IOException("An exception occurred while creating " + - "the directory. Directorys: " + dirFile.getAbsolutePath()); + if (recordsPerFile > 0) { + File file = new File(fileName); + File parentFile = file.getParentFile(); + if (!parentFile.exists()) { + boolean flg = parentFile.mkdirs(); + if (!flg) { + throw new IOException("An exception occurred while creating " + + "the directory. Directorys: " + parentFile.getAbsolutePath()); + } } } // Write to file output while (iterator.get().isValid() && withinLimit(globalCount)) { - String fileNameTarget = recordsPerFile > 0 ? fileName + File.separator + fileSuffix++ : + String fileNameTarget = recordsPerFile > 0 ? fileName + "." + fileSuffix++ : fileName; try (PrintWriter out = new PrintWriter(new BufferedWriter( new PrintWriter(fileNameTarget, UTF_8.name())))) { From 50a4bf43206f2c45b2b0779a0034a5f4a2b1fd4d Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 24 Nov 2024 18:04:30 +0800 Subject: [PATCH 3/8] Fix some checkstyle. --- .../java/org/apache/hadoop/ozone/debug/TestLDBCli.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index 2bdb5e617b4d..c724dbbf444a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -71,6 +71,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * This class tests `ozone debug ldb` CLI that reads from a RocksDB directory. @@ -365,6 +366,7 @@ void testScanWithRecordsPerFile() throws IOException { assertEquals(0, exitCode1); assertTrue(tmpDir1.isDirectory()); File[] subFiles = tmpDir1.listFiles(); + assertNotNull(subFiles); assertEquals(Math.ceil(recordsCount / (maxRecordsPerFile * 1.0)), subFiles.length); for (File subFile : subFiles) { JsonElement jsonElement = JsonParser.parseReader(new FileReader(subFile)); @@ -441,7 +443,11 @@ private void prepareTable(String tableName, boolean schemaV3, int... recordsCoun Table keyTable = dbStore.getTable(KEY_TABLE); // Insert the number of keys specified by recordsCount[0] - for (int i = 1; i <= recordsCount[0]; i++) { + int count = 5; + if (recordsCount.length > 0) { + count = recordsCount[0]; + } + for (int i = 1; i <= count; i++) { String key = "key" + i; OmKeyInfo value = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", key, ReplicationConfig.fromProtoTypeAndFactor(STAND_ALONE, HddsProtos.ReplicationFactor.ONE)).build(); From a965d3b500d8373bdf6546591a9e08ce410b23a3 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 24 Nov 2024 18:51:46 +0800 Subject: [PATCH 4/8] Fix some findbugs. --- .../java/org/apache/hadoop/ozone/debug/TestLDBCli.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index c724dbbf444a..fafd488a4b51 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -54,7 +54,8 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import java.io.FileReader; +import java.io.FileInputStream; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -369,8 +370,10 @@ void testScanWithRecordsPerFile() throws IOException { assertNotNull(subFiles); assertEquals(Math.ceil(recordsCount / (maxRecordsPerFile * 1.0)), subFiles.length); for (File subFile : subFiles) { - JsonElement jsonElement = JsonParser.parseReader(new FileReader(subFile)); - assertTrue(jsonElement.isJsonArray() || jsonElement.isJsonObject()); + try(FileInputStream inputStream =new FileInputStream(subFile)) { + JsonElement jsonElement = JsonParser.parseReader(new InputStreamReader(inputStream, "UTF-8")); + assertTrue(jsonElement.isJsonArray() || jsonElement.isJsonObject()); + } } String scanDir2 = tempDir.getAbsolutePath() + "/scandir2"; From 171887729d68ceff045e56df4c27673541d46076 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 24 Nov 2024 19:16:23 +0800 Subject: [PATCH 5/8] Fix some checkstyle. --- .../src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index fafd488a4b51..fc229bc9314e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -370,7 +370,7 @@ void testScanWithRecordsPerFile() throws IOException { assertNotNull(subFiles); assertEquals(Math.ceil(recordsCount / (maxRecordsPerFile * 1.0)), subFiles.length); for (File subFile : subFiles) { - try(FileInputStream inputStream =new FileInputStream(subFile)) { + try (FileInputStream inputStream = new FileInputStream(subFile)) { JsonElement jsonElement = JsonParser.parseReader(new InputStreamReader(inputStream, "UTF-8")); assertTrue(jsonElement.isJsonArray() || jsonElement.isJsonObject()); } From 4b8e0fee8040e43442404e2ff89e725aef87baf9 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Wed, 27 Nov 2024 11:31:40 +0800 Subject: [PATCH 6/8] Update some unit tests. --- .../apache/hadoop/ozone/debug/TestLDBCli.java | 34 ++++++++++++++----- .../apache/hadoop/ozone/debug/DBScanner.java | 16 ++++----- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index fc229bc9314e..ecc41e2f6b46 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -350,7 +350,7 @@ void testScanOfPipelinesWhenNoData() throws IOException { void testScanWithRecordsPerFile() throws IOException { // Prepare dummy table int recordsCount = 5; - prepareTable(KEY_TABLE, false, recordsCount); + prepareKeyTable(recordsCount); String scanDir1 = tempDir.getAbsolutePath() + "/scandir1"; // Prepare scan args @@ -435,7 +435,7 @@ private void assertContents(Map expected, String actualStr) * @param tableName table name * @param schemaV3 set to true for SchemaV3. applicable to block_data table */ - private void prepareTable(String tableName, boolean schemaV3, int... recordsCount) + private void prepareTable(String tableName, boolean schemaV3) throws IOException { switch (tableName) { @@ -445,12 +445,8 @@ private void prepareTable(String tableName, boolean schemaV3, int... recordsCoun .setPath(tempDir.toPath()).addTable(KEY_TABLE).build(); Table keyTable = dbStore.getTable(KEY_TABLE); - // Insert the number of keys specified by recordsCount[0] - int count = 5; - if (recordsCount.length > 0) { - count = recordsCount[0]; - } - for (int i = 1; i <= count; i++) { + // Insert 5 keys + for (int i = 1; i <= 5; i++) { String key = "key" + i; OmKeyInfo value = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", key, ReplicationConfig.fromProtoTypeAndFactor(STAND_ALONE, HddsProtos.ReplicationFactor.ONE)).build(); @@ -507,6 +503,28 @@ private void prepareTable(String tableName, boolean schemaV3, int... recordsCoun } } + /** + * Prepare the keytable for testing. + * @param recordsCount prepare the number of keys + */ + private void prepareKeyTable(int recordsCount) throws IOException { + if (recordsCount < 1) { + throw new IllegalArgumentException("recordsCount must be greater than 1."); + } + dbStore = DBStoreBuilder.newBuilder(conf).setName("om.db") + .setPath(tempDir.toPath()).addTable(KEY_TABLE).build(); + Table keyTable = dbStore.getTable(KEY_TABLE); + for (int i = 1; i <= recordsCount; i++) { + String key = "key" + i; + OmKeyInfo value = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", + key, ReplicationConfig.fromProtoTypeAndFactor(STAND_ALONE, + HddsProtos.ReplicationFactor.ONE)).build(); + keyTable.put(key.getBytes(UTF_8), value.getProtobuf(ClientVersion.CURRENT_VERSION).toByteArray()); + // Populate map + dbMap.put(key, toMap(value)); + } + } + private static Map toMap(Object obj) throws IOException { ObjectWriter objectWriter = DBScanner.JsonSerializationHelper.getWriter(); String json = objectWriter.writeValueAsString(obj); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index 89486b7e3aa5..bd5b1ed6c1c5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -251,15 +251,13 @@ private boolean displayTable(ManagedRocksIterator iterator, } // If there are no parent directories, create them - if (recordsPerFile > 0) { - File file = new File(fileName); - File parentFile = file.getParentFile(); - if (!parentFile.exists()) { - boolean flg = parentFile.mkdirs(); - if (!flg) { - throw new IOException("An exception occurred while creating " + - "the directory. Directorys: " + parentFile.getAbsolutePath()); - } + File file = new File(fileName); + File parentFile = file.getParentFile(); + if (!parentFile.exists()) { + boolean flg = parentFile.mkdirs(); + if (!flg) { + throw new IOException("An exception occurred while creating " + + "the directory. Directorys: " + parentFile.getAbsolutePath()); } } From 2844c831499171f10d8565ab27a8653b73836329 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Wed, 27 Nov 2024 15:25:43 +0800 Subject: [PATCH 7/8] Update some unit tests. --- .../org/apache/hadoop/ozone/debug/TestLDBCli.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index ecc41e2f6b46..7b4874f13428 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -17,10 +17,9 @@ package org.apache.hadoop.ozone.debug; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.gson.JsonElement; -import com.google.gson.JsonParser; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.client.BlockID; @@ -54,8 +53,6 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import java.io.FileInputStream; -import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -370,10 +367,8 @@ void testScanWithRecordsPerFile() throws IOException { assertNotNull(subFiles); assertEquals(Math.ceil(recordsCount / (maxRecordsPerFile * 1.0)), subFiles.length); for (File subFile : subFiles) { - try (FileInputStream inputStream = new FileInputStream(subFile)) { - JsonElement jsonElement = JsonParser.parseReader(new InputStreamReader(inputStream, "UTF-8")); - assertTrue(jsonElement.isJsonArray() || jsonElement.isJsonObject()); - } + JsonNode jsonNode = MAPPER.readTree(subFile); + assertNotNull(jsonNode); } String scanDir2 = tempDir.getAbsolutePath() + "/scandir2"; From f6524dc10049a6a6d461402c79ff454af28bcdec Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Thu, 28 Nov 2024 10:56:00 +0800 Subject: [PATCH 8/8] Update some unit tests. --- .../apache/hadoop/ozone/debug/TestLDBCli.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java index 7b4874f13428..b1b073619ba7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java @@ -435,22 +435,7 @@ private void prepareTable(String tableName, boolean schemaV3) switch (tableName) { case KEY_TABLE: - // Dummy om.db with only keyTable - dbStore = DBStoreBuilder.newBuilder(conf).setName("om.db") - .setPath(tempDir.toPath()).addTable(KEY_TABLE).build(); - - Table keyTable = dbStore.getTable(KEY_TABLE); - // Insert 5 keys - for (int i = 1; i <= 5; i++) { - String key = "key" + i; - OmKeyInfo value = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", - key, ReplicationConfig.fromProtoTypeAndFactor(STAND_ALONE, HddsProtos.ReplicationFactor.ONE)).build(); - keyTable.put(key.getBytes(UTF_8), - value.getProtobuf(ClientVersion.CURRENT_VERSION).toByteArray()); - - // Populate map - dbMap.put(key, toMap(value)); - } + prepareKeyTable(5); break; case BLOCK_DATA: @@ -506,6 +491,7 @@ private void prepareKeyTable(int recordsCount) throws IOException { if (recordsCount < 1) { throw new IllegalArgumentException("recordsCount must be greater than 1."); } + // Dummy om.db with only keyTable dbStore = DBStoreBuilder.newBuilder(conf).setName("om.db") .setPath(tempDir.toPath()).addTable(KEY_TABLE).build(); Table keyTable = dbStore.getTable(KEY_TABLE);