From afbd320bb8e8108ab1a3fe19f78b3136b3f0c3e6 Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Thu, 3 Apr 2025 10:29:24 +0530 Subject: [PATCH 1/4] HDDS-10091. Updating column family for FILE_SYSTEM_OPTIMIZED --- .../org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index 6e4e9ab063c2..6b36c8a0bbe0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -135,7 +135,7 @@ public void testLdbCliForOzoneSnapshot() throws Exception { GenericTestUtils .waitFor(() -> new File(snapshotCurrent).exists(), 1000, 120000); String[] args = - new String[] {"--db=" + dbPath, "scan", "--cf", "keyTable"}; + new String[] {"--db=" + dbPath, "scan", "--cf", "fileTable"}; int exitCode = cmd.execute(args); assertEquals(0, exitCode); String cmdOut = stdout.toString(); @@ -157,9 +157,8 @@ private void writeKey(String volumeName, String bucketName, repConfig = ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE); } - // see HDDS-10091 for making this work with FILE_SYSTEM_OPTIMIZED layout TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, - BucketLayout.LEGACY); + BucketLayout.FILE_SYSTEM_OPTIMIZED); TestDataUtil.createKey( client.getObjectStore().getVolume(volumeName).getBucket(bucketName), keyName, repConfig, "test".getBytes(StandardCharsets.UTF_8)); From 12c03cd218c9fc11c63d27a7f7b8afd225cba5c4 Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Fri, 4 Apr 2025 14:20:38 +0530 Subject: [PATCH 2/4] ParameterizedTest to check all layouts --- .../ozone/shell/TestOzoneDebugShell.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index 6b36c8a0bbe0..a6f908d9b997 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -58,9 +58,9 @@ import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import picocli.CommandLine; @@ -85,13 +85,15 @@ void shutdown() { } @ParameterizedTest - @ValueSource(booleans = {true, false}) - public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey) throws Exception { + @CsvSource(value = {"true,FILE_SYSTEM_OPTIMIZED", "false,FILE_SYSTEM_OPTIMIZED", + "true,LEGACY", "false,LEGACY", + "true,OBJECT_STORE", "false,OBJECT_STORE"}) + public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey, BucketLayout layout) throws Exception { final String volumeName = UUID.randomUUID().toString(); final String bucketName = UUID.randomUUID().toString(); final String keyName = UUID.randomUUID().toString(); - writeKey(volumeName, bucketName, keyName, isEcKey); + writeKey(volumeName, bucketName, keyName, isEcKey, layout); int exitCode = runChunkInfoCommand(volumeName, bucketName, keyName); assertEquals(0, exitCode); @@ -102,18 +104,20 @@ public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey) throws Ex assertEquals(0, exitCode); } - @Test - public void testChunkInfoVerifyPathsAreDifferent() throws Exception { + @ParameterizedTest + @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "LEGACY", "OBJECT_STORE"}) + public void testChunkInfoVerifyPathsAreDifferent(BucketLayout layout) throws Exception { final String volumeName = UUID.randomUUID().toString(); final String bucketName = UUID.randomUUID().toString(); final String keyName = UUID.randomUUID().toString(); - writeKey(volumeName, bucketName, keyName, false); + writeKey(volumeName, bucketName, keyName, false, layout); int exitCode = runChunkInfoAndVerifyPaths(volumeName, bucketName, keyName); assertEquals(0, exitCode); } - @Test - public void testLdbCliForOzoneSnapshot() throws Exception { + @ParameterizedTest + @CsvSource(value = {"FILE_SYSTEM_OPTIMIZED,fileTable", "LEGACY,keyTable", "OBJECT_STORE,keyTable"}) + public void testLdbCliForOzoneSnapshot(BucketLayout layout, String columnFamily) throws Exception { StringWriter stdout = new StringWriter(); PrintWriter pstdout = new PrintWriter(stdout); CommandLine cmd = new CommandLine(new RDBParser()) @@ -122,7 +126,7 @@ public void testLdbCliForOzoneSnapshot() throws Exception { final String bucketName = UUID.randomUUID().toString(); final String keyName = UUID.randomUUID().toString(); - writeKey(volumeName, bucketName, keyName, false); + writeKey(volumeName, bucketName, keyName, false, layout); String snapshotName = client.getObjectStore().createSnapshot(volumeName, bucketName, "snap1"); @@ -135,7 +139,7 @@ public void testLdbCliForOzoneSnapshot() throws Exception { GenericTestUtils .waitFor(() -> new File(snapshotCurrent).exists(), 1000, 120000); String[] args = - new String[] {"--db=" + dbPath, "scan", "--cf", "fileTable"}; + new String[] {"--db=" + dbPath, "scan", "--cf", columnFamily}; int exitCode = cmd.execute(args); assertEquals(0, exitCode); String cmdOut = stdout.toString(); @@ -149,7 +153,7 @@ private String getSnapshotDBPath(String checkPointDir) { } private void writeKey(String volumeName, String bucketName, - String keyName, boolean isEcKey) throws IOException { + String keyName, boolean isEcKey, BucketLayout layout) throws IOException { ReplicationConfig repConfig; if (isEcKey) { repConfig = new ECReplicationConfig(3, 2); @@ -158,7 +162,7 @@ private void writeKey(String volumeName, String bucketName, ReplicationFactor.THREE); } TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, - BucketLayout.FILE_SYSTEM_OPTIMIZED); + layout); TestDataUtil.createKey( client.getObjectStore().getVolume(volumeName).getBucket(bucketName), keyName, repConfig, "test".getBytes(StandardCharsets.UTF_8)); From 4ec0394aacb469b6acff27bd59dc032e4faf31de Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Sun, 6 Apr 2025 01:05:57 +0530 Subject: [PATCH 3/4] addressing review comments --- .../ozone/shell/TestOzoneDebugShell.java | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index a6f908d9b997..f23f489445e2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -20,8 +20,11 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -39,6 +42,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -50,7 +54,10 @@ import org.apache.hadoop.ozone.debug.OzoneDebug; import org.apache.hadoop.ozone.debug.ldb.RDBParser; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -59,8 +66,9 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; import picocli.CommandLine; @@ -72,11 +80,26 @@ public abstract class TestOzoneDebugShell implements NonHATests.TestCase { private OzoneClient client; private OzoneDebug ozoneDebugShell; + private OMMetadataManager omMetadataManager; + private OzoneConfiguration conf; + private OzoneManager ozoneManager; + @TempDir + private java.nio.file.Path folder; @BeforeEach void init() throws Exception { ozoneDebugShell = new OzoneDebug(); client = cluster().newClient(); + + conf = new OzoneConfiguration(); + conf.set(OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); + ozoneManager = mock(OzoneManager.class); + when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); + + OzoneConfiguration ozoneConfiguration = mock(OzoneConfiguration.class); + when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); + + omMetadataManager = new OmMetadataManagerImpl(conf, ozoneManager); } @AfterEach @@ -85,15 +108,13 @@ void shutdown() { } @ParameterizedTest - @CsvSource(value = {"true,FILE_SYSTEM_OPTIMIZED", "false,FILE_SYSTEM_OPTIMIZED", - "true,LEGACY", "false,LEGACY", - "true,OBJECT_STORE", "false,OBJECT_STORE"}) - public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey, BucketLayout layout) throws Exception { + @ValueSource(booleans = {true, false}) + public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey) throws Exception { final String volumeName = UUID.randomUUID().toString(); final String bucketName = UUID.randomUUID().toString(); final String keyName = UUID.randomUUID().toString(); - writeKey(volumeName, bucketName, keyName, isEcKey, layout); + writeKey(volumeName, bucketName, keyName, isEcKey, BucketLayout.FILE_SYSTEM_OPTIMIZED); int exitCode = runChunkInfoCommand(volumeName, bucketName, keyName); assertEquals(0, exitCode); @@ -105,7 +126,7 @@ public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey, BucketLay } @ParameterizedTest - @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "LEGACY", "OBJECT_STORE"}) + @EnumSource public void testChunkInfoVerifyPathsAreDifferent(BucketLayout layout) throws Exception { final String volumeName = UUID.randomUUID().toString(); final String bucketName = UUID.randomUUID().toString(); @@ -116,8 +137,9 @@ public void testChunkInfoVerifyPathsAreDifferent(BucketLayout layout) throws Exc } @ParameterizedTest - @CsvSource(value = {"FILE_SYSTEM_OPTIMIZED,fileTable", "LEGACY,keyTable", "OBJECT_STORE,keyTable"}) - public void testLdbCliForOzoneSnapshot(BucketLayout layout, String columnFamily) throws Exception { + @EnumSource + public void testLdbCliForOzoneSnapshot(BucketLayout layout) throws Exception { + String columnFamily = omMetadataManager.getKeyTable(layout).getName(); StringWriter stdout = new StringWriter(); PrintWriter pstdout = new PrintWriter(stdout); CommandLine cmd = new CommandLine(new RDBParser()) From 377178d421e9f5684e75c253c8fe3556c14209b9 Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Sun, 6 Apr 2025 12:20:28 +0530 Subject: [PATCH 4/4] Using running OM instead of mocking --- .../ozone/shell/TestOzoneDebugShell.java | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index f23f489445e2..bc5ee8cc3080 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -20,11 +20,8 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -42,7 +39,6 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -56,8 +52,6 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMStorage; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -66,7 +60,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; @@ -81,25 +74,12 @@ public abstract class TestOzoneDebugShell implements NonHATests.TestCase { private OzoneClient client; private OzoneDebug ozoneDebugShell; private OMMetadataManager omMetadataManager; - private OzoneConfiguration conf; - private OzoneManager ozoneManager; - @TempDir - private java.nio.file.Path folder; @BeforeEach void init() throws Exception { ozoneDebugShell = new OzoneDebug(); client = cluster().newClient(); - - conf = new OzoneConfiguration(); - conf.set(OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); - ozoneManager = mock(OzoneManager.class); - when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); - - OzoneConfiguration ozoneConfiguration = mock(OzoneConfiguration.class); - when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); - - omMetadataManager = new OmMetadataManagerImpl(conf, ozoneManager); + omMetadataManager = cluster().getOzoneManager().getMetadataManager(); } @AfterEach