From 72ed615305d73201c8888b135cf32dc6d59d9f3a Mon Sep 17 00:00:00 2001 From: chungen0126 Date: Fri, 21 Jun 2024 16:59:44 +0800 Subject: [PATCH 1/6] refactor TestContainerMetrics --- .../common/helpers/ContainerMetrics.java | 4 + .../container/common/impl/HddsDispatcher.java | 8 +- .../metrics/TestContainerMetrics.java | 142 +++++++++++------- 3 files changed, 101 insertions(+), 53 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java index 4efa935bc8a2..a717b24749c7 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java @@ -110,6 +110,10 @@ public void incContainerOpsMetrics(ContainerProtos.Type type) { numOpsArray.get(type).incr(); } + public long getContainerOpsMetrics(ContainerProtos.Type type) { + return numOpsArray.get(type).value(); + } + public void incContainerOpsLatencies(ContainerProtos.Type type, long latencyMillis) { opsLatency.get(type).add(latencyMillis); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index e494243ccc1c..f39e91a5fcc1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -216,7 +216,6 @@ private ContainerCommandResponseProto dispatchRequest( long startTime = Time.monotonicNow(); Type cmdType = msg.getCmdType(); long containerID = msg.getContainerID(); - metrics.incContainerOpsMetrics(cmdType); Container container = getContainer(containerID); boolean isWriteStage = (cmdType == Type.WriteChunk && dispatcherContext != null @@ -227,6 +226,13 @@ private ContainerCommandResponseProto dispatchRequest( (cmdType == Type.WriteChunk && dispatcherContext != null && dispatcherContext.getStage() == DispatcherContext.WriteChunkStage.COMMIT_DATA); + if (isWriteStage) { + metrics.incContainerOpsMetrics(cmdType); + } else if (cmdType != Type.WriteChunk) { + metrics.incContainerOpsMetrics(cmdType); + } else if (dispatcherContext == null) { + metrics.incContainerOpsMetrics(cmdType); + } try { if (DispatcherContext.op(dispatcherContext).validateToken()) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java index a4a5701f5491..7ebf789c6405 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java @@ -18,7 +18,9 @@ package org.apache.hadoop.ozone.container.metrics; import java.io.File; +import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; @@ -32,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientGrpc; +import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.pipeline.MockPipeline; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.metrics2.MetricsRecordBuilder; @@ -44,6 +47,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc; +import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; @@ -52,16 +56,21 @@ import org.apache.ozone.test.GenericTestUtils; import com.google.common.collect.Maps; + import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.ozone.test.MetricsAsserts.assertCounter; import static org.apache.ozone.test.MetricsAsserts.assertQuantileGauges; import static org.apache.ozone.test.MetricsAsserts.getMetrics; import static org.junit.jupiter.api.Assertions.assertEquals; +import org.apache.ratis.util.function.CheckedBiConsumer; +import org.apache.ratis.util.function.CheckedBiFunction; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; +import javax.xml.crypto.Data; + /** * Test for metrics published by storage containers. */ @@ -69,70 +78,98 @@ public class TestContainerMetrics { @TempDir private Path tempDir; + private static final OzoneConfiguration CONF = new OzoneConfiguration(); + private static final int DFS_METRICS_PERCENTILES_INTERVALS = 1; @Test public void testContainerMetrics() throws Exception { - XceiverServerGrpc server = null; - XceiverClientGrpc client = null; - long containerID = ContainerTestHelper.getTestContainerID(); + DatanodeDetails dd = randomDatanodeDetails(); String path = GenericTestUtils.getRandomizedTempPath(); + MutableVolumeSet volumeSet = createVolumeSet(dd, path); + HddsDispatcher hddsDispatcher = createDispatcher(dd, CONF, volumeSet); + runTestClientServer(volumeSet, (pipeline, conf) -> conf + .setInt(OzoneConfigKeys.HDDS_CONTAINER_IPC_PORT, + pipeline.getFirstNode() + .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue()), + XceiverClientGrpc::new, + (dn, conf) -> new XceiverServerGrpc(dd, conf, + hddsDispatcher, null), (dn, p) -> { + }); + } + + private MutableVolumeSet createVolumeSet(DatanodeDetails dn, String path) throws IOException { + CONF.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, path); + CONF.set(OzoneConfigKeys.OZONE_METADATA_DIRS, path); + return new MutableVolumeSet( + dn.getUuidString(), CONF, + null, StorageVolume.VolumeType.DATA_VOLUME, null); + } + + private HddsDispatcher createDispatcher(DatanodeDetails dd, + OzoneConfiguration conf, + VolumeSet volumeSet) { + conf.setInt(DFSConfigKeysLegacy.DFS_METRICS_PERCENTILES_INTERVALS_KEY, + DFS_METRICS_PERCENTILES_INTERVALS); + ContainerSet containerSet = new ContainerSet(1000); + StateContext context = ContainerTestUtils.getMockContext( + dd, conf); + ContainerMetrics metrics = ContainerMetrics.create(conf); + Map handlers = Maps.newHashMap(); + for (ContainerProtos.ContainerType containerType : + ContainerProtos.ContainerType.values()) { + handlers.put(containerType, + Handler.getHandlerForContainerType(containerType, conf, + context.getParent().getDatanodeDetails().getUuidString(), + containerSet, volumeSet, metrics, + c -> { })); + } + HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet, + volumeSet, handlers, context, metrics, null); + StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()) + .forEach(hddsVolume -> hddsVolume.setDbParentDir(tempDir.toFile())); + dispatcher.setClusterId(UUID.randomUUID().toString()); + return dispatcher; + } + + static void runTestClientServer( + MutableVolumeSet volumeSet, + CheckedBiConsumer initConf, + CheckedBiFunction createClient, + CheckedBiFunction createServer, + CheckedBiConsumer initServer) + throws Exception { + final List servers = new ArrayList<>(); + XceiverClientSpi client = null; + long containerID = ContainerTestHelper.getTestContainerID(); try { - final int interval = 1; - Pipeline pipeline = MockPipeline - .createSingleNodePipeline(); - OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OzoneConfigKeys.HDDS_CONTAINER_IPC_PORT, - pipeline.getFirstNode() - .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue()); - conf.setInt(DFSConfigKeysLegacy.DFS_METRICS_PERCENTILES_INTERVALS_KEY, - interval); - - DatanodeDetails datanodeDetails = randomDatanodeDetails(); - conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, path); - conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, path); - VolumeSet volumeSet = new MutableVolumeSet( - datanodeDetails.getUuidString(), conf, - null, StorageVolume.VolumeType.DATA_VOLUME, null); - ContainerSet containerSet = new ContainerSet(1000); - StateContext context = ContainerTestUtils.getMockContext( - datanodeDetails, conf); - ContainerMetrics metrics = ContainerMetrics.create(conf); - Map handlers = Maps.newHashMap(); - for (ContainerProtos.ContainerType containerType : - ContainerProtos.ContainerType.values()) { - handlers.put(containerType, - Handler.getHandlerForContainerType(containerType, conf, - context.getParent().getDatanodeDetails().getUuidString(), - containerSet, volumeSet, metrics, - c -> { })); + final Pipeline pipeline = + MockPipeline.createSingleNodePipeline(); + initConf.accept(pipeline, CONF); + + for (DatanodeDetails dn : pipeline.getNodes()) { + final XceiverServerSpi s = createServer.apply(dn, CONF); + servers.add(s); + s.start(); + initServer.accept(dn, pipeline); } - HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet, - volumeSet, handlers, context, metrics, null); - StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()) - .forEach(hddsVolume -> hddsVolume.setDbParentDir(tempDir.toFile())); - dispatcher.setClusterId(UUID.randomUUID().toString()); - server = new XceiverServerGrpc(datanodeDetails, conf, dispatcher, null); - client = new XceiverClientGrpc(pipeline, conf); - - server.start(); + client = createClient.apply(pipeline, CONF); client.connect(); // Write Chunk BlockID blockID = ContainerTestHelper.getTestBlockID(containerID); - ContainerTestHelper.getWriteChunkRequest( - pipeline, blockID, 1024); - ContainerProtos.ContainerCommandRequestProto writeChunkRequest = + final ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper.getWriteChunkRequest( pipeline, blockID, 1024); - ContainerCommandResponseProto response = - client.sendCommand(writeChunkRequest); + ContainerCommandResponseProto response = client.sendCommand(writeChunkRequest); assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); //Read Chunk - ContainerProtos.ContainerCommandRequestProto readChunkRequest = + final ContainerProtos.ContainerCommandRequestProto readChunkRequest = ContainerTestHelper.getReadChunkRequest(pipeline, writeChunkRequest .getWriteChunk()); response = client.sendCommand(readChunkRequest); @@ -147,8 +184,8 @@ public void testContainerMetrics() throws Exception { assertCounter("bytesWriteChunk", 1024L, containerMetrics); assertCounter("bytesReadChunk", 1024L, containerMetrics); - String sec = interval + "s"; - Thread.sleep((interval + 1) * 1000); + String sec = DFS_METRICS_PERCENTILES_INTERVALS + "s"; + Thread.sleep((DFS_METRICS_PERCENTILES_INTERVALS + 1) * 1000); assertQuantileGauges("WriteChunkNanos" + sec, containerMetrics); // Check VolumeIOStats metrics @@ -163,17 +200,18 @@ public void testContainerMetrics() throws Exception { assertCounter("WriteOpCount", 1L, volumeIOMetrics); } finally { + ContainerMetrics.remove(); + volumeSet.shutdown(); if (client != null) { client.close(); } - if (server != null) { - server.stop(); - } + servers.forEach(XceiverServerSpi::stop); // clean up volume dir - File file = new File(path); + File file = new File(""); if (file.exists()) { FileUtil.fullyDelete(file); } } } + } From 66f64595bd9fece4b246553d4d0364c2c380c9f3 Mon Sep 17 00:00:00 2001 From: chungen0126 Date: Fri, 21 Jun 2024 18:39:30 +0800 Subject: [PATCH 2/6] Add ratis test in TestContainerMetrics --- .../metrics/TestContainerMetrics.java | 107 ++++++++++++------ 1 file changed, 72 insertions(+), 35 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java index 7ebf789c6405..507e36ce01ac 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java @@ -34,70 +34,96 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientGrpc; +import org.apache.hadoop.hdds.scm.XceiverClientRatis; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.pipeline.MockPipeline; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.RatisTestHelper; import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.common.ContainerTestUtils; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; +import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi; +import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; +import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController; import org.apache.ozone.test.GenericTestUtils; import com.google.common.collect.Maps; -import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.ozone.test.MetricsAsserts.assertCounter; import static org.apache.ozone.test.MetricsAsserts.assertQuantileGauges; import static org.apache.ozone.test.MetricsAsserts.getMetrics; +import static org.apache.ratis.rpc.SupportedRpcType.GRPC; import static org.junit.jupiter.api.Assertions.assertEquals; import org.apache.ratis.util.function.CheckedBiConsumer; import org.apache.ratis.util.function.CheckedBiFunction; +import org.apache.ratis.util.function.CheckedConsumer; +import org.apache.ratis.util.function.CheckedFunction; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; -import javax.xml.crypto.Data; - /** * Test for metrics published by storage containers. */ @Timeout(300) public class TestContainerMetrics { + static final String TEST_DIR = GenericTestUtils.getRandomizedTempPath() + File.separator; @TempDir private Path tempDir; private static final OzoneConfiguration CONF = new OzoneConfiguration(); private static final int DFS_METRICS_PERCENTILES_INTERVALS = 1; + @BeforeAll + public static void setup() { + DefaultMetricsSystem.setMiniClusterMode(true); + CONF.setInt(DFSConfigKeysLegacy.DFS_METRICS_PERCENTILES_INTERVALS_KEY, + DFS_METRICS_PERCENTILES_INTERVALS); + CONF.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, false); + } + @Test public void testContainerMetrics() throws Exception { - DatanodeDetails dd = randomDatanodeDetails(); - String path = GenericTestUtils.getRandomizedTempPath(); - MutableVolumeSet volumeSet = createVolumeSet(dd, path); - HddsDispatcher hddsDispatcher = createDispatcher(dd, CONF, volumeSet); - runTestClientServer(volumeSet, (pipeline, conf) -> conf + runTestClientServer(1, pipeline -> CONF .setInt(OzoneConfigKeys.HDDS_CONTAINER_IPC_PORT, pipeline.getFirstNode() .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue()), - XceiverClientGrpc::new, - (dn, conf) -> new XceiverServerGrpc(dd, conf, - hddsDispatcher, null), (dn, p) -> { + pipeline -> new XceiverClientGrpc(pipeline, CONF), + (dn, volumeSet) -> new XceiverServerGrpc(dn, CONF, + createDispatcher(dn, volumeSet), null), (dn, p) -> { }); } - private MutableVolumeSet createVolumeSet(DatanodeDetails dn, String path) throws IOException { + @Test + public void testContainerMetricsRatis() throws Exception { + runTestClientServer(1, + pipeline -> RatisTestHelper.initRatisConf(GRPC, CONF), + pipeline -> XceiverClientRatis.newXceiverClientRatis(pipeline, CONF), + this::newXceiverServerRatis, (dn, p) -> + RatisTestHelper.initXceiverServerRatis(GRPC, dn, p)); + runTestClientServer(3, + pipeline -> RatisTestHelper.initRatisConf(GRPC, CONF), + pipeline -> XceiverClientRatis.newXceiverClientRatis(pipeline, CONF), + this::newXceiverServerRatis, (dn, p) -> + RatisTestHelper.initXceiverServerRatis(GRPC, dn, p)); + } + + private static MutableVolumeSet createVolumeSet(DatanodeDetails dn, String path) throws IOException { CONF.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, path); CONF.set(OzoneConfigKeys.OZONE_METADATA_DIRS, path); return new MutableVolumeSet( @@ -105,25 +131,21 @@ private MutableVolumeSet createVolumeSet(DatanodeDetails dn, String path) throws null, StorageVolume.VolumeType.DATA_VOLUME, null); } - private HddsDispatcher createDispatcher(DatanodeDetails dd, - OzoneConfiguration conf, - VolumeSet volumeSet) { - conf.setInt(DFSConfigKeysLegacy.DFS_METRICS_PERCENTILES_INTERVALS_KEY, - DFS_METRICS_PERCENTILES_INTERVALS); + private HddsDispatcher createDispatcher(DatanodeDetails dd, VolumeSet volumeSet) { ContainerSet containerSet = new ContainerSet(1000); StateContext context = ContainerTestUtils.getMockContext( - dd, conf); - ContainerMetrics metrics = ContainerMetrics.create(conf); + dd, CONF); + ContainerMetrics metrics = ContainerMetrics.create(CONF); Map handlers = Maps.newHashMap(); for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) { handlers.put(containerType, - Handler.getHandlerForContainerType(containerType, conf, + Handler.getHandlerForContainerType(containerType, CONF, context.getParent().getDatanodeDetails().getUuidString(), containerSet, volumeSet, metrics, c -> { })); } - HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet, + HddsDispatcher dispatcher = new HddsDispatcher(CONF, containerSet, volumeSet, handlers, context, metrics, null); StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()) .forEach(hddsVolume -> hddsVolume.setDbParentDir(tempDir.toFile())); @@ -132,31 +154,34 @@ private HddsDispatcher createDispatcher(DatanodeDetails dd, } static void runTestClientServer( - MutableVolumeSet volumeSet, - CheckedBiConsumer initConf, - CheckedBiFunction createClient, - CheckedBiFunction initConf, + CheckedFunction createClient, + CheckedBiFunction createServer, CheckedBiConsumer initServer) throws Exception { final List servers = new ArrayList<>(); XceiverClientSpi client = null; long containerID = ContainerTestHelper.getTestContainerID(); + List mutableVolumeSetList = new ArrayList<>(); try { final Pipeline pipeline = - MockPipeline.createSingleNodePipeline(); - initConf.accept(pipeline, CONF); + MockPipeline.createPipeline(numDatanodes); + initConf.accept(pipeline); for (DatanodeDetails dn : pipeline.getNodes()) { - final XceiverServerSpi s = createServer.apply(dn, CONF); + final MutableVolumeSet volumeSet = createVolumeSet(dn, TEST_DIR + dn.getUuidString()); + mutableVolumeSetList.add(volumeSet); + final XceiverServerSpi s = createServer.apply(dn, volumeSet); servers.add(s); s.start(); initServer.accept(dn, pipeline); } - client = createClient.apply(pipeline, CONF); + client = createClient.apply(pipeline); client.connect(); // Write Chunk @@ -188,9 +213,8 @@ static void runTestClientServer( Thread.sleep((DFS_METRICS_PERCENTILES_INTERVALS + 1) * 1000); assertQuantileGauges("WriteChunkNanos" + sec, containerMetrics); - // Check VolumeIOStats metrics List volumes = - StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()); + StorageVolumeUtil.getHddsVolumesList(mutableVolumeSetList.get(0).getVolumesList()); HddsVolume hddsVolume = volumes.get(0); MetricsRecordBuilder volumeIOMetrics = getMetrics(hddsVolume.getVolumeIOStats().getMetricsSourceName()); @@ -198,20 +222,33 @@ static void runTestClientServer( assertCounter("ReadOpCount", 1L, volumeIOMetrics); assertCounter("WriteBytes", 1024L, volumeIOMetrics); assertCounter("WriteOpCount", 1L, volumeIOMetrics); - } finally { ContainerMetrics.remove(); - volumeSet.shutdown(); + for (MutableVolumeSet volumeSet : mutableVolumeSetList) { + volumeSet.shutdown(); + } if (client != null) { client.close(); } servers.forEach(XceiverServerSpi::stop); // clean up volume dir - File file = new File(""); + File file = new File(TEST_DIR); if (file.exists()) { FileUtil.fullyDelete(file); } } } + private XceiverServerSpi newXceiverServerRatis(DatanodeDetails dn, MutableVolumeSet volumeSet) + throws IOException { + CONF.setInt(OzoneConfigKeys.HDDS_CONTAINER_RATIS_IPC_PORT, + dn.getPort(DatanodeDetails.Port.Name.RATIS).getValue()); + final String dir = TEST_DIR + dn.getUuid(); + CONF.set(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATANODE_STORAGE_DIR, dir); + final ContainerDispatcher dispatcher = createDispatcher(dn, + volumeSet); + return XceiverServerRatis.newXceiverServerRatis(dn, CONF, dispatcher, + new ContainerController(new ContainerSet(1000), Maps.newHashMap()), + null, null); + } } From 55ae901ee8b03334875519f837e79ef7ee9e7a3d Mon Sep 17 00:00:00 2001 From: chungen0126 Date: Fri, 21 Jun 2024 19:06:34 +0800 Subject: [PATCH 3/6] test --- .../metrics/TestContainerMetrics.java | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java index 507e36ce01ac..2a098dacd0f9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java @@ -20,11 +20,11 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.DFSConfigKeysLegacy; import org.apache.hadoop.hdds.client.BlockID; @@ -73,6 +73,8 @@ import org.apache.ratis.util.function.CheckedBiFunction; import org.apache.ratis.util.function.CheckedConsumer; import org.apache.ratis.util.function.CheckedFunction; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -95,11 +97,29 @@ public static void setup() { CONF.setInt(DFSConfigKeysLegacy.DFS_METRICS_PERCENTILES_INTERVALS_KEY, DFS_METRICS_PERCENTILES_INTERVALS); CONF.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, false); + CONF.set(OzoneConfigKeys.OZONE_METADATA_DIRS, TEST_DIR); + + } + + @AfterAll + public static void cleanup() { + // clean up volume dir + File file = new File(TEST_DIR); + if (file.exists()) { + FileUtil.fullyDelete(file); + } + } + + @AfterEach + public void cleanUp() throws IOException { + FileUtils.deleteQuietly(new File(CONF.get(ScmConfigKeys.HDDS_DATANODE_DIR_KEY))); + FileUtils.deleteQuietly(CONF.get(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATANODE_STORAGE_DIR) == null ? + null : new File(CONF.get(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATANODE_STORAGE_DIR))); } @Test public void testContainerMetrics() throws Exception { - runTestClientServer(1, pipeline -> CONF + runTestClientServer(pipeline -> CONF .setInt(OzoneConfigKeys.HDDS_CONTAINER_IPC_PORT, pipeline.getFirstNode() .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue()), @@ -111,12 +131,7 @@ public void testContainerMetrics() throws Exception { @Test public void testContainerMetricsRatis() throws Exception { - runTestClientServer(1, - pipeline -> RatisTestHelper.initRatisConf(GRPC, CONF), - pipeline -> XceiverClientRatis.newXceiverClientRatis(pipeline, CONF), - this::newXceiverServerRatis, (dn, p) -> - RatisTestHelper.initXceiverServerRatis(GRPC, dn, p)); - runTestClientServer(3, + runTestClientServer( pipeline -> RatisTestHelper.initRatisConf(GRPC, CONF), pipeline -> XceiverClientRatis.newXceiverClientRatis(pipeline, CONF), this::newXceiverServerRatis, (dn, p) -> @@ -125,7 +140,6 @@ public void testContainerMetricsRatis() throws Exception { private static MutableVolumeSet createVolumeSet(DatanodeDetails dn, String path) throws IOException { CONF.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, path); - CONF.set(OzoneConfigKeys.OZONE_METADATA_DIRS, path); return new MutableVolumeSet( dn.getUuidString(), CONF, null, StorageVolume.VolumeType.DATA_VOLUME, null); @@ -154,7 +168,6 @@ private HddsDispatcher createDispatcher(DatanodeDetails dd, VolumeSet volumeSet) } static void runTestClientServer( - int numDatanodes, CheckedConsumer initConf, CheckedFunction createClient, @@ -162,24 +175,21 @@ static void runTestClientServer( IOException> createServer, CheckedBiConsumer initServer) throws Exception { - final List servers = new ArrayList<>(); + XceiverServerSpi server = null; XceiverClientSpi client = null; long containerID = ContainerTestHelper.getTestContainerID(); - List mutableVolumeSetList = new ArrayList<>(); + MutableVolumeSet volumeSet = null; try { final Pipeline pipeline = - MockPipeline.createPipeline(numDatanodes); + MockPipeline.createSingleNodePipeline(); initConf.accept(pipeline); - for (DatanodeDetails dn : pipeline.getNodes()) { - final MutableVolumeSet volumeSet = createVolumeSet(dn, TEST_DIR + dn.getUuidString()); - mutableVolumeSetList.add(volumeSet); - final XceiverServerSpi s = createServer.apply(dn, volumeSet); - servers.add(s); - s.start(); - initServer.accept(dn, pipeline); - } + DatanodeDetails dn = pipeline.getFirstNode(); + volumeSet = createVolumeSet(dn, TEST_DIR + dn.getUuidString()); + server = createServer.apply(dn, volumeSet); + server.start(); + initServer.accept(dn, pipeline); client = createClient.apply(pipeline); client.connect(); @@ -214,7 +224,7 @@ static void runTestClientServer( assertQuantileGauges("WriteChunkNanos" + sec, containerMetrics); List volumes = - StorageVolumeUtil.getHddsVolumesList(mutableVolumeSetList.get(0).getVolumesList()); + StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()); HddsVolume hddsVolume = volumes.get(0); MetricsRecordBuilder volumeIOMetrics = getMetrics(hddsVolume.getVolumeIOStats().getMetricsSourceName()); @@ -224,17 +234,12 @@ static void runTestClientServer( assertCounter("WriteOpCount", 1L, volumeIOMetrics); } finally { ContainerMetrics.remove(); - for (MutableVolumeSet volumeSet : mutableVolumeSetList) { - volumeSet.shutdown(); - } + volumeSet.shutdown(); if (client != null) { client.close(); } - servers.forEach(XceiverServerSpi::stop); - // clean up volume dir - File file = new File(TEST_DIR); - if (file.exists()) { - FileUtil.fullyDelete(file); + if (server != null) { + server.stop(); } } } From 9ca2fb1bb0d8107f366f6b84eea6b97aa3366d55 Mon Sep 17 00:00:00 2001 From: chungen0126 Date: Fri, 21 Jun 2024 20:47:18 +0800 Subject: [PATCH 4/6] fix Nullpointer --- .../ozone/container/common/impl/HddsDispatcher.java | 9 ++++++--- .../ozone/container/metrics/TestContainerMetrics.java | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index f39e91a5fcc1..b47116950c53 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -226,11 +226,14 @@ private ContainerCommandResponseProto dispatchRequest( (cmdType == Type.WriteChunk && dispatcherContext != null && dispatcherContext.getStage() == DispatcherContext.WriteChunkStage.COMMIT_DATA); - if (isWriteStage) { + + if (dispatcherContext == null) { + // increase all op not through ratis metrics.incContainerOpsMetrics(cmdType); - } else if (cmdType != Type.WriteChunk) { + } else if (isWriteStage) { + // increase WriteChunk in only WRITE_STAGE metrics.incContainerOpsMetrics(cmdType); - } else if (dispatcherContext == null) { + } else if (cmdType != Type.WriteChunk) { metrics.incContainerOpsMetrics(cmdType); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java index 2a098dacd0f9..7c91ab2d1f8b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java @@ -234,7 +234,9 @@ static void runTestClientServer( assertCounter("WriteOpCount", 1L, volumeIOMetrics); } finally { ContainerMetrics.remove(); - volumeSet.shutdown(); + if (volumeSet != null) { + volumeSet.shutdown(); + } if (client != null) { client.close(); } From fc89a7e43d2ea77f517ae03d09a16510448e44dd Mon Sep 17 00:00:00 2001 From: chungen0126 Date: Fri, 21 Jun 2024 20:53:31 +0800 Subject: [PATCH 5/6] cleanup extra line --- .../hadoop/ozone/container/metrics/TestContainerMetrics.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java index 7c91ab2d1f8b..068cb01a9671 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java @@ -62,7 +62,6 @@ import org.apache.ozone.test.GenericTestUtils; import com.google.common.collect.Maps; - import static org.apache.ozone.test.MetricsAsserts.assertCounter; import static org.apache.ozone.test.MetricsAsserts.assertQuantileGauges; import static org.apache.ozone.test.MetricsAsserts.getMetrics; From ad4c17c2f9e13c6ec69b04b895dbcd95508280bd Mon Sep 17 00:00:00 2001 From: chungen0126 Date: Sat, 29 Jun 2024 02:10:12 +0800 Subject: [PATCH 6/6] remove getContainerOpsMetrics --- .../ozone/container/common/helpers/ContainerMetrics.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java index a717b24749c7..4efa935bc8a2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java @@ -110,10 +110,6 @@ public void incContainerOpsMetrics(ContainerProtos.Type type) { numOpsArray.get(type).incr(); } - public long getContainerOpsMetrics(ContainerProtos.Type type) { - return numOpsArray.get(type).value(); - } - public void incContainerOpsLatencies(ContainerProtos.Type type, long latencyMillis) { opsLatency.get(type).add(latencyMillis);