From 4c2cc488a15b606d46167c0888df6c016140cd3f Mon Sep 17 00:00:00 2001 From: guohao1 Date: Mon, 15 May 2023 16:00:44 +0800 Subject: [PATCH 1/3] HDDS-8626. Config thread pool in ReplicationServer --- .../replication/ReplicationServer.java | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java index 5472a24be7f4..feaa57d7861a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ozone.container.replication; import java.io.IOException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.conf.Config; @@ -30,6 +32,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController; +import org.apache.ratis.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.ratis.thirdparty.io.grpc.Server; import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors; import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts; @@ -62,6 +65,8 @@ public class ReplicationServer { private int port; private final ContainerImporter importer; + private ThreadPoolExecutor executor; + public ReplicationServer(ContainerController controller, ReplicationConfig replicationConfig, SecurityConfig secConf, CertificateClient caClient, ContainerImporter importer) { @@ -70,6 +75,18 @@ public ReplicationServer(ContainerController controller, this.controller = controller; this.importer = importer; this.port = replicationConfig.getPort(); + + int replicationServerWorkers = + replicationConfig.getReplicationServerWorkers(); + this.executor = + new ThreadPoolExecutor(replicationServerWorkers, + replicationServerWorkers, + 60, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("ReplicationContainerReader-%d") + .build()); + init(); } @@ -79,7 +96,8 @@ public void init() { .addService(ServerInterceptors.intercept(new GrpcReplicationService( new OnDemandContainerReplicationSource(controller), importer - ), new GrpcServerInterceptor())); + ), new GrpcServerInterceptor())) + .executor(executor); if (secConf.isSecurityEnabled() && secConf.isGrpcTlsEnabled()) { try { @@ -112,6 +130,8 @@ public void start() throws IOException { public void stop() { try { + executor.shutdown(); + executor.awaitTermination(5L, TimeUnit.SECONDS); server.shutdown().awaitTermination(10L, TimeUnit.SECONDS); } catch (InterruptedException ex) { LOG.warn("{} couldn't be stopped gracefully", getClass().getSimpleName()); @@ -174,6 +194,20 @@ public static final class ReplicationConfig { ) private double outOfServiceFactor = OUTOFSERVICE_FACTOR_DEFAULT; + @Config(key = "server.workers", defaultValue = "10", description = "server workers.", tags = { + DATANODE, MANAGEMENT}) + private int replicationServerWorkers; + + public int getReplicationServerWorkers() { + return replicationServerWorkers; + } + + public ReplicationConfig setReplicationServerWorkers( + int replicationServerWorkers) { + this.replicationServerWorkers = replicationServerWorkers; + return this; + } + public double getOutOfServiceFactor() { return outOfServiceFactor; } From ab006332123bb7fd502f0458a9aa591781bac111 Mon Sep 17 00:00:00 2001 From: guohao1 Date: Tue, 16 May 2023 14:42:35 +0800 Subject: [PATCH 2/3] change config --- .../container/replication/ReplicationServer.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java index feaa57d7861a..799a0e7b58ea 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java @@ -77,7 +77,7 @@ public ReplicationServer(ContainerController controller, this.port = replicationConfig.getPort(); int replicationServerWorkers = - replicationConfig.getReplicationServerWorkers(); + replicationConfig.getReplicationMaxStreams(); this.executor = new ThreadPoolExecutor(replicationServerWorkers, replicationServerWorkers, @@ -194,20 +194,6 @@ public static final class ReplicationConfig { ) private double outOfServiceFactor = OUTOFSERVICE_FACTOR_DEFAULT; - @Config(key = "server.workers", defaultValue = "10", description = "server workers.", tags = { - DATANODE, MANAGEMENT}) - private int replicationServerWorkers; - - public int getReplicationServerWorkers() { - return replicationServerWorkers; - } - - public ReplicationConfig setReplicationServerWorkers( - int replicationServerWorkers) { - this.replicationServerWorkers = replicationServerWorkers; - return this; - } - public double getOutOfServiceFactor() { return outOfServiceFactor; } From 29534c0125fe57ee9ec8bf211f02ea9d6e659c2d Mon Sep 17 00:00:00 2001 From: guohao1 Date: Tue, 16 May 2023 14:58:59 +0800 Subject: [PATCH 3/3] find bugs --- .../hadoop/ozone/container/replication/ReplicationServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java index 799a0e7b58ea..ee1faf8917d9 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationServer.java @@ -22,6 +22,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.hdds.conf.Config; import org.apache.hadoop.hdds.conf.ConfigGroup; import org.apache.hadoop.hdds.conf.ConfigType; @@ -32,7 +33,6 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController; -import org.apache.ratis.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.ratis.thirdparty.io.grpc.Server; import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors; import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts;