From 3951836889cd5f98de8b8f64e4bf9c14289b89c8 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 8 Nov 2019 16:04:15 +0000 Subject: [PATCH 1/2] Added the config key hdds.datanode.replication.streams.limit to make container replication threads configurable --- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 4 ++++ .../common/src/main/resources/ozone-default.xml | 8 ++++++++ .../statemachine/DatanodeStateMachine.java | 16 +++++++++++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 66ec5faa3210..0e9c109d0266 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -374,6 +374,10 @@ public final class OzoneConfigKeys { public static final String OZONE_CONTAINER_COPY_WORKDIR = "hdds.datanode.replication.work.dir"; + public static final String HDDS_DATANODE_REPLICATION_STREAMS_LIMIT = + "hdds.datanode.replication.streams.limit"; + public static final int HDDS_DATANODE_REPLICATION_STREAMS_LIMIT_DEFAULT = 10; + /** * Config properties to set client side checksum properties. */ diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 331f489a0dd0..f73e6e3adb9a 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1385,6 +1385,14 @@ + + hdds.datanode.replication.streams.limit + DATANODE + Maximum number of container replication commands which + can be executed simultaneously on a single datanode. + + + hdds.lock.max.concurrency 100 diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index c9eb7024eaf1..d1cdeb3bf0f4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -35,6 +35,7 @@ .StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.ozone.HddsDatanodeStopService; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.container.common.report.ReportManager; import org.apache.hadoop.ozone.container.common.statemachine.commandhandler .CloseContainerCommandHandler; @@ -115,8 +116,21 @@ public DatanodeStateMachine(DatanodeDetails datanodeDetails, container.getController(), new SimpleContainerDownloader(conf), new TarContainerPacker()); + int replicationThreads = conf.getInt( + OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT, + OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT_DEFAULT); + if (replicationThreads < 1) { + LOG.warn("{} is set to {} and must be greater than 0. Defaulting to {}", + OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT, + replicationThreads, + OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT_DEFAULT); + replicationThreads = + OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT_DEFAULT; + } + supervisor = - new ReplicationSupervisor(container.getContainerSet(), replicator, 10); + new ReplicationSupervisor(container.getContainerSet(), replicator, + replicationThreads); // When we add new handlers just adding a new handler here should do the // trick. From 92095a38e21e269a7cd67170c9202348c87f740f Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 8 Nov 2019 17:15:07 +0000 Subject: [PATCH 2/2] Switched to use the new config framework --- .../apache/hadoop/ozone/OzoneConfigKeys.java | 4 -- .../src/main/resources/ozone-default.xml | 8 --- .../apache/hadoop/hdds/conf/ConfigTag.java | 3 +- .../statemachine/DatanodeConfiguration.java | 61 +++++++++++++++++++ .../statemachine/DatanodeStateMachine.java | 21 ++----- 5 files changed, 69 insertions(+), 28 deletions(-) create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 0e9c109d0266..66ec5faa3210 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -374,10 +374,6 @@ public final class OzoneConfigKeys { public static final String OZONE_CONTAINER_COPY_WORKDIR = "hdds.datanode.replication.work.dir"; - public static final String HDDS_DATANODE_REPLICATION_STREAMS_LIMIT = - "hdds.datanode.replication.streams.limit"; - public static final int HDDS_DATANODE_REPLICATION_STREAMS_LIMIT_DEFAULT = 10; - /** * Config properties to set client side checksum properties. */ diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f73e6e3adb9a..331f489a0dd0 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1385,14 +1385,6 @@ - - hdds.datanode.replication.streams.limit - DATANODE - Maximum number of container replication commands which - can be executed simultaneously on a single datanode. - - - hdds.lock.max.concurrency 100 diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java index de50d2afe9ea..b969a68a92ab 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java @@ -40,5 +40,6 @@ public enum ConfigTag { STORAGE, PIPELINE, STANDALONE, - S3GATEWAY + S3GATEWAY, + DATANODE } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java new file mode 100644 index 000000000000..dd0fa6722ea6 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.container.common.statemachine; + +import org.apache.hadoop.hdds.conf.Config; +import org.apache.hadoop.hdds.conf.ConfigGroup; +import static org.apache.hadoop.hdds.conf.ConfigTag.DATANODE; +import org.apache.hadoop.hdds.conf.ConfigType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Configuration class used for high level datanode configuration parameters. + */ +@ConfigGroup(prefix = "hdds.datanode") +public class DatanodeConfiguration { + static final Logger LOG = + LoggerFactory.getLogger(DatanodeConfiguration.class); + + /** + * The maximum number of replication commands a single datanode can execute + * simultaneously. + */ + private int replicationMaxStreams = 10; + + @Config(key = "replication.streams.limit", + type = ConfigType.INT, + defaultValue = "10", + tags = {DATANODE}, + description = "The maximum number of replication commands a single " + + "datanode can execute simultaneously" + ) + public void setReplicationMaxStreams(int val) { + if (val < 1) { + LOG.warn("hdds.datanode.replication.streams.limit must be greater than" + + "zero and was set to {}. Defaulting to {}", + val, replicationMaxStreams); + } else { + this.replicationMaxStreams = val; + } + } + + public int getReplicationMaxStreams() { + return replicationMaxStreams; + } + +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index d1cdeb3bf0f4..2763278b0d2b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -35,7 +35,6 @@ .StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.ozone.HddsDatanodeStopService; -import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.container.common.report.ReportManager; import org.apache.hadoop.ozone.container.common.statemachine.commandhandler .CloseContainerCommandHandler; @@ -98,6 +97,10 @@ public class DatanodeStateMachine implements Closeable { public DatanodeStateMachine(DatanodeDetails datanodeDetails, Configuration conf, CertificateClient certClient, HddsDatanodeStopService hddsDatanodeStopService) throws IOException { + OzoneConfiguration ozoneConf = new OzoneConfiguration(conf); + DatanodeConfiguration dnConf = + ozoneConf.getObject(DatanodeConfiguration.class); + this.hddsDatanodeStopService = hddsDatanodeStopService; this.conf = conf; this.datanodeDetails = datanodeDetails; @@ -107,7 +110,7 @@ public DatanodeStateMachine(DatanodeDetails datanodeDetails, connectionManager = new SCMConnectionManager(conf); context = new StateContext(this.conf, DatanodeStates.getInitState(), this); container = new OzoneContainer(this.datanodeDetails, - new OzoneConfiguration(conf), context, certClient); + ozoneConf, context, certClient); dnCertClient = certClient; nextHB = new AtomicLong(Time.monotonicNow()); @@ -116,21 +119,9 @@ public DatanodeStateMachine(DatanodeDetails datanodeDetails, container.getController(), new SimpleContainerDownloader(conf), new TarContainerPacker()); - int replicationThreads = conf.getInt( - OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT, - OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT_DEFAULT); - if (replicationThreads < 1) { - LOG.warn("{} is set to {} and must be greater than 0. Defaulting to {}", - OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT, - replicationThreads, - OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT_DEFAULT); - replicationThreads = - OzoneConfigKeys.HDDS_DATANODE_REPLICATION_STREAMS_LIMIT_DEFAULT; - } - supervisor = new ReplicationSupervisor(container.getContainerSet(), replicator, - replicationThreads); + dnConf.getReplicationMaxStreams()); // When we add new handlers just adding a new handler here should do the // trick.