From 8e5bfc24e6e3abfc9f7d7b3b18998f014b9b0fdd Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Sat, 5 Sep 2020 11:31:36 -0700 Subject: [PATCH 1/2] HDDS-3927. Rename HDFS_OM_OPTS/HDDS_DN_OPTS to OZONE_MANAGER_OPTS/OZONE_DATANODE_OPTS Change-Id: I629f87486b62b6a3b7f108addf89ce1ec2bc9e18 --- hadoop-hdds/common/src/main/conf/hadoop-env.sh | 11 ++++++++++- .../src/main/compose/ozone-topology/docker-config | 4 ++-- hadoop-ozone/dist/src/shell/ozone/ozone | 10 ++++++---- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/common/src/main/conf/hadoop-env.sh b/hadoop-hdds/common/src/main/conf/hadoop-env.sh index 51ee585ba5f5..bbe6c825ee55 100644 --- a/hadoop-hdds/common/src/main/conf/hadoop-env.sh +++ b/hadoop-hdds/common/src/main/conf/hadoop-env.sh @@ -400,7 +400,16 @@ export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export HDFS_OM_OPTS="" +# export OZONE_MANAGER_OPTS="" + +### +# Ozone DataNode specific parameters +### +# Specify the JVM options to be used when starting Ozone DataNodes. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export OZONE_DATANODE_OPTS="" ### # HDFS StorageContainerManager specific parameters diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config index 9cf5822cc73d..119e7afca631 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config @@ -36,8 +36,8 @@ OZONE-SITE.XML_ozone.network.topology.aware.read=true HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 ASYNC_PROFILER_HOME=/opt/profiler -HDDS_DN_OPTS=-Dmodule.name=datanode -HDFS_OM_OPTS=-Dmodule.name=om +OZONE_DATANODE_OPTS=-Dmodule.name=datanode +OZONE_MANAGER_OPTS=-Dmodule.name=om HDFS_STORAGECONTAINERMANAGER_OPTS=-Dmodule.name=scm HDFS_OM_SH_OPTS=-Dmodule.name=sh HDFS_SCM_CLI_OPTS=-Dmodule.name=admin diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index c536484e9b56..e0c0309ac4bb 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -105,8 +105,9 @@ function ozonecmd_case # Corresponding Ratis issue https://issues.apache.org/jira/browse/RATIS-534. # TODO: Fix the problem related to netty resource leak detector throwing # exception as mentioned in HDDS-3812 - HDDS_DN_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false -Dorg.apache.ratis.thirdparty.io.netty.leakDetection.level=disabled -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/dn-audit-log4j2.properties ${HDDS_DN_OPTS}" - HADOOP_OPTS="${HADOOP_OPTS} ${HDDS_DN_OPTS}" + hadoop_deprecate_envvar HDDS_DN_OPTS OZONE_DATANODE_OPTS + OZONE_DATANODE_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false -Dorg.apache.ratis.thirdparty.io.netty.leakDetection.level=disabled -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/dn-audit-log4j2.properties ${OZONE_DATANODE_OPTS}" + HADOOP_OPTS="${HADOOP_OPTS} ${OZONE_DATANODE_OPTS}" HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-datanode" ;; @@ -154,8 +155,9 @@ function ozonecmd_case om) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManagerStarter - HDFS_OM_OPTS="${HDFS_OM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties" - HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_OM_OPTS}" + hadoop_deprecate_envvar HDFS_OM_OPTS OZONE_MANAGER_OPTS + OZONE_MANAGER_OPTS="${OZONE_MANAGER_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties" + HADOOP_OPTS="${HADOOP_OPTS} ${OZONE_MANAGER_OPTS}" OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager" ;; sh | shell) From 350b98b5f8c4400e19cd481e8681da101b4e7674 Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Mon, 7 Sep 2020 16:35:37 -0700 Subject: [PATCH 2/2] Conform to hadoop_subcommand_opts's OPTS naming conventions. Change-Id: I67259db97eabf04a55b9b5bbc582a6e5732cdb58 --- hadoop-hdds/common/src/main/conf/hadoop-env.sh | 4 ++-- hadoop-ozone/dist/src/shell/ozone/ozone | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/common/src/main/conf/hadoop-env.sh b/hadoop-hdds/common/src/main/conf/hadoop-env.sh index bbe6c825ee55..07f2ed862639 100644 --- a/hadoop-hdds/common/src/main/conf/hadoop-env.sh +++ b/hadoop-hdds/common/src/main/conf/hadoop-env.sh @@ -400,7 +400,7 @@ export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export OZONE_MANAGER_OPTS="" +# export OZONE_OM_OPTS="" ### # Ozone DataNode specific parameters @@ -418,7 +418,7 @@ export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export HDFS_STORAGECONTAINERMANAGER_OPTS="" +# export OZONE_SCM_OPTS="" ### # Advanced Users Only! diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone b/hadoop-ozone/dist/src/shell/ozone/ozone index e0c0309ac4bb..13a416ae71fc 100755 --- a/hadoop-ozone/dist/src/shell/ozone/ozone +++ b/hadoop-ozone/dist/src/shell/ozone/ozone @@ -107,7 +107,6 @@ function ozonecmd_case # exception as mentioned in HDDS-3812 hadoop_deprecate_envvar HDDS_DN_OPTS OZONE_DATANODE_OPTS OZONE_DATANODE_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false -Dorg.apache.ratis.thirdparty.io.netty.leakDetection.level=disabled -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/dn-audit-log4j2.properties ${OZONE_DATANODE_OPTS}" - HADOOP_OPTS="${HADOOP_OPTS} ${OZONE_DATANODE_OPTS}" HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-datanode" ;; @@ -155,9 +154,8 @@ function ozonecmd_case om) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManagerStarter - hadoop_deprecate_envvar HDFS_OM_OPTS OZONE_MANAGER_OPTS - OZONE_MANAGER_OPTS="${OZONE_MANAGER_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties" - HADOOP_OPTS="${HADOOP_OPTS} ${OZONE_MANAGER_OPTS}" + hadoop_deprecate_envvar HDFS_OM_OPTS OZONE_OM_OPTS + OZONE_OM_OPTS="${OZONE_OM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties" OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager" ;; sh | shell) @@ -174,9 +172,8 @@ function ozonecmd_case scm) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManagerStarter' - hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS" - HDFS_STORAGECONTAINERMANAGER_OPTS="${HDFS_STORAGECONTAINERMANAGER_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/scm-audit-log4j2.properties" - HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}" + hadoop_deprecate_envvar HDFS_STORAGECONTAINERMANAGER_OPTS OZONE_SCM_OPTS + OZONE_SCM_OPTS="${OZONE_SCM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/scm-audit-log4j2.properties" OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-server-scm" ;; s3g)