From 58d7db2e4ad006046e728867ba004d5aa22d6f91 Mon Sep 17 00:00:00 2001 From: saketa Date: Sun, 12 Feb 2023 18:01:10 -0800 Subject: [PATCH 1/8] HDDS-7869. Log configuration on component startup. --- .../java/org/apache/hadoop/hdds/StringUtils.java | 9 +++++---- .../hadoop/hdds/conf/OzoneConfiguration.java | 15 +++++++++++++++ .../apache/hadoop/ozone/HddsDatanodeService.java | 8 ++++++-- .../server/StorageContainerManagerStarter.java | 4 +++- .../org/apache/hadoop/ozone/csi/CsiServer.java | 8 ++++++-- .../hadoop/ozone/om/OzoneManagerStarter.java | 4 +++- .../apache/hadoop/ozone/recon/ReconServer.java | 7 +++++-- .../java/org/apache/hadoop/ozone/s3/Gateway.java | 5 ++++- 8 files changed, 47 insertions(+), 13 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java index 792a9d0d8407..f891fa49b5a0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java @@ -105,12 +105,12 @@ public static String toStartupShutdownString(String prefix, String... msg) { } public static void startupShutdownMessage(VersionInfo versionInfo, - Class clazz, String[] args, Logger log) { + Class clazz, String[] args, Logger log, String conf) { final String hostname = NetUtils.getHostname(); final String className = clazz.getSimpleName(); if (log.isInfoEnabled()) { log.info(createStartupShutdownMessage(versionInfo, className, hostname, - args)); + args, conf)); } if (SystemUtils.IS_OS_UNIX) { @@ -135,7 +135,7 @@ public static void startupShutdownMessage(VersionInfo versionInfo, * @return a string to log. */ public static String createStartupShutdownMessage(VersionInfo versionInfo, - String className, String hostname, String[] args) { + String className, String hostname, String[] args, String conf) { return toStartupShutdownString("STARTUP_MSG: ", "Starting " + className, " host = " + hostname, @@ -146,7 +146,8 @@ public static String createStartupShutdownMessage(VersionInfo versionInfo, + versionInfo.getRevision() + " ; compiled by '" + versionInfo.getUser() + "' on " + versionInfo.getDate(), - " java = " + System.getProperty("java.version")); + " java = " + System.getProperty("java.version"), + " conf = " + conf); } public static String appendIfNotPresent(String str, char c) { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 91415c34ecce..a4f7e73bf694 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -275,6 +275,21 @@ public Properties getAllPropertiesByTag(String tag) { return props; } + public Properties getAllPropertiesByPrefix( + String keyPrefix) { + Properties updatedProps = getProps(); + Properties props = new Properties(); + for (String name : updatedProps.stringPropertyNames()) { + if (name.startsWith(keyPrefix)) { + String value = get(name); + if (value != null) { + props.put(name, value); + } + } + } + return props; + } + @Override public Collection getConfigKeys() { return getProps().keySet() diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index a9cf1427455f..bc16bae71fe5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore; import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStoreImpl; @@ -162,11 +163,14 @@ public static Logger getLogger() { @Override public Void call() throws Exception { + OzoneConfiguration configuration = createOzoneConfiguration(); if (printBanner) { StringUtils.startupShutdownMessage(HddsVersionInfo.HDDS_VERSION_INFO, - HddsDatanodeService.class, args, LOG); + HddsDatanodeService.class, args, LOG, + configuration.getAllPropertiesByTag( + ConfigTag.DATANODE.name()).toString()); } - start(createOzoneConfiguration()); + start(configuration); ShutdownHookManager.get().addShutdownHook(() -> { try { stop(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java index bf8c1f70e174..aabc10c372a4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.HddsVersionInfo; @@ -156,7 +157,8 @@ private void commonInit() { String[] originalArgs = getCmd().getParseResult().originalArgs() .toArray(new String[0]); StringUtils.startupShutdownMessage(HddsVersionInfo.HDDS_VERSION_INFO, - StorageContainerManager.class, originalArgs, LOG); + StorageContainerManager.class, originalArgs, LOG, + conf.getAllPropertiesByTag(ConfigTag.SCM.name()).toString()); } /** diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java index 015f9f1b7d0d..7278db99e5a0 100644 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java +++ b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java @@ -52,7 +52,13 @@ public class CsiServer extends GenericCli implements Callable { @Override public Void call() throws Exception { + String[] originalArgs = getCmd().getParseResult().originalArgs() + .toArray(new String[0]); OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); + StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, + CsiServer.class, originalArgs, LOG, + ozoneConfiguration.getAllPropertiesByPrefix( + "ozone.csi").toString()); CsiConfig csiConfig = ozoneConfiguration.getObject(CsiConfig.class); OzoneClient rpcClient = OzoneClientFactory.getRpcClient(ozoneConfiguration); @@ -85,8 +91,6 @@ public Void call() throws Exception { } public static void main(String[] args) { - StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, - CsiServer.class, args, LOG); new CsiServer().run(args); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java index af321199bdf2..a89e9be03370 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.ozone.util.OzoneNetUtils; @@ -173,7 +174,8 @@ private void commonInit() { String[] originalArgs = getCmd().getParseResult().originalArgs() .toArray(new String[0]); StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, - OzoneManager.class, originalArgs, LOG); + OzoneManager.class, originalArgs, LOG, + conf.getAllPropertiesByTag(ConfigTag.OM.name()).toString()); } /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index 124e48981d60..9962553e7229 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.recon.ReconConfig; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; @@ -91,10 +92,12 @@ public static void main(String[] args) { public Void call() throws Exception { String[] originalArgs = getCmd().getParseResult().originalArgs() .toArray(new String[0]); - StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, - ReconServer.class, originalArgs, LOG); configuration = createOzoneConfiguration(); + StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, + ReconServer.class, originalArgs, LOG, + configuration.getAllPropertiesByTag( + ConfigTag.RECON.name()).toString()); ConfigurationProvider.setConfiguration(configuration); injector = Guice.createInjector(new ReconControllerModule(), diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java index b8153f0209d2..0535e2101b72 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.HddsServerUtil; @@ -88,7 +89,9 @@ public void start() throws IOException { String[] originalArgs = getCmd().getParseResult().originalArgs() .toArray(new String[0]); StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, - Gateway.class, originalArgs, LOG); + Gateway.class, originalArgs, LOG, + ozoneConfiguration.getAllPropertiesByTag( + ConfigTag.S3GATEWAY.name()).toString()); LOG.info("Starting Ozone S3 gateway"); HddsServerUtil.initializeMetrics(ozoneConfiguration, "S3Gateway"); From 286734e1adeac5f62b518e6a288377b11f31e927 Mon Sep 17 00:00:00 2001 From: saketa Date: Thu, 16 Feb 2023 15:08:01 -0800 Subject: [PATCH 2/8] HDDS-7869. Printing all configs. Added utility function to process and sort conf object for logging. --- .../org/apache/hadoop/hdds/HddsUtils.java | 22 +++++++++++++++ .../org/apache/hadoop/hdds/StringUtils.java | 11 +++++--- .../hadoop/hdds/conf/OzoneConfiguration.java | 15 ++--------- .../org/apache/hadoop/hdds/TestHddsUtils.java | 27 +++++++++++++++++++ .../hadoop/ozone/HddsDatanodeService.java | 4 +-- .../StorageContainerManagerStarter.java | 3 +-- .../apache/hadoop/ozone/csi/CsiServer.java | 4 +-- .../hadoop/ozone/om/OzoneManagerStarter.java | 3 +-- .../hadoop/ozone/recon/ReconServer.java | 4 +-- .../org/apache/hadoop/ozone/s3/Gateway.java | 4 +-- 10 files changed, 65 insertions(+), 32 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 382451b1296c..4c9e973f164d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -34,13 +34,17 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; +import java.util.Properties; +import java.util.TreeMap; +import org.apache.hadoop.conf.ConfigRedactor; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProtoOrBuilder; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State; @@ -776,4 +780,22 @@ public static ContainerProtos.ContainerCommandResponseProto processForDebug( return msg; } + + /** + * Redacts sensitive configuration and + * sorts all config properties by key name + * + * @param conf OzoneConfiguration object to be printed. + * @return Sorted Map of properties + */ + public static Map processForLogging(OzoneConfiguration conf) { + Properties props = conf.getAllProperties(); + ConfigRedactor redactor = new ConfigRedactor(conf); + Map configMap = new TreeMap<>(); + for (String name : props.stringPropertyNames()) { + String value = redactor.redact(name, props.getProperty(name)); + configMap.put(name, value); + } + return configMap; + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java index f891fa49b5a0..fd9839640677 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java @@ -21,9 +21,12 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Map; import com.google.common.base.Preconditions; import org.apache.commons.lang3.SystemUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.utils.SignalLogger; import org.apache.hadoop.hdds.utils.VersionInfo; import org.apache.hadoop.net.NetUtils; @@ -105,12 +108,13 @@ public static String toStartupShutdownString(String prefix, String... msg) { } public static void startupShutdownMessage(VersionInfo versionInfo, - Class clazz, String[] args, Logger log, String conf) { + Class clazz, String[] args, Logger log, OzoneConfiguration conf) { final String hostname = NetUtils.getHostname(); final String className = clazz.getSimpleName(); + if (log.isInfoEnabled()) { log.info(createStartupShutdownMessage(versionInfo, className, hostname, - args, conf)); + args, HddsUtils.processForLogging(conf))); } if (SystemUtils.IS_OS_UNIX) { @@ -135,7 +139,8 @@ public static void startupShutdownMessage(VersionInfo versionInfo, * @return a string to log. */ public static String createStartupShutdownMessage(VersionInfo versionInfo, - String className, String hostname, String[] args, String conf) { + String className, String hostname, String[] args, + Map conf) { return toStartupShutdownString("STARTUP_MSG: ", "Starting " + className, " host = " + hostname, diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index a4f7e73bf694..5035a652d24b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -275,19 +275,8 @@ public Properties getAllPropertiesByTag(String tag) { return props; } - public Properties getAllPropertiesByPrefix( - String keyPrefix) { - Properties updatedProps = getProps(); - Properties props = new Properties(); - for (String name : updatedProps.stringPropertyNames()) { - if (name.startsWith(keyPrefix)) { - String value = get(name); - if (value != null) { - props.put(name, value); - } - } - } - return props; + public Properties getAllProperties() { + return getProps(); } @Override diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java index 3dbee3efbdc3..6a6f25952405 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java @@ -31,8 +31,10 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.ozone.test.LambdaTestUtils; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import static org.apache.hadoop.hdds.HddsUtils.getSCMAddressForDatanodes; +import static org.apache.hadoop.hdds.HddsUtils.processForLogging; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; @@ -50,6 +52,11 @@ */ public class TestHddsUtils { + private static final String REDACTED_TEXT = ""; + private static final String ORIGINAL_VALUE = "Hello, World!"; + private static final String SENSITIVE_CONFIG_KEYS = + CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS; + @Test public void testGetHostName() { assertEquals(Optional.of("localhost"), @@ -248,4 +255,24 @@ public void testGetNumberFromConfigKeys() { serviceId, nodeId), OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT).orElse(0)); } + + @Test + public void testRedactSensitivePropsForLogging() { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(SENSITIVE_CONFIG_KEYS, String.join("\n", + "password$", + "key$")); + /* Sensitive properties */ + conf.set("test.password", ORIGINAL_VALUE); + conf.set("fs.s3a.secret.key", ORIGINAL_VALUE); + /* Non-Sensitive properties */ + conf.set("normal.config", ORIGINAL_VALUE); + Map processed_conf = processForLogging(conf); + + /* Verify that sensitive properties are redacted */ + assertEquals(processed_conf.get("test.password"),REDACTED_TEXT); + assertEquals(processed_conf.get("fs.s3a.secret.key"),REDACTED_TEXT); + /* Verify that non-sensitive properties retain their value */ + assertEquals(processed_conf.get("normal.config"),ORIGINAL_VALUE); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index bc16bae71fe5..c0c84fc63a10 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -166,9 +166,7 @@ public Void call() throws Exception { OzoneConfiguration configuration = createOzoneConfiguration(); if (printBanner) { StringUtils.startupShutdownMessage(HddsVersionInfo.HDDS_VERSION_INFO, - HddsDatanodeService.class, args, LOG, - configuration.getAllPropertiesByTag( - ConfigTag.DATANODE.name()).toString()); + HddsDatanodeService.class, args, LOG, configuration); } start(configuration); ShutdownHookManager.get().addShutdownHook(() -> { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java index aabc10c372a4..19073d8125b7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java @@ -157,8 +157,7 @@ private void commonInit() { String[] originalArgs = getCmd().getParseResult().originalArgs() .toArray(new String[0]); StringUtils.startupShutdownMessage(HddsVersionInfo.HDDS_VERSION_INFO, - StorageContainerManager.class, originalArgs, LOG, - conf.getAllPropertiesByTag(ConfigTag.SCM.name()).toString()); + StorageContainerManager.class, originalArgs, LOG, conf); } /** diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java index 7278db99e5a0..3b0c8f3c20fd 100644 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java +++ b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java @@ -56,9 +56,7 @@ public Void call() throws Exception { .toArray(new String[0]); OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, - CsiServer.class, originalArgs, LOG, - ozoneConfiguration.getAllPropertiesByPrefix( - "ozone.csi").toString()); + CsiServer.class, originalArgs, LOG, ozoneConfiguration); CsiConfig csiConfig = ozoneConfiguration.getObject(CsiConfig.class); OzoneClient rpcClient = OzoneClientFactory.getRpcClient(ozoneConfiguration); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java index a89e9be03370..53a24f4bae3a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java @@ -174,8 +174,7 @@ private void commonInit() { String[] originalArgs = getCmd().getParseResult().originalArgs() .toArray(new String[0]); StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, - OzoneManager.class, originalArgs, LOG, - conf.getAllPropertiesByTag(ConfigTag.OM.name()).toString()); + OzoneManager.class, originalArgs, LOG, conf); } /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index 9962553e7229..8bcfde471bc7 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -95,9 +95,7 @@ public Void call() throws Exception { configuration = createOzoneConfiguration(); StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, - ReconServer.class, originalArgs, LOG, - configuration.getAllPropertiesByTag( - ConfigTag.RECON.name()).toString()); + ReconServer.class, originalArgs, LOG, configuration); ConfigurationProvider.setConfiguration(configuration); injector = Guice.createInjector(new ReconControllerModule(), diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java index 0535e2101b72..0c717abfa865 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java @@ -89,9 +89,7 @@ public void start() throws IOException { String[] originalArgs = getCmd().getParseResult().originalArgs() .toArray(new String[0]); StringUtils.startupShutdownMessage(OzoneVersionInfo.OZONE_VERSION_INFO, - Gateway.class, originalArgs, LOG, - ozoneConfiguration.getAllPropertiesByTag( - ConfigTag.S3GATEWAY.name()).toString()); + Gateway.class, originalArgs, LOG, ozoneConfiguration); LOG.info("Starting Ozone S3 gateway"); HddsServerUtil.initializeMetrics(ozoneConfiguration, "S3Gateway"); From d362e5d566b08317a50e36e3e26710d9c69aaacb Mon Sep 17 00:00:00 2001 From: saketa Date: Thu, 16 Feb 2023 15:30:42 -0800 Subject: [PATCH 3/8] HDDS-7869. Fixed checkstyle. Removed unused import. --- .../main/java/org/apache/hadoop/ozone/HddsDatanodeService.java | 1 - .../hadoop/hdds/scm/server/StorageContainerManagerStarter.java | 1 - .../java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java | 1 - .../src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java | 1 - .../src/main/java/org/apache/hadoop/ozone/s3/Gateway.java | 1 - 5 files changed, 5 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index c0c84fc63a10..49784eabc8f0 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore; import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStoreImpl; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java index 19073d8125b7..5c4e15a843cd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.HddsVersionInfo; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java index 53a24f4bae3a..42d657aecc98 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.ozone.util.OzoneNetUtils; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index 8bcfde471bc7..f571618b98c6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; -import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.recon.ReconConfig; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java index 0c717abfa865..2a96f050013c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java @@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.HddsServerUtil; From ab57e03ff160ba4a8f86b691576a99ae8a94d2bd Mon Sep 17 00:00:00 2001 From: saketa Date: Thu, 16 Feb 2023 15:40:10 -0800 Subject: [PATCH 4/8] HDDS-7869. Fixed variable name and whitespace checkstyle errors. --- .../test/java/org/apache/hadoop/hdds/TestHddsUtils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java index 6a6f25952405..ef5b6b57726c 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java @@ -267,12 +267,12 @@ public void testRedactSensitivePropsForLogging() { conf.set("fs.s3a.secret.key", ORIGINAL_VALUE); /* Non-Sensitive properties */ conf.set("normal.config", ORIGINAL_VALUE); - Map processed_conf = processForLogging(conf); + Map processedConf = processForLogging(conf); /* Verify that sensitive properties are redacted */ - assertEquals(processed_conf.get("test.password"),REDACTED_TEXT); - assertEquals(processed_conf.get("fs.s3a.secret.key"),REDACTED_TEXT); + assertEquals(processedConf.get("test.password"), REDACTED_TEXT); + assertEquals(processedConf.get("fs.s3a.secret.key"), REDACTED_TEXT); /* Verify that non-sensitive properties retain their value */ - assertEquals(processed_conf.get("normal.config"),ORIGINAL_VALUE); + assertEquals(processedConf.get("normal.config"), ORIGINAL_VALUE); } } From 96167b6ead63256db475184e59b4bf83e1ec2947 Mon Sep 17 00:00:00 2001 From: saketa Date: Thu, 16 Feb 2023 18:08:30 -0800 Subject: [PATCH 5/8] HDDS-7869. Fixed checkstyle errors. Removed redundant import and fixed whitespace errors. --- .../src/main/java/org/apache/hadoop/hdds/HddsUtils.java | 4 ++-- .../src/main/java/org/apache/hadoop/hdds/StringUtils.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 4c9e973f164d..a25acf3079b5 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -782,8 +782,8 @@ public static ContainerProtos.ContainerCommandResponseProto processForDebug( } /** - * Redacts sensitive configuration and - * sorts all config properties by key name + * Redacts sensitive configuration. + * Sorts all properties by key name * * @param conf OzoneConfiguration object to be printed. * @return Sorted Map of properties diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java index fd9839640677..ed8f410d0e65 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java @@ -26,7 +26,6 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang3.SystemUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.utils.SignalLogger; import org.apache.hadoop.hdds.utils.VersionInfo; import org.apache.hadoop.net.NetUtils; From e2b0e6cb9c1b0243aca21c991d692e58d6123b99 Mon Sep 17 00:00:00 2001 From: saketa Date: Mon, 27 Feb 2023 17:57:15 -0800 Subject: [PATCH 6/8] HDDS-7869. Filtered Ozone configs only. --- .../main/java/org/apache/hadoop/hdds/HddsUtils.java | 13 ++++++------- .../apache/hadoop/hdds/conf/OzoneConfiguration.java | 5 +++-- .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index a25acf3079b5..58abf847b1dd 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -34,7 +34,6 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; -import java.util.Properties; import java.util.TreeMap; import org.apache.hadoop.conf.ConfigRedactor; @@ -789,13 +788,13 @@ public static ContainerProtos.ContainerCommandResponseProto processForDebug( * @return Sorted Map of properties */ public static Map processForLogging(OzoneConfiguration conf) { - Properties props = conf.getAllProperties(); + Map ozoneProps = conf.getOzoneProperties(); ConfigRedactor redactor = new ConfigRedactor(conf); - Map configMap = new TreeMap<>(); - for (String name : props.stringPropertyNames()) { - String value = redactor.redact(name, props.getProperty(name)); - configMap.put(name, value); + Map sortedOzoneProps = new TreeMap<>(); + for (String name : ozoneProps.keySet()) { + String value = redactor.redact(name, ozoneProps.get(name)); + sortedOzoneProps.put(name, value); } - return configMap; + return sortedOzoneProps; } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 5035a652d24b..450da59168c7 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -275,8 +275,9 @@ public Properties getAllPropertiesByTag(String tag) { return props; } - public Properties getAllProperties() { - return getProps(); + public Map getOzoneProperties() { + String ozoneRegex = ".*(ozone|hdds|ratis|container|scm|recon)\\..*"; + return getValByRegex(ozoneRegex); } @Override diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java index ef5b6b57726c..dc192d358085 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java @@ -263,16 +263,16 @@ public void testRedactSensitivePropsForLogging() { "password$", "key$")); /* Sensitive properties */ - conf.set("test.password", ORIGINAL_VALUE); - conf.set("fs.s3a.secret.key", ORIGINAL_VALUE); + conf.set("ozone.test.password", ORIGINAL_VALUE); + conf.set("hdds.test.secret.key", ORIGINAL_VALUE); /* Non-Sensitive properties */ - conf.set("normal.config", ORIGINAL_VALUE); + conf.set("ozone.normal.config", ORIGINAL_VALUE); Map processedConf = processForLogging(conf); /* Verify that sensitive properties are redacted */ - assertEquals(processedConf.get("test.password"), REDACTED_TEXT); - assertEquals(processedConf.get("fs.s3a.secret.key"), REDACTED_TEXT); + assertEquals(processedConf.get("ozone.test.password"), REDACTED_TEXT); + assertEquals(processedConf.get("hdds.test.secret.key"), REDACTED_TEXT); /* Verify that non-sensitive properties retain their value */ - assertEquals(processedConf.get("normal.config"), ORIGINAL_VALUE); + assertEquals(processedConf.get("ozone.normal.config"), ORIGINAL_VALUE); } } From 316c8126fb3b3c4cc402e6ec6400e826ddbd4d00 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Tue, 28 Feb 2023 07:35:49 +0100 Subject: [PATCH 7/8] Fix findbugs --- .../src/main/java/org/apache/hadoop/hdds/HddsUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 58abf847b1dd..677901ce08be 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -791,9 +791,9 @@ public static Map processForLogging(OzoneConfiguration conf) { Map ozoneProps = conf.getOzoneProperties(); ConfigRedactor redactor = new ConfigRedactor(conf); Map sortedOzoneProps = new TreeMap<>(); - for (String name : ozoneProps.keySet()) { - String value = redactor.redact(name, ozoneProps.get(name)); - sortedOzoneProps.put(name, value); + for (Map.Entry entry : ozoneProps.entrySet()) { + String value = redactor.redact(entry.getKey(), entry.getValue()); + sortedOzoneProps.put(entry.getKey(), value); } return sortedOzoneProps; } From f5accde32bfdee53e4c5e65d229a90fcfc92a543 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Tue, 28 Feb 2023 19:03:17 +0100 Subject: [PATCH 8/8] Trim config values --- .../common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 677901ce08be..344dd3e6ffcc 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -793,6 +793,9 @@ public static Map processForLogging(OzoneConfiguration conf) { Map sortedOzoneProps = new TreeMap<>(); for (Map.Entry entry : ozoneProps.entrySet()) { String value = redactor.redact(entry.getKey(), entry.getValue()); + if (value != null) { + value = value.trim(); + } sortedOzoneProps.put(entry.getKey(), value); } return sortedOzoneProps;