From fe7c70977d77c99ecbdef56cf5c6975b53ca2cc0 Mon Sep 17 00:00:00 2001 From: yuanyi Date: Mon, 28 Jun 2021 19:10:16 +0800 Subject: [PATCH 1/2] adapt DynamicConfigProvider to metadata --- .../metadata/MetadataStorageConnectorConfig.java | 15 +++++++++++---- .../storage/mysql/MySQLConnectorSslConfig.java | 16 ++++++++++++++-- .../updater/MetadataStorageUpdaterJobSpec.java | 12 ++++++++++-- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/druid/metadata/MetadataStorageConnectorConfig.java b/core/src/main/java/org/apache/druid/metadata/MetadataStorageConnectorConfig.java index c9850096ec6e..19ed1f782961 100644 --- a/core/src/main/java/org/apache/druid/metadata/MetadataStorageConnectorConfig.java +++ b/core/src/main/java/org/apache/druid/metadata/MetadataStorageConnectorConfig.java @@ -49,7 +49,9 @@ public class MetadataStorageConnectorConfig @JsonProperty("dbcp") private Properties dbcpProperties; - @JsonProperty + @JsonProperty("dynamicConfigProvider") + private DynamicConfigProvider dynamicConfigProvider; + public boolean isCreateTables() { return createTables; @@ -79,13 +81,17 @@ public String getConnectURI() @JsonProperty public String getUser() { - return user; + return (dynamicConfigProvider != null && dynamicConfigProvider.getConfig().get("user") != null) ? dynamicConfigProvider.getConfig().get("user").toString() : user; } @JsonProperty public String getPassword() { - return passwordProvider == null ? null : passwordProvider.getPassword(); + if (dynamicConfigProvider != null && dynamicConfigProvider.getConfig().get("passward") != null) { + return dynamicConfigProvider.getConfig().get("passward").toString(); + } else { + return passwordProvider == null ? null : passwordProvider.getPassword(); + } } @JsonProperty("dbcp") @@ -100,7 +106,7 @@ public String toString() return "DbConnectorConfig{" + "createTables=" + createTables + ", connectURI='" + getConnectURI() + '\'' + - ", user='" + user + '\'' + + ", user='" + getUser() + '\'' + ", passwordProvider=" + passwordProvider + ", dbcpProperties=" + dbcpProperties + '}'; @@ -153,6 +159,7 @@ public int hashCode() result = 31 * result + (getUser() != null ? getUser().hashCode() : 0); result = 31 * result + (passwordProvider != null ? passwordProvider.hashCode() : 0); result = 31 * result + (dbcpProperties != null ? dbcpProperties.hashCode() : 0); + result = 31 * result + (dynamicConfigProvider != null ? dynamicConfigProvider.hashCode() : 0); return result; } } diff --git a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorSslConfig.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorSslConfig.java index 1ead8b50f28b..e1d5a4c80ee6 100644 --- a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorSslConfig.java +++ b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorSslConfig.java @@ -20,6 +20,7 @@ package org.apache.druid.metadata.storage.mysql; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.metadata.DynamicConfigProvider; import org.apache.druid.metadata.PasswordProvider; import java.util.List; @@ -56,6 +57,9 @@ public class MySQLConnectorSslConfig @JsonProperty private boolean verifyServerCertificate = false; + @JsonProperty("dynamicConfigProvider") + private DynamicConfigProvider dynamicConfigProvider; + public boolean isUseSSL() { return useSSL; @@ -73,7 +77,11 @@ public String getTrustCertificateKeyStoreType() public String getTrustCertificateKeyStorePassword() { - return trustCertificateKeyStorePasswordProvider == null ? null : trustCertificateKeyStorePasswordProvider.getPassword(); + if (dynamicConfigProvider != null && dynamicConfigProvider.getConfig().get("trustCertificateKeyStorePassword") != null) { + return dynamicConfigProvider.getConfig().get("trustCertificateKeyStorePassword").toString(); + } else { + return trustCertificateKeyStorePasswordProvider == null ? null : trustCertificateKeyStorePasswordProvider.getPassword(); + } } public String getClientCertificateKeyStoreUrl() @@ -88,7 +96,11 @@ public String getClientCertificateKeyStoreType() public String getClientCertificateKeyStorePassword() { - return clientCertificateKeyStorePasswordProvider == null ? null : clientCertificateKeyStorePasswordProvider.getPassword(); + if (dynamicConfigProvider != null && dynamicConfigProvider.getConfig().get("clientCertificateKeyStorePassword") != null) { + return dynamicConfigProvider.getConfig().get("clientCertificateKeyStorePassword").toString(); + } else { + return clientCertificateKeyStorePasswordProvider == null ? null : clientCertificateKeyStorePasswordProvider.getPassword(); + } } public List getEnabledSSLCipherSuites() diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java index 0f8ca0acfccc..84d2e199a115 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Supplier; +import org.apache.druid.metadata.DynamicConfigProvider; import org.apache.druid.metadata.MetadataStorageConnectorConfig; import org.apache.druid.metadata.MetadataStorageTablesConfig; import org.apache.druid.metadata.PasswordProvider; @@ -57,6 +58,9 @@ public String getType() return type; } + @JsonProperty("dynamicConfigProvider") + private DynamicConfigProvider dynamicConfigProvider; + @Override public MetadataStorageConnectorConfig get() { @@ -71,13 +75,17 @@ public String getConnectURI() @Override public String getUser() { - return user; + return (dynamicConfigProvider != null && dynamicConfigProvider.getConfig().get("user") != null) ? dynamicConfigProvider.getConfig().get("user").toString() : user; } @Override public String getPassword() { - return passwordProvider == null ? null : passwordProvider.getPassword(); + if (dynamicConfigProvider != null && dynamicConfigProvider.getConfig().get("passward") != null) { + return dynamicConfigProvider.getConfig().get("passward").toString(); + } else { + return passwordProvider == null ? null : passwordProvider.getPassword(); + } } }; } From cb9705fd0c5022c277894d26e8e0989c7739d712 Mon Sep 17 00:00:00 2001 From: yuanyi Date: Thu, 5 Aug 2021 16:00:32 +0800 Subject: [PATCH 2/2] add some unit tests --- .../mysql/MySQLMetadataStorageModuleTest.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java index 4c395061e86b..4f74a86c6af5 100644 --- a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java +++ b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java @@ -75,6 +75,39 @@ public void testSslConfig() Assert.assertTrue(config.isVerifyServerCertificate()); } + @Test + public void testSslConfigWithDynamicConfigProvider() + { + final Injector injector = createInjector(); + final String propertyPrefix = "druid.metadata.mysql.ssl"; + final JsonConfigProvider provider = JsonConfigProvider.of( + propertyPrefix, + MySQLConnectorSslConfig.class + ); + final Properties properties = new Properties(); + properties.setProperty(propertyPrefix + ".useSSL", "true"); + properties.setProperty(propertyPrefix + ".trustCertificateKeyStoreUrl", "url"); + properties.setProperty(propertyPrefix + ".trustCertificateKeyStoreType", "type"); + properties.setProperty(propertyPrefix + ".clientCertificateKeyStoreUrl", "url"); + properties.setProperty(propertyPrefix + ".clientCertificateKeyStoreType", "type"); + properties.setProperty(propertyPrefix + ".dynamicConfigProvider", "{\"type\":\"mapString\",\"config\":{\"trustCertificateKeyStorePassword\":\"secret\",\"clientCertificateKeyStorePassword\":\"secret\"}}"); + properties.setProperty(propertyPrefix + ".enabledSSLCipherSuites", "[\"some\", \"ciphers\"]"); + properties.setProperty(propertyPrefix + ".enabledTLSProtocols", "[\"some\", \"protocols\"]"); + properties.setProperty(propertyPrefix + ".verifyServerCertificate", "true"); + provider.inject(properties, injector.getInstance(JsonConfigurator.class)); + final MySQLConnectorSslConfig config = provider.get().get(); + Assert.assertTrue(config.isUseSSL()); + Assert.assertEquals("url", config.getTrustCertificateKeyStoreUrl()); + Assert.assertEquals("type", config.getTrustCertificateKeyStoreType()); + Assert.assertEquals("secret", config.getTrustCertificateKeyStorePassword()); + Assert.assertEquals("url", config.getClientCertificateKeyStoreUrl()); + Assert.assertEquals("type", config.getClientCertificateKeyStoreType()); + Assert.assertEquals("secret", config.getClientCertificateKeyStorePassword()); + Assert.assertEquals(ImmutableList.of("some", "ciphers"), config.getEnabledSSLCipherSuites()); + Assert.assertEquals(ImmutableList.of("some", "protocols"), config.getEnabledTLSProtocols()); + Assert.assertTrue(config.isVerifyServerCertificate()); + } + @Test public void testDriverConfigDefault() {