From c024ec6a79c99a3e364f94965b273d088e97ddc3 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Fri, 28 Feb 2020 17:04:34 -0800 Subject: [PATCH 01/25] refresh db pwd --- .../druid/metadata/BasicDataSourceExt.java | 171 ++++++++++++++++++ .../druid/metadata/SQLMetadataConnector.java | 2 +- 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java diff --git a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java new file mode 100644 index 000000000000..642938c05275 --- /dev/null +++ b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.metadata; + +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.dbcp2.ConnectionFactory; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.logger.Logger; + +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +/** + * This class exists so that PasswordProvider is asked for password every time a brand new connection is established + * with DB. PasswordProvider impls such as based on AWS tokens refresh the underlying token periodically since + * each token is valid for a certain period of time only. + * So, This class overrides[ummm copies] the methods from base class in order to keep track of connection properties + * and call MetadataStorageConnectorConfig.getPassword() everytime a new connection is setup. + */ +public class BasicDataSourceExt extends BasicDataSource +{ + private static final Logger LOGGER = new Logger(BasicDataSourceExt.class); + + private Properties connectionProperties; + private final MetadataStorageConnectorConfig connectorConfig; + + public BasicDataSourceExt(MetadataStorageConnectorConfig connectorConfig) + { + this.connectorConfig = connectorConfig; + this.connectionProperties = new Properties(); + } + + @Override + public void addConnectionProperty(String name, String value) + { + connectionProperties.put(name, value); + super.addConnectionProperty(name, value); + } + + @Override + public void removeConnectionProperty(String name) + { + connectionProperties.remove(name); + super.removeConnectionProperty(name); + } + + @Override + public void setConnectionProperties(String connectionProperties) + { + if (connectionProperties == null) { + throw new NullPointerException("connectionProperties is null"); + } + + String[] entries = connectionProperties.split(";"); + Properties properties = new Properties(); + for (String entry : entries) { + if (entry.length() > 0) { + int index = entry.indexOf('='); + if (index > 0) { + String name = entry.substring(0, index); + String value = entry.substring(index + 1); + properties.setProperty(name, value); + } else { + // no value is empty string which is how java.util.Properties works + properties.setProperty(entry, ""); + } + } + } + this.connectionProperties = properties; + super.setConnectionProperties(connectionProperties); + } + + @Override + protected ConnectionFactory createConnectionFactory() throws SQLException + { + Driver driverToUse = getDriver(); + + if (driverToUse == null) { + Class driverFromCCL = null; + if (getDriverClassName() != null) { + try { + try { + if (getDriverClassLoader() == null) { + driverFromCCL = Class.forName(getDriverClassName()); + } else { + driverFromCCL = Class.forName( + getDriverClassName(), true, getDriverClassLoader()); + } + } + catch (ClassNotFoundException cnfe) { + driverFromCCL = Thread.currentThread( + ).getContextClassLoader().loadClass( + getDriverClassName()); + } + } + catch (Exception t) { + String message = "Cannot load JDBC driver class '" + + getDriverClassName() + "'"; + LOGGER.error(t, message); + throw new SQLException(message, t); + } + } + + try { + if (driverFromCCL == null) { + driverToUse = DriverManager.getDriver(getUrl()); + } else { + // Usage of DriverManager is not possible, as it does not + // respect the ContextClassLoader + // N.B. This cast may cause ClassCastException which is handled below + driverToUse = (Driver) driverFromCCL.newInstance(); + if (!driverToUse.acceptsURL(getUrl())) { + throw new SQLException("No suitable driver", "08001"); + } + } + } + catch (Exception t) { + String message = "Cannot create JDBC driver of class '" + + (getDriverClassName() != null ? getDriverClassName() : "") + + "' for connect URL '" + getUrl() + "'"; + LOGGER.error(t, message); + throw new SQLException(message, t); + } + } + + if (driverToUse == null) { + throw new RE("WTH! Couln't find a Driver"); + } + + final Driver finalDriverToUse = driverToUse; + + return () -> { + String user = connectorConfig.getUser(); + if (user != null) { + connectionProperties.put("user", user); + } else { + log("DBCP DataSource configured without a 'username'"); + } + + // Note: This is the main point of this class where we are getting fresh password before setting up + // every new connection. + String password = connectorConfig.getPassword(); + if (password != null) { + connectionProperties.put("password", password); + } else { + log("DBCP DataSource configured without a 'password'"); + } + + return finalDriverToUse.connect(connectorConfig.getConnectURI(), connectionProperties); + }; + } +} diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java index e3cfe299b2b1..82497136a017 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java +++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java @@ -661,7 +661,7 @@ protected BasicDataSource getDatasource() if (dbcpProperties != null) { dataSource = BasicDataSourceFactory.createDataSource(dbcpProperties); } else { - dataSource = new BasicDataSource(); + dataSource = new BasicDataSourceExt(connectorConfig); } } catch (Exception e) { From 5cef48e27a7ae8f558b35adf2621440fe1faadda Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 2 Mar 2020 17:38:37 -0800 Subject: [PATCH 02/25] aws iam token password provider --- cloud/aws-common/pom.xml | 4 + .../AWSMetadataStorageConnectorConfig.java | 49 ++++++++++ .../apache/druid/common/aws/AWSModule.java | 10 +- .../aws/AWSRDSTokenPasswordProvider.java | 92 +++++++++++++++++++ docs/operations/password-provider.md | 14 +++ pom.xml | 5 + 6 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSMetadataStorageConnectorConfig.java create mode 100644 cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java diff --git a/cloud/aws-common/pom.xml b/cloud/aws-common/pom.xml index 2cd860fae8fc..99123f15e865 100644 --- a/cloud/aws-common/pom.xml +++ b/cloud/aws-common/pom.xml @@ -38,6 +38,10 @@ druid-core ${project.parent.version} + + com.amazonaws + aws-java-sdk + com.amazonaws aws-java-sdk-ec2 diff --git a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSMetadataStorageConnectorConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSMetadataStorageConnectorConfig.java new file mode 100644 index 000000000000..d76ec1f09530 --- /dev/null +++ b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSMetadataStorageConnectorConfig.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.common.aws; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class AWSMetadataStorageConnectorConfig +{ + @JsonProperty + private String user = null; + + @JsonProperty + private String host = null; + + @JsonProperty + private int port = 0; + + public String getUser() + { + return user; + } + + public String getHost() + { + return host; + } + + public int getPort() + { + return port; + } +} diff --git a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java index 79b60c956c34..86674bc7d276 100644 --- a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java +++ b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java @@ -23,6 +23,8 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.inject.Binder; import com.google.inject.Provides; import org.apache.druid.guice.JsonConfigProvider; @@ -41,6 +43,8 @@ public void configure(Binder binder) JsonConfigProvider.bind(binder, "druid.s3", AWSClientConfig.class); JsonConfigProvider.bind(binder, "druid.s3.proxy", AWSProxyConfig.class); JsonConfigProvider.bind(binder, "druid.s3.endpoint", AWSEndpointConfig.class); + + JsonConfigProvider.bind(binder, "druid.metadata.storage.connector", AWSMetadataStorageConnectorConfig.class); } @Provides @@ -60,6 +64,10 @@ public AmazonEC2 getEc2Client(AWSCredentialsProvider credentials) @Override public List getJacksonModules() { - return Collections.emptyList(); + return Collections.singletonList( + new SimpleModule("AWSModule").registerSubtypes( + new NamedType(AWSRDSTokenPasswordProvider.class, "awsrdstoken") + ) + ); } } diff --git a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java new file mode 100644 index 000000000000..05bd6e55e040 --- /dev/null +++ b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.common.aws; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.rds.auth.GetIamAuthTokenRequest; +import com.amazonaws.services.rds.auth.RdsIamAuthTokenGenerator; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.metadata.PasswordProvider; + +/** + * Generates the AWS token same as aws cli + * aws rds generate-db-auth-token --hostname HOST --port PORT --region REGION --username USER + * and returns that as password. + * + * Before using this, please make sure that you have connected all dots for db user to connect using token. + * See https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html + */ +public class AWSRDSTokenPasswordProvider implements PasswordProvider +{ + private static final Logger LOGGER = new Logger(AWSRDSTokenPasswordProvider.class); + private final String userName; + private final String hostName; + private final int port; + private final String region; + + private final AWSCredentialsProvider awsCredentialsProvider; + + @JsonCreator + public AWSRDSTokenPasswordProvider( + @JsonProperty("region") String region, + @JacksonInject AWSMetadataStorageConnectorConfig metadataStorageConnectorConfig, + @JacksonInject AWSCredentialsProvider awsCredentialsProvider + ) + { + userName = Preconditions.checkNotNull(metadataStorageConnectorConfig.getUser(), "null metadataStorage user"); + hostName = Preconditions.checkNotNull(metadataStorageConnectorConfig.getHost(), "null metadataStorage host"); + Preconditions.checkArgument(metadataStorageConnectorConfig.getPort() > 0, "must provide port"); + port = metadataStorageConnectorConfig.getPort(); + + this.region = Preconditions.checkNotNull(region, "null region"); + + LOGGER.info("AWS RDS Config user[%s], host[%s], port[%d], region[%s]", userName, hostName, port, this.region); + this.awsCredentialsProvider = awsCredentialsProvider; + } + + @Override + public String getPassword() + { + try { + RdsIamAuthTokenGenerator generator = RdsIamAuthTokenGenerator.builder() + .credentials(awsCredentialsProvider) + .region(region) + .build(); + + String authToken = generator.getAuthToken( + GetIamAuthTokenRequest.builder() + .hostname(hostName) + .port(port) + .userName(userName) + .build()); + + return authToken; + } + catch (Exception ex) { + LOGGER.error(ex, "Couldn't generate AWS IAM token."); + throw new RE(ex, "Couldn't generate AWS IAM token."); + } + } +} diff --git a/docs/operations/password-provider.md b/docs/operations/password-provider.md index 4a28e64aae8b..acf375ff4de4 100644 --- a/docs/operations/password-provider.md +++ b/docs/operations/password-provider.md @@ -43,6 +43,20 @@ The values are described below. |`type`|String|password provider type|Yes: `environment`| |`variable`|String|environment variable to read password from|Yes| +AWS RDS token password provider provides temp token for accesing AWS RDS DB instances. + +```json +{ "type": "awsrdstoken", "region": "AWS_REGION" } +``` + +Before using this password provider, please make sure that you have connected all dots for db user to connect using token. +See https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html + +Following additional metadata storage configuration must be provided to use `awsrdstokene` password provider. +`druid.metadata.storage.connector.user` +`druid.metadata.storage.connector.host` +`druid.metadata.storage.connector.port` + However, many times users may want their own way to optionally securely fetch password during runtime of the Druid process. Druid allows this by users to implement their own `PasswordProvider` interface and create a Druid extension to register this implementation at Druid process startup. Please have a look at "Adding a new Password Provider implementation" on this [page](../development/modules.md) to learn more. diff --git a/pom.xml b/pom.xml index a4c3cf9a5ecc..8a54d2176eab 100644 --- a/pom.xml +++ b/pom.xml @@ -238,6 +238,11 @@ aws-java-sdk-core ${aws.sdk.version} + + com.amazonaws + aws-java-sdk + ${aws.sdk.version} + com.amazonaws aws-java-sdk-ec2 From 85932c03fdd33e0c1e87bb4ead7243c36846e240 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 14 Mar 2020 14:52:48 -0700 Subject: [PATCH 03/25] fix analyze-dependencies build --- cloud/aws-common/pom.xml | 2 +- .../apache/druid/common/aws/AWSRDSTokenPasswordProvider.java | 4 ++-- pom.xml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cloud/aws-common/pom.xml b/cloud/aws-common/pom.xml index 99123f15e865..eea52417b12d 100644 --- a/cloud/aws-common/pom.xml +++ b/cloud/aws-common/pom.xml @@ -40,7 +40,7 @@ com.amazonaws - aws-java-sdk + aws-java-sdk-rds com.amazonaws diff --git a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java index 05bd6e55e040..32cac8c28012 100644 --- a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java +++ b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java @@ -85,8 +85,8 @@ public String getPassword() return authToken; } catch (Exception ex) { - LOGGER.error(ex, "Couldn't generate AWS IAM token."); - throw new RE(ex, "Couldn't generate AWS IAM token."); + LOGGER.error(ex, "Couldn't generate AWS token."); + throw new RE(ex, "Couldn't generate AWS token."); } } } diff --git a/pom.xml b/pom.xml index 8a54d2176eab..68bd65de9635 100644 --- a/pom.xml +++ b/pom.xml @@ -240,7 +240,7 @@ com.amazonaws - aws-java-sdk + aws-java-sdk-rds ${aws.sdk.version} From 2314fa4f72b0bc8d4b92fadb8f7d11d3899ae676 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 14 Mar 2020 14:52:55 -0700 Subject: [PATCH 04/25] fix doc build --- docs/operations/password-provider.md | 2 +- website/.spelling | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/operations/password-provider.md b/docs/operations/password-provider.md index acf375ff4de4..bd026060958c 100644 --- a/docs/operations/password-provider.md +++ b/docs/operations/password-provider.md @@ -43,7 +43,7 @@ The values are described below. |`type`|String|password provider type|Yes: `environment`| |`variable`|String|environment variable to read password from|Yes| -AWS RDS token password provider provides temp token for accesing AWS RDS DB instances. +AWS RDS token password provider provides temp token for accessing AWS RDS DB instances. ```json { "type": "awsrdstoken", "region": "AWS_REGION" } diff --git a/website/.spelling b/website/.spelling index 24a8ce5336bc..916ebcb6b17f 100644 --- a/website/.spelling +++ b/website/.spelling @@ -143,6 +143,7 @@ ParseSpecs Protobuf RDBMS RDDs +RDS Rackspace Redis S3 From 955b62fc607b63dbb79392d944034172c0bc59cf Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 14 Mar 2020 15:21:26 -0700 Subject: [PATCH 05/25] add ut for BasicDataSourceExt --- .../metadata/BasicDataSourceExtTest.java | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java diff --git a/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java b/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java new file mode 100644 index 000000000000..937d38615436 --- /dev/null +++ b/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.metadata; + +import org.apache.commons.dbcp2.ConnectionFactory; +import org.assertj.core.util.Lists; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Test; + +import java.sql.Driver; +import java.util.List; +import java.util.Properties; + +public class BasicDataSourceExtTest +{ + @Test + public void testCreateConnectionFactory() throws Exception + { + MetadataStorageConnectorConfig connectorConfig = new MetadataStorageConnectorConfig() + { + private final List passwords = Lists.newArrayList("pwd1", "pwd2"); + + @Override + public String getUser() + { + return "testuser"; + } + + @Override + public String getPassword() + { + return passwords.remove(0); + } + }; + + BasicDataSourceExt basicDataSourceExt = new BasicDataSourceExt(connectorConfig); + + basicDataSourceExt.setConnectionProperties("p1=v1;p2=v2;p3=v3"); + basicDataSourceExt.addConnectionProperty("p4", "v4"); + basicDataSourceExt.addConnectionProperty("p5", "v5"); + basicDataSourceExt.removeConnectionProperty("p2"); + basicDataSourceExt.removeConnectionProperty("p5"); + + Driver driver = EasyMock.mock(Driver.class); + Capture uriArg = Capture.newInstance(); + Capture propsArg = Capture.newInstance(); + EasyMock.expect(driver.connect(EasyMock.capture(uriArg), EasyMock.capture(propsArg))).andReturn(null).times(2); + EasyMock.replay(driver); + + basicDataSourceExt.setDriver(driver); + + ConnectionFactory connectionFactory = basicDataSourceExt.createConnectionFactory(); + + Properties expectedProps = new Properties(); + expectedProps.put("p1", "v1"); + expectedProps.put("p3", "v3"); + expectedProps.put("p4", "v4"); + expectedProps.put("user", connectorConfig.getUser()); + + + Assert.assertNull(connectionFactory.createConnection()); + Assert.assertEquals(connectorConfig.getConnectURI(), uriArg.getValue()); + + expectedProps.put("password", "pwd1"); + Assert.assertEquals(expectedProps, propsArg.getValue()); + + Assert.assertNull(connectionFactory.createConnection()); + Assert.assertEquals(connectorConfig.getConnectURI(), uriArg.getValue()); + + expectedProps.put("password", "pwd2"); + Assert.assertEquals(expectedProps, propsArg.getValue()); + } +} From 553f622fd204bdac14d042bf7419899281f1648e Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 14 Mar 2020 15:26:20 -0700 Subject: [PATCH 06/25] more doc updates --- docs/operations/password-provider.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/operations/password-provider.md b/docs/operations/password-provider.md index bd026060958c..9bb5c4872e7e 100644 --- a/docs/operations/password-provider.md +++ b/docs/operations/password-provider.md @@ -52,7 +52,8 @@ AWS RDS token password provider provides temp token for accessing AWS RDS DB ins Before using this password provider, please make sure that you have connected all dots for db user to connect using token. See https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html -Following additional metadata storage configuration must be provided to use `awsrdstokene` password provider. +Following additional metadata storage configuration must be provided to use `awsrdstoken` password provider. +`druid.metadata.storage.connector.connectURI` `druid.metadata.storage.connector.user` `druid.metadata.storage.connector.host` `druid.metadata.storage.connector.port` From ede86cca0feedf003c38fb498b6acad4cce53825 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 14 Mar 2020 15:44:58 -0700 Subject: [PATCH 07/25] more doc update --- docs/operations/password-provider.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/password-provider.md b/docs/operations/password-provider.md index 9bb5c4872e7e..15bf78209884 100644 --- a/docs/operations/password-provider.md +++ b/docs/operations/password-provider.md @@ -50,7 +50,7 @@ AWS RDS token password provider provides temp token for accessing AWS RDS DB ins ``` Before using this password provider, please make sure that you have connected all dots for db user to connect using token. -See https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html +See [AWS Guide](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html). Following additional metadata storage configuration must be provided to use `awsrdstoken` password provider. `druid.metadata.storage.connector.connectURI` From f9087568d35c61c18047ec993d1d0c61fb8c930b Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 26 Mar 2020 15:31:20 -0700 Subject: [PATCH 08/25] moving aws token password provider to new extension --- docs/development/extensions.md | 1 + .../druid-aws-rds-extensions/pom.xml | 50 ++++++++++ .../AWSMetadataStorageConnectorConfig.java | 49 ++++++++++ .../org/apache/druid/aws/rds/AWSModule.java | 48 ++++++++++ .../aws/rds/AWSRDSTokenPasswordProvider.java | 92 +++++++++++++++++++ ...rg.apache.druid.initialization.DruidModule | 16 ++++ pom.xml | 1 + 7 files changed, 257 insertions(+) create mode 100644 extensions-core/druid-aws-rds-extensions/pom.xml create mode 100644 extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSMetadataStorageConnectorConfig.java create mode 100644 extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java create mode 100644 extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java create mode 100644 extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule diff --git a/docs/development/extensions.md b/docs/development/extensions.md index 21bd8508b770..a4e10c5b1c2a 100644 --- a/docs/development/extensions.md +++ b/docs/development/extensions.md @@ -56,6 +56,7 @@ Core extensions are maintained by Druid committers. |druid-protobuf-extensions| Support for data in Protobuf data format.|[link](../development/extensions-core/protobuf.md)| |druid-s3-extensions|Interfacing with data in AWS S3, and using S3 as deep storage.|[link](../development/extensions-core/s3.md)| |druid-ec2-extensions|Interfacing with AWS EC2 for autoscaling middle managers|UNDOCUMENTED| +|druid-aws-rds-extensions|Support for AWS token based access to AWS RDS DB Cluster.|[link](../development/extensions-core/druid-aws-rds.md)| |druid-stats|Statistics related module including variance and standard deviation.|[link](../development/extensions-core/stats.md)| |mysql-metadata-storage|MySQL metadata store.|[link](../development/extensions-core/mysql.md)| |postgresql-metadata-storage|PostgreSQL metadata store.|[link](../development/extensions-core/postgresql.md)| diff --git a/extensions-core/druid-aws-rds-extensions/pom.xml b/extensions-core/druid-aws-rds-extensions/pom.xml new file mode 100644 index 000000000000..8f478e389dcd --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/pom.xml @@ -0,0 +1,50 @@ + + + + + 4.0.0 + + org.apache.druid.extensions + druid-aws-rds-extensions + druid-aws-rds-extensions + druid-aws-rds-extensions + + + org.apache.druid + druid + 0.18.0-SNAPSHOT + ../../pom.xml + + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + com.amazonaws + aws-java-sdk-rds + provided + + + diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSMetadataStorageConnectorConfig.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSMetadataStorageConnectorConfig.java new file mode 100644 index 000000000000..d7fe74f3cf88 --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSMetadataStorageConnectorConfig.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.aws.rds; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class AWSMetadataStorageConnectorConfig +{ + @JsonProperty + private String user = null; + + @JsonProperty + private String host = null; + + @JsonProperty + private int port = 0; + + public String getUser() + { + return user; + } + + public String getHost() + { + return host; + } + + public int getPort() + { + return port; + } +} diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java new file mode 100644 index 000000000000..310e047fc0f2 --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.aws.rds; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.initialization.DruidModule; + +import java.util.List; + +public class AWSModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return ImmutableList.of( + new SimpleModule("DruidAwsExtentionsModule").registerSubtypes( + AWSModule.class + ) + ); + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.metadata.storage.connector", AWSMetadataStorageConnectorConfig.class); + } +} diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java new file mode 100644 index 000000000000..c295df069902 --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.aws.rds; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.rds.auth.GetIamAuthTokenRequest; +import com.amazonaws.services.rds.auth.RdsIamAuthTokenGenerator; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.metadata.PasswordProvider; + +/** + * Generates the AWS token same as aws cli + * aws rds generate-db-auth-token --hostname HOST --port PORT --region REGION --username USER + * and returns that as password. + * + * Before using this, please make sure that you have connected all dots for db user to connect using token. + * See https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html + */ +public class AWSRDSTokenPasswordProvider implements PasswordProvider +{ + private static final Logger LOGGER = new Logger(AWSRDSTokenPasswordProvider.class); + private final String userName; + private final String hostName; + private final int port; + private final String region; + + private final AWSCredentialsProvider awsCredentialsProvider; + + @JsonCreator + public AWSRDSTokenPasswordProvider( + @JsonProperty("region") String region, + @JacksonInject AWSMetadataStorageConnectorConfig metadataStorageConnectorConfig, + @JacksonInject AWSCredentialsProvider awsCredentialsProvider + ) + { + userName = Preconditions.checkNotNull(metadataStorageConnectorConfig.getUser(), "null metadataStorage user"); + hostName = Preconditions.checkNotNull(metadataStorageConnectorConfig.getHost(), "null metadataStorage host"); + Preconditions.checkArgument(metadataStorageConnectorConfig.getPort() > 0, "must provide port"); + port = metadataStorageConnectorConfig.getPort(); + + this.region = Preconditions.checkNotNull(region, "null region"); + + LOGGER.info("AWS RDS Config user[%s], host[%s], port[%d], region[%s]", userName, hostName, port, this.region); + this.awsCredentialsProvider = awsCredentialsProvider; + } + + @Override + public String getPassword() + { + try { + RdsIamAuthTokenGenerator generator = RdsIamAuthTokenGenerator.builder() + .credentials(awsCredentialsProvider) + .region(region) + .build(); + + String authToken = generator.getAuthToken( + GetIamAuthTokenRequest.builder() + .hostname(hostName) + .port(port) + .userName(userName) + .build()); + + return authToken; + } + catch (Exception ex) { + LOGGER.error(ex, "Couldn't generate AWS token."); + throw new RE(ex, "Couldn't generate AWS token."); + } + } +} diff --git a/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..85ad1c293284 --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.aws.rds.AWSModule diff --git a/pom.xml b/pom.xml index 0ea5ca5605e6..910fe379bb14 100644 --- a/pom.xml +++ b/pom.xml @@ -159,6 +159,7 @@ extensions-core/lookups-cached-single extensions-core/ec2-extensions extensions-core/s3-extensions + extensions-core/druid-aws-rds-extensions extensions-core/simple-client-sslcontext extensions-core/druid-basic-security extensions-core/google-extensions From 35a05274c64ec115c92a72fa9a5f0fe35456250e Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 26 Mar 2020 15:34:16 -0700 Subject: [PATCH 09/25] remove duplicate changes --- cloud/aws-common/pom.xml | 4 - .../AWSMetadataStorageConnectorConfig.java | 49 ---------- .../apache/druid/common/aws/AWSModule.java | 10 +- .../aws/AWSRDSTokenPasswordProvider.java | 92 ------------------- docs/operations/password-provider.md | 15 --- 5 files changed, 1 insertion(+), 169 deletions(-) delete mode 100644 cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSMetadataStorageConnectorConfig.java delete mode 100644 cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java diff --git a/cloud/aws-common/pom.xml b/cloud/aws-common/pom.xml index eea52417b12d..2cd860fae8fc 100644 --- a/cloud/aws-common/pom.xml +++ b/cloud/aws-common/pom.xml @@ -38,10 +38,6 @@ druid-core ${project.parent.version} - - com.amazonaws - aws-java-sdk-rds - com.amazonaws aws-java-sdk-ec2 diff --git a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSMetadataStorageConnectorConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSMetadataStorageConnectorConfig.java deleted file mode 100644 index d76ec1f09530..000000000000 --- a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSMetadataStorageConnectorConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.common.aws; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class AWSMetadataStorageConnectorConfig -{ - @JsonProperty - private String user = null; - - @JsonProperty - private String host = null; - - @JsonProperty - private int port = 0; - - public String getUser() - { - return user; - } - - public String getHost() - { - return host; - } - - public int getPort() - { - return port; - } -} diff --git a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java index 86674bc7d276..79b60c956c34 100644 --- a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java +++ b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java @@ -23,8 +23,6 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; import com.fasterxml.jackson.databind.Module; -import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.inject.Binder; import com.google.inject.Provides; import org.apache.druid.guice.JsonConfigProvider; @@ -43,8 +41,6 @@ public void configure(Binder binder) JsonConfigProvider.bind(binder, "druid.s3", AWSClientConfig.class); JsonConfigProvider.bind(binder, "druid.s3.proxy", AWSProxyConfig.class); JsonConfigProvider.bind(binder, "druid.s3.endpoint", AWSEndpointConfig.class); - - JsonConfigProvider.bind(binder, "druid.metadata.storage.connector", AWSMetadataStorageConnectorConfig.class); } @Provides @@ -64,10 +60,6 @@ public AmazonEC2 getEc2Client(AWSCredentialsProvider credentials) @Override public List getJacksonModules() { - return Collections.singletonList( - new SimpleModule("AWSModule").registerSubtypes( - new NamedType(AWSRDSTokenPasswordProvider.class, "awsrdstoken") - ) - ); + return Collections.emptyList(); } } diff --git a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java deleted file mode 100644 index 32cac8c28012..000000000000 --- a/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSRDSTokenPasswordProvider.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.common.aws; - -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.services.rds.auth.GetIamAuthTokenRequest; -import com.amazonaws.services.rds.auth.RdsIamAuthTokenGenerator; -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; -import org.apache.druid.java.util.common.RE; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.metadata.PasswordProvider; - -/** - * Generates the AWS token same as aws cli - * aws rds generate-db-auth-token --hostname HOST --port PORT --region REGION --username USER - * and returns that as password. - * - * Before using this, please make sure that you have connected all dots for db user to connect using token. - * See https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html - */ -public class AWSRDSTokenPasswordProvider implements PasswordProvider -{ - private static final Logger LOGGER = new Logger(AWSRDSTokenPasswordProvider.class); - private final String userName; - private final String hostName; - private final int port; - private final String region; - - private final AWSCredentialsProvider awsCredentialsProvider; - - @JsonCreator - public AWSRDSTokenPasswordProvider( - @JsonProperty("region") String region, - @JacksonInject AWSMetadataStorageConnectorConfig metadataStorageConnectorConfig, - @JacksonInject AWSCredentialsProvider awsCredentialsProvider - ) - { - userName = Preconditions.checkNotNull(metadataStorageConnectorConfig.getUser(), "null metadataStorage user"); - hostName = Preconditions.checkNotNull(metadataStorageConnectorConfig.getHost(), "null metadataStorage host"); - Preconditions.checkArgument(metadataStorageConnectorConfig.getPort() > 0, "must provide port"); - port = metadataStorageConnectorConfig.getPort(); - - this.region = Preconditions.checkNotNull(region, "null region"); - - LOGGER.info("AWS RDS Config user[%s], host[%s], port[%d], region[%s]", userName, hostName, port, this.region); - this.awsCredentialsProvider = awsCredentialsProvider; - } - - @Override - public String getPassword() - { - try { - RdsIamAuthTokenGenerator generator = RdsIamAuthTokenGenerator.builder() - .credentials(awsCredentialsProvider) - .region(region) - .build(); - - String authToken = generator.getAuthToken( - GetIamAuthTokenRequest.builder() - .hostname(hostName) - .port(port) - .userName(userName) - .build()); - - return authToken; - } - catch (Exception ex) { - LOGGER.error(ex, "Couldn't generate AWS token."); - throw new RE(ex, "Couldn't generate AWS token."); - } - } -} diff --git a/docs/operations/password-provider.md b/docs/operations/password-provider.md index 15bf78209884..4a28e64aae8b 100644 --- a/docs/operations/password-provider.md +++ b/docs/operations/password-provider.md @@ -43,21 +43,6 @@ The values are described below. |`type`|String|password provider type|Yes: `environment`| |`variable`|String|environment variable to read password from|Yes| -AWS RDS token password provider provides temp token for accessing AWS RDS DB instances. - -```json -{ "type": "awsrdstoken", "region": "AWS_REGION" } -``` - -Before using this password provider, please make sure that you have connected all dots for db user to connect using token. -See [AWS Guide](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html). - -Following additional metadata storage configuration must be provided to use `awsrdstoken` password provider. -`druid.metadata.storage.connector.connectURI` -`druid.metadata.storage.connector.user` -`druid.metadata.storage.connector.host` -`druid.metadata.storage.connector.port` - However, many times users may want their own way to optionally securely fetch password during runtime of the Druid process. Druid allows this by users to implement their own `PasswordProvider` interface and create a Druid extension to register this implementation at Druid process startup. Please have a look at "Adding a new Password Provider implementation" on this [page](../development/modules.md) to learn more. From 52ff609d08e05abd51b39355280480885af6d96a Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 26 Mar 2020 16:07:24 -0700 Subject: [PATCH 10/25] make all config inline --- .../AWSMetadataStorageConnectorConfig.java | 49 ------------------- .../org/apache/druid/aws/rds/AWSModule.java | 7 ++- .../aws/rds/AWSRDSTokenPasswordProvider.java | 33 +++++++------ 3 files changed, 22 insertions(+), 67 deletions(-) delete mode 100644 extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSMetadataStorageConnectorConfig.java diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSMetadataStorageConnectorConfig.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSMetadataStorageConnectorConfig.java deleted file mode 100644 index d7fe74f3cf88..000000000000 --- a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSMetadataStorageConnectorConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.aws.rds; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class AWSMetadataStorageConnectorConfig -{ - @JsonProperty - private String user = null; - - @JsonProperty - private String host = null; - - @JsonProperty - private int port = 0; - - public String getUser() - { - return user; - } - - public String getHost() - { - return host; - } - - public int getPort() - { - return port; - } -} diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java index 310e047fc0f2..c16bd333e519 100644 --- a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java +++ b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java @@ -20,10 +20,10 @@ package org.apache.druid.aws.rds; import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; -import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.initialization.DruidModule; import java.util.List; @@ -34,8 +34,8 @@ public class AWSModule implements DruidModule public List getJacksonModules() { return ImmutableList.of( - new SimpleModule("DruidAwsExtentionsModule").registerSubtypes( - AWSModule.class + new SimpleModule("DruidAwsRdsExtentionsModule").registerSubtypes( + new NamedType(AWSRDSTokenPasswordProvider.class, "awsrdstoken") ) ); } @@ -43,6 +43,5 @@ public List getJacksonModules() @Override public void configure(Binder binder) { - JsonConfigProvider.bind(binder, "druid.metadata.storage.connector", AWSMetadataStorageConnectorConfig.class); } } diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java index c295df069902..a0429335d63d 100644 --- a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java +++ b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java @@ -50,15 +50,17 @@ public class AWSRDSTokenPasswordProvider implements PasswordProvider @JsonCreator public AWSRDSTokenPasswordProvider( + @JsonProperty("user") String user, + @JsonProperty("host") String host, + @JsonProperty("port") int port, @JsonProperty("region") String region, - @JacksonInject AWSMetadataStorageConnectorConfig metadataStorageConnectorConfig, @JacksonInject AWSCredentialsProvider awsCredentialsProvider ) { - userName = Preconditions.checkNotNull(metadataStorageConnectorConfig.getUser(), "null metadataStorage user"); - hostName = Preconditions.checkNotNull(metadataStorageConnectorConfig.getHost(), "null metadataStorage host"); - Preconditions.checkArgument(metadataStorageConnectorConfig.getPort() > 0, "must provide port"); - port = metadataStorageConnectorConfig.getPort(); + userName = Preconditions.checkNotNull(user, "null metadataStorage user"); + hostName = Preconditions.checkNotNull(host, "null metadataStorage host"); + Preconditions.checkArgument(port > 0, "must provide port"); + this.port = port; this.region = Preconditions.checkNotNull(region, "null region"); @@ -70,17 +72,20 @@ public AWSRDSTokenPasswordProvider( public String getPassword() { try { - RdsIamAuthTokenGenerator generator = RdsIamAuthTokenGenerator.builder() - .credentials(awsCredentialsProvider) - .region(region) - .build(); + RdsIamAuthTokenGenerator generator = RdsIamAuthTokenGenerator + .builder() + .credentials(awsCredentialsProvider) + .region(region) + .build(); String authToken = generator.getAuthToken( - GetIamAuthTokenRequest.builder() - .hostname(hostName) - .port(port) - .userName(userName) - .build()); + GetIamAuthTokenRequest + .builder() + .hostname(hostName) + .port(port) + .userName(userName) + .build() + ); return authToken; } From 52efefc289c6c49fc2c9a1af21a36e76114c51b3 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 26 Mar 2020 16:21:38 -0700 Subject: [PATCH 11/25] extension docs --- .../extensions-core/druid-aws-rds.md | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/development/extensions-core/druid-aws-rds.md diff --git a/docs/development/extensions-core/druid-aws-rds.md b/docs/development/extensions-core/druid-aws-rds.md new file mode 100644 index 000000000000..b24e3a20b69a --- /dev/null +++ b/docs/development/extensions-core/druid-aws-rds.md @@ -0,0 +1,38 @@ +--- +id: druid-aws-rds +title: "Druid AWS RDS Module" +--- + + + +This module provides AWS RDS token [password provider](../../operations/password-provider.md) provides temp token for accessing AWS RDS DB cluster. + +```json +{ "type": "awsrdstoken", "user": "USER", "host": "HOST", "port": PORT, "region": "AWS_REGION" } +``` + +Before using this password provider, please make sure that you have connected all dots for db user to connect using token. +See [AWS Guide](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html). + +To use this extension, make sure you [include](../../development/extensions.md#loading-extensions) it in your config file: + +``` +druid.extensions.loadList=["druid-aws-rds-extensions"] +``` From 6969403c5f59378812efe6b796527d030c530ec6 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 26 Mar 2020 16:24:37 -0700 Subject: [PATCH 12/25] refresh db password in SQL Firehose code path as well --- .../org/apache/druid/metadata/SQLFirehoseDatabaseConnector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/metadata/SQLFirehoseDatabaseConnector.java b/server/src/main/java/org/apache/druid/metadata/SQLFirehoseDatabaseConnector.java index 242f04bccfcb..f72d9ae1fb5b 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLFirehoseDatabaseConnector.java +++ b/server/src/main/java/org/apache/druid/metadata/SQLFirehoseDatabaseConnector.java @@ -64,7 +64,7 @@ public final boolean isTransientException(Throwable e) protected BasicDataSource getDatasource(MetadataStorageConnectorConfig connectorConfig) { - BasicDataSource dataSource = new BasicDataSource(); + BasicDataSource dataSource = new BasicDataSourceExt(connectorConfig); dataSource.setUsername(connectorConfig.getUser()); dataSource.setPassword(connectorConfig.getPassword()); String uri = connectorConfig.getConnectURI(); From 27f56523323aab10ff37d46e32c12154c280417d Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 26 Mar 2020 17:38:06 -0700 Subject: [PATCH 13/25] add ut --- .../druid-aws-rds-extensions/pom.xml | 6 ++ .../aws/rds/AWSRDSTokenPasswordProvider.java | 42 ++++++++-- .../rds/AWSRDSTokenPasswordProviderTest.java | 82 +++++++++++++++++++ 3 files changed, 122 insertions(+), 8 deletions(-) create mode 100644 extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java diff --git a/extensions-core/druid-aws-rds-extensions/pom.xml b/extensions-core/druid-aws-rds-extensions/pom.xml index 8f478e389dcd..704f8040b2ca 100644 --- a/extensions-core/druid-aws-rds-extensions/pom.xml +++ b/extensions-core/druid-aws-rds-extensions/pom.xml @@ -46,5 +46,11 @@ aws-java-sdk-rds provided + + + junit + junit + test + diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java index a0429335d63d..9c54d45124cb 100644 --- a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java +++ b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java @@ -24,6 +24,7 @@ import com.amazonaws.services.rds.auth.RdsIamAuthTokenGenerator; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import org.apache.druid.java.util.common.RE; @@ -41,8 +42,8 @@ public class AWSRDSTokenPasswordProvider implements PasswordProvider { private static final Logger LOGGER = new Logger(AWSRDSTokenPasswordProvider.class); - private final String userName; - private final String hostName; + private final String user; + private final String host; private final int port; private final String region; @@ -57,17 +58,42 @@ public AWSRDSTokenPasswordProvider( @JacksonInject AWSCredentialsProvider awsCredentialsProvider ) { - userName = Preconditions.checkNotNull(user, "null metadataStorage user"); - hostName = Preconditions.checkNotNull(host, "null metadataStorage host"); + this.user = Preconditions.checkNotNull(user, "null metadataStorage user"); + this.host = Preconditions.checkNotNull(host, "null metadataStorage host"); Preconditions.checkArgument(port > 0, "must provide port"); this.port = port; this.region = Preconditions.checkNotNull(region, "null region"); - LOGGER.info("AWS RDS Config user[%s], host[%s], port[%d], region[%s]", userName, hostName, port, this.region); - this.awsCredentialsProvider = awsCredentialsProvider; + LOGGER.info("AWS RDS Config user[%s], host[%s], port[%d], region[%s]", this.user, this.host, port, this.region); + this.awsCredentialsProvider = Preconditions.checkNotNull(awsCredentialsProvider, "null AWSCredentialsProvider"); } + @JsonProperty + public String getUser() + { + return user; + } + + @JsonProperty + public String getHost() + { + return host; + } + + @JsonProperty + public int getPort() + { + return port; + } + + @JsonProperty + public String getRegion() + { + return region; + } + + @JsonIgnore @Override public String getPassword() { @@ -81,9 +107,9 @@ public String getPassword() String authToken = generator.getAuthToken( GetIamAuthTokenRequest .builder() - .hostname(hostName) + .hostname(host) .port(port) - .userName(userName) + .userName(user) .build() ); diff --git a/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java b/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java new file mode 100644 index 000000000000..6278afef71e0 --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.aws.rds; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.metadata.PasswordProvider; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class AWSRDSTokenPasswordProviderTest +{ + @Test + public void testSerde() throws IOException + { + ObjectMapper jsonMapper = new ObjectMapper(); + + for (Module module : new AWSModule().getJacksonModules()) { + jsonMapper.registerModule(module); + } + + jsonMapper.setInjectableValues( + new InjectableValues.Std().addValue(AWSCredentialsProvider.class, new AWSCredentialsProvider() + { + @Override + public AWSCredentials getCredentials() + { + return null; + } + + @Override + public void refresh() + { + + } + }) + ); + + String jsonStr = "{\n" + + " \"type\": \"awsrdstoken\",\n" + + " \"user\": \"testuser\",\n" + + " \"host\": \"testhost\",\n" + + " \"port\": 5273,\n" + + " \"region\": \"testregion\"\n" + + "}\n"; + + PasswordProvider pp = jsonMapper.readValue( + jsonMapper.writeValueAsString( + jsonMapper.readValue(jsonStr, PasswordProvider.class) + ), + PasswordProvider.class + ); + + AWSRDSTokenPasswordProvider awsPwdProvider = (AWSRDSTokenPasswordProvider) pp; + Assert.assertEquals("testuser", awsPwdProvider.getUser()); + Assert.assertEquals("testhost", awsPwdProvider.getHost()); + Assert.assertEquals(5273, awsPwdProvider.getPort()); + Assert.assertEquals("testregion", awsPwdProvider.getRegion()); + } +} From ee98fc81c92df882a1020e43142bccd11e4a26b6 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 26 Mar 2020 21:12:21 -0700 Subject: [PATCH 14/25] fix build --- .../druid-aws-rds-extensions/pom.xml | 31 ++++++++++++++++++- website/.spelling | 1 + 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/extensions-core/druid-aws-rds-extensions/pom.xml b/extensions-core/druid-aws-rds-extensions/pom.xml index 704f8040b2ca..6317fe567e9a 100644 --- a/extensions-core/druid-aws-rds-extensions/pom.xml +++ b/extensions-core/druid-aws-rds-extensions/pom.xml @@ -46,7 +46,36 @@ aws-java-sdk-rds provided - + + com.google.guava + guava + 16.0.1 + provided + + + com.google.inject + guice + 4.1.0 + provided + + + com.amazonaws + aws-java-sdk-core + 1.11.199 + provided + + + com.fasterxml.jackson.core + jackson-databind + 2.10.2 + provided + + + com.fasterxml.jackson.core + jackson-annotations + 2.10.2 + provided + junit junit diff --git a/website/.spelling b/website/.spelling index b35544928133..98d33c8f9369 100644 --- a/website/.spelling +++ b/website/.spelling @@ -816,6 +816,7 @@ DistinctCount artifactId com.example common.runtime.properties +druid-aws-rds-extensions druid-cassandra-storage druid-distinctcount druid-ec2-extensions From 46e22642f4be54b69d28a35d2a15d0ae242b4c86 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 26 Mar 2020 23:56:31 -0700 Subject: [PATCH 15/25] add new extension to distribution --- distribution/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 446e79a0de7a..c12a46aa7016 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -231,6 +231,8 @@ -c org.apache.druid.extensions:druid-s3-extensions -c + org.apache.druid.extensions:druid-aws-rds-extensions + -c org.apache.druid.extensions:druid-ec2-extensions -c org.apache.druid.extensions:druid-google-extensions From ca825017a94e59939a038093bde109b17517cd83 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Fri, 27 Mar 2020 00:03:18 -0700 Subject: [PATCH 16/25] rds lib is not provided --- extensions-core/druid-aws-rds-extensions/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/extensions-core/druid-aws-rds-extensions/pom.xml b/extensions-core/druid-aws-rds-extensions/pom.xml index 6317fe567e9a..eb91c2e7680f 100644 --- a/extensions-core/druid-aws-rds-extensions/pom.xml +++ b/extensions-core/druid-aws-rds-extensions/pom.xml @@ -44,7 +44,6 @@ com.amazonaws aws-java-sdk-rds - provided com.google.guava From 1a7726c07ec1e88a2f8ce685538ca61344c0b763 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Fri, 27 Mar 2020 11:36:43 -0700 Subject: [PATCH 17/25] fix license build --- extensions-core/druid-aws-rds-extensions/pom.xml | 6 +----- licenses.yaml | 9 +++++++++ pom.xml | 5 ----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/extensions-core/druid-aws-rds-extensions/pom.xml b/extensions-core/druid-aws-rds-extensions/pom.xml index eb91c2e7680f..479a5afb5ba2 100644 --- a/extensions-core/druid-aws-rds-extensions/pom.xml +++ b/extensions-core/druid-aws-rds-extensions/pom.xml @@ -44,35 +44,31 @@ com.amazonaws aws-java-sdk-rds + ${aws.sdk.version} com.google.guava guava - 16.0.1 provided com.google.inject guice - 4.1.0 provided com.amazonaws aws-java-sdk-core - 1.11.199 provided com.fasterxml.jackson.core jackson-databind - 2.10.2 provided com.fasterxml.jackson.core jackson-annotations - 2.10.2 provided diff --git a/licenses.yaml b/licenses.yaml index 7af752619474..32d29ea33d27 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -146,6 +146,15 @@ source_paths: --- +name: AWS RDS SDK for Java +license_category: source +module: extensions/druid-aws-rds-extensions +license_name: Apache License version 2.0 +libraries: + - com.amazonaws: aws-java-sdk-rds + +--- + name: AWS SDK for Java license_category: binary module: java-core diff --git a/pom.xml b/pom.xml index 910fe379bb14..ff33ca286c44 100644 --- a/pom.xml +++ b/pom.xml @@ -241,11 +241,6 @@ aws-java-sdk-core ${aws.sdk.version} - - com.amazonaws - aws-java-sdk-rds - ${aws.sdk.version} - com.amazonaws aws-java-sdk-ec2 From f5214e03c1d94ab65713d0a11c6f20096058f1f6 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 28 Mar 2020 00:48:17 -0700 Subject: [PATCH 18/25] add version to license --- licenses.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/licenses.yaml b/licenses.yaml index 32d29ea33d27..c18157430479 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -150,6 +150,7 @@ name: AWS RDS SDK for Java license_category: source module: extensions/druid-aws-rds-extensions license_name: Apache License version 2.0 +version: 1.11.199 libraries: - com.amazonaws: aws-java-sdk-rds From d65edc01e891da8b7df81fb38e3d196004d7e091 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 4 Apr 2020 18:20:27 -0700 Subject: [PATCH 19/25] change parent version to 0.19.0-snapshot --- extensions-core/druid-aws-rds-extensions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-core/druid-aws-rds-extensions/pom.xml b/extensions-core/druid-aws-rds-extensions/pom.xml index 479a5afb5ba2..69516adcf125 100644 --- a/extensions-core/druid-aws-rds-extensions/pom.xml +++ b/extensions-core/druid-aws-rds-extensions/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 0.18.0-SNAPSHOT + 0.19.0-SNAPSHOT ../../pom.xml From 6ab56872d007d82f229caafa1e1f753f92a5126c Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Tue, 5 Jan 2021 07:59:02 -0800 Subject: [PATCH 20/25] address review comments --- docs/development/extensions-core/druid-aws-rds.md | 2 +- extensions-core/druid-aws-rds-extensions/pom.xml | 2 +- .../aws/rds/{AWSModule.java => AWSRDSModule.java} | 4 ++-- .../org.apache.druid.initialization.DruidModule | 2 +- .../aws/rds/AWSRDSTokenPasswordProviderTest.java | 4 ++-- .../org/apache/druid/metadata/BasicDataSourceExt.java | 11 ++++++----- 6 files changed, 13 insertions(+), 12 deletions(-) rename extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/{AWSModule.java => AWSRDSModule.java} (95%) diff --git a/docs/development/extensions-core/druid-aws-rds.md b/docs/development/extensions-core/druid-aws-rds.md index b24e3a20b69a..aabff6339ca2 100644 --- a/docs/development/extensions-core/druid-aws-rds.md +++ b/docs/development/extensions-core/druid-aws-rds.md @@ -25,7 +25,7 @@ title: "Druid AWS RDS Module" This module provides AWS RDS token [password provider](../../operations/password-provider.md) provides temp token for accessing AWS RDS DB cluster. ```json -{ "type": "awsrdstoken", "user": "USER", "host": "HOST", "port": PORT, "region": "AWS_REGION" } +{ "type": "aws-rds-token", "user": "USER", "host": "HOST", "port": PORT, "region": "AWS_REGION" } ``` Before using this password provider, please make sure that you have connected all dots for db user to connect using token. diff --git a/extensions-core/druid-aws-rds-extensions/pom.xml b/extensions-core/druid-aws-rds-extensions/pom.xml index 69516adcf125..397039e36eaa 100644 --- a/extensions-core/druid-aws-rds-extensions/pom.xml +++ b/extensions-core/druid-aws-rds-extensions/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 0.19.0-SNAPSHOT + 0.21.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSModule.java similarity index 95% rename from extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java rename to extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSModule.java index c16bd333e519..298792a09b6b 100644 --- a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSModule.java +++ b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSModule.java @@ -28,14 +28,14 @@ import java.util.List; -public class AWSModule implements DruidModule +public class AWSRDSModule implements DruidModule { @Override public List getJacksonModules() { return ImmutableList.of( new SimpleModule("DruidAwsRdsExtentionsModule").registerSubtypes( - new NamedType(AWSRDSTokenPasswordProvider.class, "awsrdstoken") + new NamedType(AWSRDSTokenPasswordProvider.class, "aws-rds-token") ) ); } diff --git a/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule index 85ad1c293284..93b5bbc38faa 100644 --- a/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.druid.aws.rds.AWSModule +org.apache.druid.aws.rds.AWSRDSModule diff --git a/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java b/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java index 6278afef71e0..c49a7862cbcb 100644 --- a/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java +++ b/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java @@ -37,7 +37,7 @@ public void testSerde() throws IOException { ObjectMapper jsonMapper = new ObjectMapper(); - for (Module module : new AWSModule().getJacksonModules()) { + for (Module module : new AWSRDSModule().getJacksonModules()) { jsonMapper.registerModule(module); } @@ -59,7 +59,7 @@ public void refresh() ); String jsonStr = "{\n" - + " \"type\": \"awsrdstoken\",\n" + + " \"type\": \"aws-rds-token\",\n" + " \"user\": \"testuser\",\n" + " \"host\": \"testhost\",\n" + " \"port\": 5273,\n" diff --git a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java index 642938c05275..f2ced9575702 100644 --- a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java +++ b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java @@ -30,11 +30,12 @@ import java.util.Properties; /** - * This class exists so that PasswordProvider is asked for password every time a brand new connection is established - * with DB. PasswordProvider impls such as based on AWS tokens refresh the underlying token periodically since - * each token is valid for a certain period of time only. - * So, This class overrides[ummm copies] the methods from base class in order to keep track of connection properties - * and call MetadataStorageConnectorConfig.getPassword() everytime a new connection is setup. + * This class exists so that {@link MetadataStorageConnectorConfig} is asked for password every time a brand new + * connection is established with DB. {@link PasswordProvider} impls such as those based on AWS tokens refresh the + * underlying token periodically since each token is valid for a certain period of time only. + * So, This class overrides,ummm copies due to lack of extensibility, the methods from base class in order to keep + * track of connection properties and call {@link MetadataStorageConnectorConfig#getPassword()} everytime a new + * connection is setup. */ public class BasicDataSourceExt extends BasicDataSource { From f04c23acbc057823cce5369038adfb07ed6340c3 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Tue, 5 Jan 2021 13:14:52 -0800 Subject: [PATCH 21/25] fix core/ code coverage --- pom.xml | 3 +- server/pom.xml | 11 ++++++ .../druid/metadata/BasicDataSourceExt.java | 7 ++++ .../metadata/BasicDataSourceExtTest.java | 35 +++++++++++++++---- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index d4509a8c4c20..812c2267eab9 100644 --- a/pom.xml +++ b/pom.xml @@ -112,6 +112,7 @@ 2.0.2 1.11.199 2.8.0 + 0.8.5 @@ -1270,7 +1271,7 @@ org.jacoco jacoco-maven-plugin - 0.8.5 + ${jacoco.version} diff --git a/server/pom.xml b/server/pom.xml index 01b0d034d251..414b5f24e569 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -459,6 +459,17 @@ + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + + org/apache/druid/metadata/BasicDataSourceExt.class + + + org.apache.maven.plugins maven-jar-plugin diff --git a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java index f2ced9575702..438013464977 100644 --- a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java +++ b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java @@ -19,6 +19,7 @@ package org.apache.druid.metadata; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.dbcp2.ConnectionFactory; import org.apache.druid.java.util.common.RE; @@ -90,6 +91,12 @@ public void setConnectionProperties(String connectionProperties) super.setConnectionProperties(connectionProperties); } + @VisibleForTesting + public Properties getConnectionProperties() + { + return connectionProperties; + } + @Override protected ConnectionFactory createConnectionFactory() throws SQLException { diff --git a/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java b/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java index 937d38615436..1f8af8f6c981 100644 --- a/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java +++ b/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java @@ -54,11 +54,8 @@ public String getPassword() BasicDataSourceExt basicDataSourceExt = new BasicDataSourceExt(connectorConfig); - basicDataSourceExt.setConnectionProperties("p1=v1;p2=v2;p3=v3"); - basicDataSourceExt.addConnectionProperty("p4", "v4"); - basicDataSourceExt.addConnectionProperty("p5", "v5"); - basicDataSourceExt.removeConnectionProperty("p2"); - basicDataSourceExt.removeConnectionProperty("p5"); + basicDataSourceExt.setConnectionProperties("p1=v1"); + basicDataSourceExt.addConnectionProperty("p2", "v2"); Driver driver = EasyMock.mock(Driver.class); Capture uriArg = Capture.newInstance(); @@ -72,8 +69,7 @@ public String getPassword() Properties expectedProps = new Properties(); expectedProps.put("p1", "v1"); - expectedProps.put("p3", "v3"); - expectedProps.put("p4", "v4"); + expectedProps.put("p2", "v2"); expectedProps.put("user", connectorConfig.getUser()); @@ -89,4 +85,29 @@ public String getPassword() expectedProps.put("password", "pwd2"); Assert.assertEquals(expectedProps, propsArg.getValue()); } + + @Test + public void testConnectionPropertiesHanding() + { + BasicDataSourceExt basicDataSourceExt = new BasicDataSourceExt(EasyMock.mock(MetadataStorageConnectorConfig.class)); + Properties expectedProps = new Properties(); + + basicDataSourceExt.setConnectionProperties(""); + Assert.assertEquals(expectedProps, basicDataSourceExt.getConnectionProperties()); + + basicDataSourceExt.setConnectionProperties("p0;p1=v1;p2=v2;p3=v3"); + basicDataSourceExt.addConnectionProperty("p4", "v4"); + basicDataSourceExt.addConnectionProperty("p5", "v5"); + basicDataSourceExt.removeConnectionProperty("p2"); + basicDataSourceExt.removeConnectionProperty("p5"); + + expectedProps.put("p0", ""); + expectedProps.put("p1", "v1"); + expectedProps.put("p3", "v3"); + expectedProps.put("p4", "v4"); + + Assert.assertEquals(expectedProps, basicDataSourceExt.getConnectionProperties()); + + + } } From 254de29b3688941e38048b64b41f8b7b2dd34119 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Wed, 6 Jan 2021 13:57:07 -0800 Subject: [PATCH 22/25] Update server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java Co-authored-by: Clint Wylie --- .../main/java/org/apache/druid/metadata/BasicDataSourceExt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java index 438013464977..c24ace035754 100644 --- a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java +++ b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java @@ -34,7 +34,7 @@ * This class exists so that {@link MetadataStorageConnectorConfig} is asked for password every time a brand new * connection is established with DB. {@link PasswordProvider} impls such as those based on AWS tokens refresh the * underlying token periodically since each token is valid for a certain period of time only. - * So, This class overrides,ummm copies due to lack of extensibility, the methods from base class in order to keep + * So, This class overrides (and largely copies due to lack of extensibility), the methods from base class in order to keep * track of connection properties and call {@link MetadataStorageConnectorConfig#getPassword()} everytime a new * connection is setup. */ From fe2ab6a58e49dc4e7d3be834193e51304a27b7ac Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 6 Jan 2021 13:58:10 -0800 Subject: [PATCH 23/25] address review comments --- docs/development/extensions-core/druid-aws-rds.md | 6 +++--- .../java/org/apache/druid/metadata/BasicDataSourceExt.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/development/extensions-core/druid-aws-rds.md b/docs/development/extensions-core/druid-aws-rds.md index aabff6339ca2..d31b121c435a 100644 --- a/docs/development/extensions-core/druid-aws-rds.md +++ b/docs/development/extensions-core/druid-aws-rds.md @@ -22,7 +22,7 @@ title: "Druid AWS RDS Module" ~ under the License. --> -This module provides AWS RDS token [password provider](../../operations/password-provider.md) provides temp token for accessing AWS RDS DB cluster. +[AWS RDS](https://aws.amazon.com/rds/) is a managed service to operate relation databases such as PostgreSQL, Mysql etc. These databases could be accessed using tranditional static db password mechanism or via [AWS IAM](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html) temporary tokens. This module provides AWS RDS token [password provider](../../operations/password-provider.md) implementation to be used with [mysql-metadata-store](mysql.md) or [postgresql-metadata-store](postgresql.md) when mysql/postgresql is operated using AWS RDS. ```json { "type": "aws-rds-token", "user": "USER", "host": "HOST", "port": PORT, "region": "AWS_REGION" } @@ -31,8 +31,8 @@ This module provides AWS RDS token [password provider](../../operations/password Before using this password provider, please make sure that you have connected all dots for db user to connect using token. See [AWS Guide](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html). -To use this extension, make sure you [include](../../development/extensions.md#loading-extensions) it in your config file: +To use this extension, make sure you [include](../../development/extensions.md#loading-extensions) it in your config file along with other extensions e.g. ``` -druid.extensions.loadList=["druid-aws-rds-extensions"] +druid.extensions.loadList=["druid-aws-rds-extensions", "postgresql-metadata-storage", ...] ``` diff --git a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java index c24ace035754..e077aae55f90 100644 --- a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java +++ b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java @@ -151,7 +151,7 @@ protected ConnectionFactory createConnectionFactory() throws SQLException } if (driverToUse == null) { - throw new RE("WTH! Couln't find a Driver"); + throw new RE("Failed to find the DB Driver"); } final Driver finalDriverToUse = driverToUse; From 71c452b97e19a6422cf8e49c4636047eecafddb5 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 6 Jan 2021 14:01:00 -0800 Subject: [PATCH 24/25] fix spellchecker --- docs/development/extensions-core/druid-aws-rds.md | 2 +- website/.spelling | 1 + website/i18n/en.json | 9 +++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/development/extensions-core/druid-aws-rds.md b/docs/development/extensions-core/druid-aws-rds.md index d31b121c435a..6bb0cd826be4 100644 --- a/docs/development/extensions-core/druid-aws-rds.md +++ b/docs/development/extensions-core/druid-aws-rds.md @@ -22,7 +22,7 @@ title: "Druid AWS RDS Module" ~ under the License. --> -[AWS RDS](https://aws.amazon.com/rds/) is a managed service to operate relation databases such as PostgreSQL, Mysql etc. These databases could be accessed using tranditional static db password mechanism or via [AWS IAM](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html) temporary tokens. This module provides AWS RDS token [password provider](../../operations/password-provider.md) implementation to be used with [mysql-metadata-store](mysql.md) or [postgresql-metadata-store](postgresql.md) when mysql/postgresql is operated using AWS RDS. +[AWS RDS](https://aws.amazon.com/rds/) is a managed service to operate relation databases such as PostgreSQL, Mysql etc. These databases could be accessed using static db password mechanism or via [AWS IAM](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html) temporary tokens. This module provides AWS RDS token [password provider](../../operations/password-provider.md) implementation to be used with [mysql-metadata-store](mysql.md) or [postgresql-metadata-store](postgresql.md) when mysql/postgresql is operated using AWS RDS. ```json { "type": "aws-rds-token", "user": "USER", "host": "HOST", "port": PORT, "region": "AWS_REGION" } diff --git a/website/.spelling b/website/.spelling index 938f74cdfb92..e538296b2c18 100644 --- a/website/.spelling +++ b/website/.spelling @@ -88,6 +88,7 @@ HLL HashSet Homebrew HyperLogLog +IAM IANA IETF IP diff --git a/website/i18n/en.json b/website/i18n/en.json index 9de85a69ad68..f67380992222 100644 --- a/website/i18n/en.json +++ b/website/i18n/en.json @@ -179,6 +179,9 @@ "development/extensions-core/datasketches-tuple": { "title": "DataSketches Tuple Sketch module" }, + "development/extensions-core/druid-aws-rds": { + "title": "Druid AWS RDS Module" + }, "development/extensions-core/druid-basic-security": { "title": "Basic Security" }, @@ -214,6 +217,9 @@ "title": "Amazon Kinesis ingestion", "sidebar_label": "Amazon Kinesis" }, + "development/extensions-core/druid-kubernetes": { + "title": "Kubernetes" + }, "development/extensions-core/lookups-cached-global": { "title": "Globally Cached Lookups" }, @@ -324,6 +330,9 @@ "operations/dump-segment": { "title": "dump-segment tool" }, + "operations/dynamic-config-provider": { + "title": "Dynamic Config Providers" + }, "operations/export-metadata": { "title": "Export Metadata Tool" }, From 3cee6fc4b211eef9dbd2bd2e8e76f2fc840df74f Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 6 Jan 2021 14:04:58 -0800 Subject: [PATCH 25/25] remove inadvertant website file change --- website/i18n/en.json | 9 --------- 1 file changed, 9 deletions(-) diff --git a/website/i18n/en.json b/website/i18n/en.json index f67380992222..9de85a69ad68 100644 --- a/website/i18n/en.json +++ b/website/i18n/en.json @@ -179,9 +179,6 @@ "development/extensions-core/datasketches-tuple": { "title": "DataSketches Tuple Sketch module" }, - "development/extensions-core/druid-aws-rds": { - "title": "Druid AWS RDS Module" - }, "development/extensions-core/druid-basic-security": { "title": "Basic Security" }, @@ -217,9 +214,6 @@ "title": "Amazon Kinesis ingestion", "sidebar_label": "Amazon Kinesis" }, - "development/extensions-core/druid-kubernetes": { - "title": "Kubernetes" - }, "development/extensions-core/lookups-cached-global": { "title": "Globally Cached Lookups" }, @@ -330,9 +324,6 @@ "operations/dump-segment": { "title": "dump-segment tool" }, - "operations/dynamic-config-provider": { - "title": "Dynamic Config Providers" - }, "operations/export-metadata": { "title": "Export Metadata Tool" },