From 43023000edd907a6ff0d0fc0e90a49b568c80734 Mon Sep 17 00:00:00 2001 From: egor-ryashin Date: Mon, 15 Mar 2021 20:57:57 +0300 Subject: [PATCH 1/9] feature s3 assume role --- docs/ingestion/native-batch.md | 43 ++++++++++ extensions-core/s3-extensions/pom.xml | 2 +- .../druid/data/input/s3/S3InputSource.java | 83 +++++++++++++++++-- .../data/input/s3/S3InputSourceConfig.java | 73 +++++++++++++--- 4 files changed, 181 insertions(+), 20 deletions(-) diff --git a/docs/ingestion/native-batch.md b/docs/ingestion/native-batch.md index 5032ed1a518c..2736f8066f06 100644 --- a/docs/ingestion/native-batch.md +++ b/docs/ingestion/native-batch.md @@ -862,6 +862,47 @@ Sample specs: ... ``` +```json +... + "ioConfig": { + "type": "index_parallel", + "inputSource": { + "type": "s3", + "uris": ["s3://foo/bar/file.json", "s3://bar/foo/file2.json"], + "properties": { + "accessKeyId": "KLJ78979SDFdS2", + "secretAccessKey": "KLS89s98sKJHKJKJH8721lljkd" + } + }, + "inputFormat": { + "type": "json" + }, + ... + }, +... +``` + +```json +... + "ioConfig": { + "type": "index_parallel", + "inputSource": { + "type": "s3", + "uris": ["s3://foo/bar/file.json", "s3://bar/foo/file2.json"], + "properties": { + "accessKeyId": "KLJ78979SDFdS2", + "secretAccessKey": "KLS89s98sKJHKJKJH8721lljkd", + "awsAssumedRoleArn": "arn:aws:iam::2981002874992:role/role-s3" + } + }, + "inputFormat": { + "type": "json" + }, + ... + }, +... +``` + |property|description|default|required?| |--------|-----------|-------|---------| |type|This should be `s3`.|None|yes| @@ -885,6 +926,8 @@ Properties Object: |--------|-----------|-------|---------| |accessKeyId|The [Password Provider](../operations/password-provider.md) or plain text string of this S3 InputSource's access key|None|yes if secretAccessKey is given| |secretAccessKey|The [Password Provider](../operations/password-provider.md) or plain text string of this S3 InputSource's secret key|None|yes if accessKeyId is given| +|awsAssumedRoleArn|AWS ARN of the role to assume [see](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html). **awsAssumedRoleArn** can be used either with the ingestion spec AWS credentials or with the default S3 credentials|None|no| +|awsExternalId|A unique identifier that might be required when you assume a role in another account [see](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)|None|no| **Note :** *If accessKeyId and secretAccessKey are not given, the default [S3 credentials provider chain](../development/extensions-core/s3.md#s3-authentication-methods) is used.* diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 30c63f672bbf..d32ae7a641d3 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -116,7 +116,7 @@ com.amazonaws aws-java-sdk-sts provided - + org.apache.druid diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java index 22c10680f809..a84c938c3f0b 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java @@ -19,9 +19,13 @@ package org.apache.druid.data.input.s3; +import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.securitytoken.AWSSecurityTokenService; +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -35,6 +39,7 @@ import org.apache.druid.data.input.impl.CloudObjectInputSource; import org.apache.druid.data.input.impl.CloudObjectLocation; import org.apache.druid.data.input.impl.SplittableInputSource; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.storage.s3.S3InputDataConfig; import org.apache.druid.storage.s3.S3StorageDruidModule; import org.apache.druid.storage.s3.S3Utils; @@ -47,6 +52,7 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -59,6 +65,7 @@ public class S3InputSource extends CloudObjectInputSource @JsonProperty("properties") private final S3InputSourceConfig s3InputSourceConfig; private final S3InputDataConfig inputDataConfig; + private final AWSCredentialsProvider awsCredentialsProvider; /** * Constructor for S3InputSource @@ -84,7 +91,8 @@ public S3InputSource( @JsonProperty("uris") @Nullable List uris, @JsonProperty("prefixes") @Nullable List prefixes, @JsonProperty("objects") @Nullable List objects, - @JsonProperty("properties") @Nullable S3InputSourceConfig s3InputSourceConfig + @JsonProperty("properties") @Nullable S3InputSourceConfig s3InputSourceConfig, + @JacksonInject AWSCredentialsProvider awsCredentialsProvider ) { super(S3StorageDruidModule.SCHEME, uris, prefixes, objects); @@ -95,13 +103,19 @@ public S3InputSource( () -> { if (s3ClientBuilder != null && s3InputSourceConfig != null) { if (s3InputSourceConfig.isCredentialsConfigured()) { - AWSStaticCredentialsProvider credentials = new AWSStaticCredentialsProvider( - new BasicAWSCredentials( - s3InputSourceConfig.getAccessKeyId().getPassword(), - s3InputSourceConfig.getSecretAccessKey().getPassword() - ) - ); - s3ClientBuilder.getAmazonS3ClientBuilder().withCredentials(credentials); + if (s3InputSourceConfig.getAwsAssumedRoleArn() == null) { + s3ClientBuilder + .getAmazonS3ClientBuilder() + .withCredentials(createStaticCredentialsProvider(s3InputSourceConfig)); + } else { + applyAssumeRole( + s3ClientBuilder, + s3InputSourceConfig, + createStaticCredentialsProvider(s3InputSourceConfig) + ); + } + } else { + applyAssumeRole(s3ClientBuilder, s3InputSourceConfig, awsCredentialsProvider); } return s3ClientBuilder.build(); } else { @@ -109,6 +123,56 @@ public S3InputSource( } } ); + this.awsCredentialsProvider = awsCredentialsProvider; + } + + // for testing + public S3InputSource( + ServerSideEncryptingAmazonS3 s3Client, + ServerSideEncryptingAmazonS3.Builder s3ClientBuilder, + S3InputDataConfig inputDataConfig, + List uris, + List prefixes, + List objects, + S3InputSourceConfig s3InputSourceConfig + ) + { + this(s3Client, s3ClientBuilder, inputDataConfig, uris, prefixes, objects, s3InputSourceConfig, null); + } + + private void applyAssumeRole( + ServerSideEncryptingAmazonS3.Builder s3ClientBuilder, + S3InputSourceConfig s3InputSourceConfig, + AWSCredentialsProvider awsCredentialsProvider + ) + { + String awsAssumedRoleArn = s3InputSourceConfig.getAwsAssumedRoleArn(); + if (awsAssumedRoleArn != null) { + String roleSessionName = StringUtils.format("druid-s3-indexing-%s", UUID.randomUUID().toString()); + AWSSecurityTokenService securityTokenService = AWSSecurityTokenServiceClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .build(); + STSAssumeRoleSessionCredentialsProvider.Builder roleCredentialsProviderBuilder; + roleCredentialsProviderBuilder = new STSAssumeRoleSessionCredentialsProvider + .Builder(awsAssumedRoleArn, roleSessionName).withStsClient(securityTokenService); + + if (s3InputSourceConfig.getAwsAssumeRoleExternalId() != null) { + roleCredentialsProviderBuilder.withExternalId(s3InputSourceConfig.getAwsAssumeRoleExternalId()); + } + + s3ClientBuilder.getAmazonS3ClientBuilder().withCredentials(roleCredentialsProviderBuilder.build()); + } + } + + @Nonnull + private AWSStaticCredentialsProvider createStaticCredentialsProvider(S3InputSourceConfig s3InputSourceConfig) + { + return new AWSStaticCredentialsProvider( + new BasicAWSCredentials( + s3InputSourceConfig.getAccessKeyId().getPassword(), + s3InputSourceConfig.getSecretAccessKey().getPassword() + ) + ); } @Nullable @@ -149,7 +213,8 @@ public SplittableInputSource> withSplit(InputSplit Date: Mon, 15 Mar 2021 22:47:57 +0300 Subject: [PATCH 4/9] feature s3 assume role --- .../druid/data/input/s3/S3InputSourceConfig.java | 16 ---------------- .../druid/data/input/s3/S3InputSourceTest.java | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java index 9dad96561a62..5560efbc319a 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java @@ -60,22 +60,6 @@ public S3InputSourceConfig( } } - public S3InputSourceConfig( - @JsonProperty("accessKeyId") @Nullable PasswordProvider accessKeyId, - @JsonProperty("secretAccessKey") @Nullable PasswordProvider secretAccessKey) - { - if (accessKeyId != null || secretAccessKey != null) { - this.accessKeyId = Preconditions.checkNotNull( - accessKeyId, - "accessKeyId cannot be null if secretAccessKey is given" - ); - this.secretAccessKey = Preconditions.checkNotNull( - secretAccessKey, - "secretAccessKey cannot be null if accessKeyId is given" - ); - } - } - @Nullable public String getAwsAssumedRoleArn() { diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java index 9efaf4fd09c3..b2f40b0c6343 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java @@ -119,7 +119,7 @@ public class S3InputSourceTest extends InitializedNullHandlingTest ); private static final S3InputSourceConfig CLOUD_CONFIG_PROPERTIES = new S3InputSourceConfig( - new DefaultPasswordProvider("myKey"), new DefaultPasswordProvider("mySecret")); + new DefaultPasswordProvider("myKey"), new DefaultPasswordProvider("mySecret"), null, null); private static final List EXPECTED_LOCATION = ImmutableList.of(new CloudObjectLocation("foo", "bar/file.csv")); From 05c60be824e2d4a3fd3eaf298de44be544ece4cf Mon Sep 17 00:00:00 2001 From: egor-ryashin Date: Mon, 15 Mar 2021 23:03:37 +0300 Subject: [PATCH 5/9] feature s3 assume role --- docs/ingestion/native-batch.md | 6 +-- .../druid/data/input/s3/S3InputSource.java | 12 ++--- .../data/input/s3/S3InputSourceConfig.java | 51 +++++++------------ 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/docs/ingestion/native-batch.md b/docs/ingestion/native-batch.md index 2736f8066f06..e6d504e4984c 100644 --- a/docs/ingestion/native-batch.md +++ b/docs/ingestion/native-batch.md @@ -892,7 +892,7 @@ Sample specs: "properties": { "accessKeyId": "KLJ78979SDFdS2", "secretAccessKey": "KLS89s98sKJHKJKJH8721lljkd", - "awsAssumedRoleArn": "arn:aws:iam::2981002874992:role/role-s3" + "assumeRoleArn": "arn:aws:iam::2981002874992:role/role-s3" } }, "inputFormat": { @@ -926,8 +926,8 @@ Properties Object: |--------|-----------|-------|---------| |accessKeyId|The [Password Provider](../operations/password-provider.md) or plain text string of this S3 InputSource's access key|None|yes if secretAccessKey is given| |secretAccessKey|The [Password Provider](../operations/password-provider.md) or plain text string of this S3 InputSource's secret key|None|yes if accessKeyId is given| -|awsAssumedRoleArn|AWS ARN of the role to assume [see](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html). **awsAssumedRoleArn** can be used either with the ingestion spec AWS credentials or with the default S3 credentials|None|no| -|awsExternalId|A unique identifier that might be required when you assume a role in another account [see](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)|None|no| +|assumeRoleArn|AWS ARN of the role to assume [see](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html). **assumeRoleArn** can be used either with the ingestion spec AWS credentials or with the default S3 credentials|None|no| +|assumeRoleExternalId|A unique identifier that might be required when you assume a role in another account [see](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)|None|no| **Note :** *If accessKeyId and secretAccessKey are not given, the default [S3 credentials provider chain](../development/extensions-core/s3.md#s3-authentication-methods) is used.* diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java index 050ce65f0ac8..24fd99cf5202 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java @@ -104,7 +104,7 @@ public S3InputSource( () -> { if (s3ClientBuilder != null && s3InputSourceConfig != null) { if (s3InputSourceConfig.isCredentialsConfigured()) { - if (s3InputSourceConfig.getAwsAssumedRoleArn() == null) { + if (s3InputSourceConfig.getAssumeRoleArn() == null) { s3ClientBuilder .getAmazonS3ClientBuilder() .withCredentials(createStaticCredentialsProvider(s3InputSourceConfig)); @@ -147,18 +147,18 @@ private void applyAssumeRole( AWSCredentialsProvider awsCredentialsProvider ) { - String awsAssumedRoleArn = s3InputSourceConfig.getAwsAssumedRoleArn(); - if (awsAssumedRoleArn != null) { + String assumeRoleArn = s3InputSourceConfig.getAssumeRoleArn(); + if (assumeRoleArn != null) { String roleSessionName = StringUtils.format("druid-s3-input-source-%s", UUID.randomUUID().toString()); AWSSecurityTokenService securityTokenService = AWSSecurityTokenServiceClientBuilder.standard() .withCredentials(awsCredentialsProvider) .build(); STSAssumeRoleSessionCredentialsProvider.Builder roleCredentialsProviderBuilder; roleCredentialsProviderBuilder = new STSAssumeRoleSessionCredentialsProvider - .Builder(awsAssumedRoleArn, roleSessionName).withStsClient(securityTokenService); + .Builder(assumeRoleArn, roleSessionName).withStsClient(securityTokenService); - if (s3InputSourceConfig.getAwsAssumeRoleExternalId() != null) { - roleCredentialsProviderBuilder.withExternalId(s3InputSourceConfig.getAwsAssumeRoleExternalId()); + if (s3InputSourceConfig.getAssumeRoleExternalId() != null) { + roleCredentialsProviderBuilder.withExternalId(s3InputSourceConfig.getAssumeRoleExternalId()); } s3ClientBuilder.getAmazonS3ClientBuilder().withCredentials(roleCredentialsProviderBuilder.build()); diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java index 5560efbc319a..1c0b2748d12e 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java @@ -26,6 +26,7 @@ import org.apache.druid.metadata.PasswordProvider; import javax.annotation.Nullable; +import java.util.Objects; /** * Contains properties for s3 input source. @@ -35,10 +36,10 @@ public class S3InputSourceConfig { @Nullable @JsonProperty - private String awsAssumedRoleArn; + private String assumeRoleArn; @Nullable @JsonProperty - private String externalId; + private String assumeRoleExternalId; @JsonProperty private PasswordProvider accessKeyId; @JsonProperty @@ -48,12 +49,12 @@ public class S3InputSourceConfig public S3InputSourceConfig( @JsonProperty("accessKeyId") @Nullable PasswordProvider accessKeyId, @JsonProperty("secretAccessKey") @Nullable PasswordProvider secretAccessKey, - @JsonProperty("awsAssumedRoleArn") @Nullable String awsAssumedRoleArn, - @JsonProperty("awsExternalId") @Nullable String exteranlId + @JsonProperty("assumeRoleArn") @Nullable String assumeRoleArn, + @JsonProperty("assumeRoleExternalId") @Nullable String assumeRoleExternalId ) { - this.awsAssumedRoleArn = awsAssumedRoleArn; - this.externalId = exteranlId; + this.assumeRoleArn = assumeRoleArn; + this.assumeRoleExternalId = assumeRoleExternalId; if (accessKeyId != null || secretAccessKey != null) { this.accessKeyId = Preconditions.checkNotNull(accessKeyId, "accessKeyId cannot be null if secretAccessKey is given"); this.secretAccessKey = Preconditions.checkNotNull(secretAccessKey, "secretAccessKey cannot be null if accessKeyId is given"); @@ -61,15 +62,15 @@ public S3InputSourceConfig( } @Nullable - public String getAwsAssumedRoleArn() + public String getAssumeRoleArn() { - return awsAssumedRoleArn; + return assumeRoleArn; } @Nullable - public String getAwsAssumeRoleExternalId() + public String getAssumeRoleExternalId() { - return externalId; + return assumeRoleExternalId; } public PasswordProvider getAccessKeyId() @@ -93,10 +94,10 @@ public boolean isCredentialsConfigured() public String toString() { return "S3InputSourceConfig{" + - "awsAssumedRoleArn='" + awsAssumedRoleArn + '\'' + - ", externalId='" + externalId + '\'' + - ", accessKeyId=" + accessKeyId + + "accessKeyId=" + accessKeyId + ", secretAccessKey=" + secretAccessKey + + ", assumeRoleArn=" + assumeRoleArn + + ", assumeRoleExternalId=" + assumeRoleExternalId + '}'; } @@ -109,30 +110,16 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - S3InputSourceConfig that = (S3InputSourceConfig) o; - - if (awsAssumedRoleArn != null - ? !awsAssumedRoleArn.equals(that.awsAssumedRoleArn) - : that.awsAssumedRoleArn != null) { - return false; - } - if (externalId != null ? !externalId.equals(that.externalId) : that.externalId != null) { - return false; - } - if (accessKeyId != null ? !accessKeyId.equals(that.accessKeyId) : that.accessKeyId != null) { - return false; - } - return secretAccessKey != null ? secretAccessKey.equals(that.secretAccessKey) : that.secretAccessKey == null; + return Objects.equals(accessKeyId, that.accessKeyId) && + Objects.equals(secretAccessKey, that.secretAccessKey) && + Objects.equals(assumeRoleArn, that.assumeRoleArn) && + Objects.equals(assumeRoleExternalId, that.assumeRoleExternalId); } @Override public int hashCode() { - int result = awsAssumedRoleArn != null ? awsAssumedRoleArn.hashCode() : 0; - result = 31 * result + (externalId != null ? externalId.hashCode() : 0); - result = 31 * result + (accessKeyId != null ? accessKeyId.hashCode() : 0); - result = 31 * result + (secretAccessKey != null ? secretAccessKey.hashCode() : 0); - return result; + return Objects.hash(accessKeyId, secretAccessKey, assumeRoleArn, assumeRoleExternalId); } } From 4ebf4fc642d21e8e7b6888df2588a4a81f47624a Mon Sep 17 00:00:00 2001 From: egor-ryashin Date: Sat, 10 Apr 2021 14:12:24 +0300 Subject: [PATCH 6/9] feature s3 assume role --- .../org/apache/druid/data/input/s3/S3InputSourceConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java index 1c0b2748d12e..6b837e703a07 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java @@ -34,10 +34,8 @@ */ public class S3InputSourceConfig { - @Nullable @JsonProperty private String assumeRoleArn; - @Nullable @JsonProperty private String assumeRoleExternalId; @JsonProperty @@ -73,11 +71,13 @@ public String getAssumeRoleExternalId() return assumeRoleExternalId; } + @Nullable public PasswordProvider getAccessKeyId() { return accessKeyId; } + @Nullable public PasswordProvider getSecretAccessKey() { return secretAccessKey; From a599118e145788dd9194b543a81bf70719ac48b0 Mon Sep 17 00:00:00 2001 From: egor-ryashin Date: Fri, 30 Apr 2021 14:37:15 +0300 Subject: [PATCH 7/9] tests fix --- .../data/input/s3/S3InputSourceTest.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java index b2f40b0c6343..6512dbcfbd1f 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java @@ -19,6 +19,7 @@ package org.apache.druid.data.input.s3; +import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @@ -40,6 +41,7 @@ import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Provides; +import org.apache.druid.common.aws.AWSCredentialsUtils; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.InputSourceReader; @@ -220,6 +222,7 @@ public void testS3InputSourceUseDefaultPasswordWhenCloudConfigPropertiesWithoutC { S3InputSourceConfig mockConfigPropertiesWithoutKeyAndSecret = EasyMock.createMock(S3InputSourceConfig.class); EasyMock.reset(mockConfigPropertiesWithoutKeyAndSecret); + EasyMock.expect(mockConfigPropertiesWithoutKeyAndSecret.getAssumeRoleArn()).andStubReturn(null); EasyMock.expect(mockConfigPropertiesWithoutKeyAndSecret.isCredentialsConfigured()) .andStubReturn(false); EasyMock.replay(mockConfigPropertiesWithoutKeyAndSecret); @@ -278,6 +281,7 @@ public void testSerdeS3ClientLazyInitializedWithoutCrediential() throws Exceptio null, EXPECTED_LOCATION, null + ); final S3InputSource serdeWithPrefixes = MAPPER.readValue(MAPPER.writeValueAsString(withPrefixes), S3InputSource.class); @@ -663,7 +667,25 @@ public static ObjectMapper createS3ObjectMapper() DruidModule baseModule = new TestS3Module(); final Injector injector = Guice.createInjector( new ObjectMapperModule(), - baseModule + baseModule, + new DruidModule() + { + @Provides + public AWSCredentialsProvider getAWSCredentialsProvider() + { + return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(null); + } + + @Override public List getJacksonModules() + { + return Collections.emptyList(); + } + + @Override public void configure(Binder binder) + { + + } + } ); final ObjectMapper baseMapper = injector.getInstance(ObjectMapper.class); From 0bcd4caa54b30440ceb056f2986b7a4bc480996a Mon Sep 17 00:00:00 2001 From: egor-ryashin Date: Sat, 1 May 2021 20:46:33 +0300 Subject: [PATCH 8/9] spelling fix --- website/.spelling | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/.spelling b/website/.spelling index d717fbf9ecf7..6d2ae63828db 100644 --- a/website/.spelling +++ b/website/.spelling @@ -24,6 +24,7 @@ ACL APIs AvroStorage +ARN AWS AWS_CONTAINER_CREDENTIALS_RELATIVE_URI AWS_CONTAINER_CREDENTIALS_FULL_URI @@ -194,6 +195,8 @@ aggregator aggregators ambari analytics +assumeRoleArn +assumeRoleExternalId async authorizer authorizers From 5cd40b78e576f53c842e7f375d15d5b692bccc27 Mon Sep 17 00:00:00 2001 From: egor-ryashin Date: Thu, 3 Jun 2021 19:29:10 +0300 Subject: [PATCH 9/9] sts fix --- extensions-core/s3-extensions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index d32ae7a641d3..1a2d2fc39830 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -115,7 +115,7 @@ com.amazonaws aws-java-sdk-sts - provided + ${aws.sdk.version}