diff --git a/cloud/aws-common/pom.xml b/cloud/aws-common/pom.xml
index ddedf0b9bb73..c0f45fdd4aca 100644
--- a/cloud/aws-common/pom.xml
+++ b/cloud/aws-common/pom.xml
@@ -46,6 +46,10 @@
com.amazonaws
aws-java-sdk-s3
+
+ com.amazonaws
+ aws-java-sdk-sts
+
org.checkerframework
checker-qual
diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml
index 48e5b3d223d1..9fe84ae6ad08 100644
--- a/extensions-core/s3-extensions/pom.xml
+++ b/extensions-core/s3-extensions/pom.xml
@@ -114,7 +114,7 @@
com.amazonaws
aws-java-sdk-sts
- ${aws.sdk.version}
+ provided
diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java
index 20688ab07204..5989565f21d6 100644
--- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java
+++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java
@@ -105,22 +105,31 @@ public S3InputSource(
this.s3ClientSupplier = Suppliers.memoize(
() -> {
if (s3ClientBuilder != null && s3InputSourceConfig != null) {
- if (s3InputSourceConfig.isCredentialsConfigured()) {
- if (s3InputSourceConfig.getAssumeRoleArn() == null) {
- s3ClientBuilder
- .getAmazonS3ClientBuilder()
- .withCredentials(createStaticCredentialsProvider(s3InputSourceConfig));
- } else {
- applyAssumeRole(
- s3ClientBuilder,
- s3InputSourceConfig,
- createStaticCredentialsProvider(s3InputSourceConfig)
- );
- }
- } else {
+ // Each of these if statements will manipulate s3ClientBuilder if the condition is fulfilled.
+
+ // If both static key-pair and assume role ARN are defined, use the static key-pair to assume role.
+ if (s3InputSourceConfig.isCredentialsConfigured() && s3InputSourceConfig.isAssumeRoleArnConfigured()) {
+ applyAssumeRole(
+ s3ClientBuilder,
+ s3InputSourceConfig,
+ createStaticCredentialsProvider(s3InputSourceConfig)
+ );
+
+ // If only static key-pair is defined, build the S3 client with the static key-pair
+ } else if (s3InputSourceConfig.isCredentialsConfigured() && !s3InputSourceConfig.isAssumeRoleArnConfigured()) {
+ s3ClientBuilder.getAmazonS3ClientBuilder().withCredentials(createStaticCredentialsProvider(s3InputSourceConfig));
+
+ // If assume role ARN is defined, static key-pair is undefined, and WebIdentityToken file from the environment variable is undefined.
+ } else if (s3InputSourceConfig.isAssumeRoleArnConfigured() && !s3InputSourceConfig.isCredentialsConfigured() && !s3InputSourceConfig.isWebIdentityTokenEnvConfigured()) {
applyAssumeRole(s3ClientBuilder, s3InputSourceConfig, awsCredentialsProvider);
}
+
+ // Actually build the ServerSideEncryptingAmazonS3 object.
+ return s3ClientBuilder.build();
+
+ } else if (s3ClientBuilder != null) {
return s3ClientBuilder.build();
+
} else {
return s3Client;
}
@@ -166,15 +175,21 @@ private void applyAssumeRole(
AWSCredentialsProvider awsCredentialsProvider
)
{
- String assumeRoleArn = s3InputSourceConfig.getAssumeRoleArn();
- if (assumeRoleArn != null) {
+ // Do not run if WebIdentityToken file and assumeRole ARN are detected from the environment variable,
+ // we want the default s3ClientBuilder behavior for ServiceAccount + eks.amazonaws.com/role-arn annotation to work.
+ if (s3InputSourceConfig.isWebIdentityTokenEnvConfigured() && s3InputSourceConfig.isAssumeRoleArnEnvConfigured()) {
+ return;
+ }
+
+ if (s3InputSourceConfig.isAssumeRoleArnConfigured()) {
String roleSessionName = StringUtils.format("druid-s3-input-source-%s", UUID.randomUUID().toString());
AWSSecurityTokenService securityTokenService = AWSSecurityTokenServiceClientBuilder.standard()
.withCredentials(awsCredentialsProvider)
.build();
+
STSAssumeRoleSessionCredentialsProvider.Builder roleCredentialsProviderBuilder;
roleCredentialsProviderBuilder = new STSAssumeRoleSessionCredentialsProvider
- .Builder(assumeRoleArn, roleSessionName).withStsClient(securityTokenService);
+ .Builder(s3InputSourceConfig.getAssumeRoleArn(), roleSessionName).withStsClient(securityTokenService);
if (s3InputSourceConfig.getAssumeRoleExternalId() != null) {
roleCredentialsProviderBuilder.withExternalId(s3InputSourceConfig.getAssumeRoleExternalId());
diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java
index 6b837e703a07..94aed87ba199 100644
--- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java
+++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java
@@ -19,6 +19,7 @@
package org.apache.druid.data.input.s3;
+import com.amazonaws.SDKGlobalConfiguration;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -90,6 +91,28 @@ public boolean isCredentialsConfigured()
secretAccessKey != null;
}
+ @JsonIgnore
+ public boolean isAssumeRoleArnConfigured()
+ {
+ return assumeRoleArn != null && !assumeRoleArn.isEmpty();
+ }
+
+ @JsonIgnore
+ public boolean isAssumeRoleArnEnvConfigured()
+ {
+ String conf = null;
+ conf = System.getenv(SDKGlobalConfiguration.AWS_ROLE_ARN_ENV_VAR);
+ return conf != null && !conf.isEmpty();
+ }
+
+ @JsonIgnore
+ public boolean isWebIdentityTokenEnvConfigured()
+ {
+ String conf = null;
+ conf = System.getenv(SDKGlobalConfiguration.AWS_WEB_IDENTITY_ENV_VAR);
+ return conf != null && !conf.isEmpty();
+ }
+
@Override
public String toString()
{
diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java
index ff8f9682ff28..3aef08e70d93 100644
--- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java
+++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java
@@ -21,6 +21,7 @@
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.SDKGlobalConfiguration;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
@@ -223,6 +224,61 @@ public void testSerdeWithCloudConfigPropertiesWithKeyAndSecret() throws Exceptio
EasyMock.verify(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
}
+ @Test
+ public void testSerdeWithCloudConfigPropertiesWithIdentityFileConfigured() throws Exception
+ {
+ EasyMock.reset(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
+ EasyMock.expect(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER.getAmazonS3ClientBuilder())
+ .andStubReturn(AMAZON_S3_CLIENT_BUILDER);
+ EasyMock.expect(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER.build())
+ .andReturn(SERVICE);
+ EasyMock.replay(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
+
+ // Mock setting the AWS's environment variables.
+ String oldRoleARN = System.getProperty(SDKGlobalConfiguration.AWS_ROLE_ARN_ENV_VAR);
+ if(oldRoleARN.equals("")) {
+ System.setProperty(SDKGlobalConfiguration.AWS_ROLE_ARN_ENV_VAR, "mockROLEARN");
+ }
+
+ String oldIdentityFile = System.getProperty(SDKGlobalConfiguration.AWS_WEB_IDENTITY_ENV_VAR);
+ if(oldIdentityFile.equals("")) {
+ System.setProperty(SDKGlobalConfiguration.AWS_WEB_IDENTITY_ENV_VAR, "mockIdentityFile");
+ }
+
+ S3InputSourceConfig cloudConfigProperties = new S3InputSourceConfig(
+ null,
+ null,
+ null,
+ null
+ );
+
+ final S3InputSource withPrefixes = new S3InputSource(
+ SERVICE,
+ SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER,
+ INPUT_DATA_CONFIG,
+ null,
+ null,
+ EXPECTED_LOCATION,
+ cloudConfigProperties
+ );
+ final S3InputSource serdeWithPrefixes =
+ MAPPER.readValue(MAPPER.writeValueAsString(withPrefixes), S3InputSource.class);
+
+ // This is to force the s3ClientSupplier to initialize the ServerSideEncryptingAmazonS3
+ serdeWithPrefixes.createEntity(new CloudObjectLocation("bucket", "path"));
+
+ Assert.assertEquals(withPrefixes, serdeWithPrefixes);
+
+ if(oldRoleARN.equals("")) {
+ System.clearProperty(SDKGlobalConfiguration.AWS_ROLE_ARN_ENV_VAR);
+ }
+ if(oldIdentityFile.equals("")) {
+ System.clearProperty(SDKGlobalConfiguration.AWS_WEB_IDENTITY_ENV_VAR);
+ }
+
+ EasyMock.verify(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER);
+ }
+
@Test
public void testS3InputSourceUseDefaultPasswordWhenCloudConfigPropertiesWithoutCrediential()
{