diff --git a/cloud/aws-common/pom.xml b/cloud/aws-common/pom.xml index ddedf0b9bb73..c0f45fdd4aca 100644 --- a/cloud/aws-common/pom.xml +++ b/cloud/aws-common/pom.xml @@ -46,6 +46,10 @@ com.amazonaws aws-java-sdk-s3 + + com.amazonaws + aws-java-sdk-sts + org.checkerframework checker-qual diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 48e5b3d223d1..9fe84ae6ad08 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -114,7 +114,7 @@ com.amazonaws aws-java-sdk-sts - ${aws.sdk.version} + provided diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java index 20688ab07204..5989565f21d6 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java @@ -105,22 +105,31 @@ public S3InputSource( this.s3ClientSupplier = Suppliers.memoize( () -> { if (s3ClientBuilder != null && s3InputSourceConfig != null) { - if (s3InputSourceConfig.isCredentialsConfigured()) { - if (s3InputSourceConfig.getAssumeRoleArn() == null) { - s3ClientBuilder - .getAmazonS3ClientBuilder() - .withCredentials(createStaticCredentialsProvider(s3InputSourceConfig)); - } else { - applyAssumeRole( - s3ClientBuilder, - s3InputSourceConfig, - createStaticCredentialsProvider(s3InputSourceConfig) - ); - } - } else { + // Each of these if statements will manipulate s3ClientBuilder if the condition is fulfilled. + + // If both static key-pair and assume role ARN are defined, use the static key-pair to assume role. + if (s3InputSourceConfig.isCredentialsConfigured() && s3InputSourceConfig.isAssumeRoleArnConfigured()) { + applyAssumeRole( + s3ClientBuilder, + s3InputSourceConfig, + createStaticCredentialsProvider(s3InputSourceConfig) + ); + + // If only static key-pair is defined, build the S3 client with the static key-pair + } else if (s3InputSourceConfig.isCredentialsConfigured() && !s3InputSourceConfig.isAssumeRoleArnConfigured()) { + s3ClientBuilder.getAmazonS3ClientBuilder().withCredentials(createStaticCredentialsProvider(s3InputSourceConfig)); + + // If assume role ARN is defined, static key-pair is undefined, and WebIdentityToken file from the environment variable is undefined. + } else if (s3InputSourceConfig.isAssumeRoleArnConfigured() && !s3InputSourceConfig.isCredentialsConfigured() && !s3InputSourceConfig.isWebIdentityTokenEnvConfigured()) { applyAssumeRole(s3ClientBuilder, s3InputSourceConfig, awsCredentialsProvider); } + + // Actually build the ServerSideEncryptingAmazonS3 object. + return s3ClientBuilder.build(); + + } else if (s3ClientBuilder != null) { return s3ClientBuilder.build(); + } else { return s3Client; } @@ -166,15 +175,21 @@ private void applyAssumeRole( AWSCredentialsProvider awsCredentialsProvider ) { - String assumeRoleArn = s3InputSourceConfig.getAssumeRoleArn(); - if (assumeRoleArn != null) { + // Do not run if WebIdentityToken file and assumeRole ARN are detected from the environment variable, + // we want the default s3ClientBuilder behavior for ServiceAccount + eks.amazonaws.com/role-arn annotation to work. + if (s3InputSourceConfig.isWebIdentityTokenEnvConfigured() && s3InputSourceConfig.isAssumeRoleArnEnvConfigured()) { + return; + } + + if (s3InputSourceConfig.isAssumeRoleArnConfigured()) { String roleSessionName = StringUtils.format("druid-s3-input-source-%s", UUID.randomUUID().toString()); AWSSecurityTokenService securityTokenService = AWSSecurityTokenServiceClientBuilder.standard() .withCredentials(awsCredentialsProvider) .build(); + STSAssumeRoleSessionCredentialsProvider.Builder roleCredentialsProviderBuilder; roleCredentialsProviderBuilder = new STSAssumeRoleSessionCredentialsProvider - .Builder(assumeRoleArn, roleSessionName).withStsClient(securityTokenService); + .Builder(s3InputSourceConfig.getAssumeRoleArn(), roleSessionName).withStsClient(securityTokenService); if (s3InputSourceConfig.getAssumeRoleExternalId() != null) { roleCredentialsProviderBuilder.withExternalId(s3InputSourceConfig.getAssumeRoleExternalId()); diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java index 6b837e703a07..94aed87ba199 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java @@ -19,6 +19,7 @@ package org.apache.druid.data.input.s3; +import com.amazonaws.SDKGlobalConfiguration; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -90,6 +91,28 @@ public boolean isCredentialsConfigured() secretAccessKey != null; } + @JsonIgnore + public boolean isAssumeRoleArnConfigured() + { + return assumeRoleArn != null && !assumeRoleArn.isEmpty(); + } + + @JsonIgnore + public boolean isAssumeRoleArnEnvConfigured() + { + String conf = null; + conf = System.getenv(SDKGlobalConfiguration.AWS_ROLE_ARN_ENV_VAR); + return conf != null && !conf.isEmpty(); + } + + @JsonIgnore + public boolean isWebIdentityTokenEnvConfigured() + { + String conf = null; + conf = System.getenv(SDKGlobalConfiguration.AWS_WEB_IDENTITY_ENV_VAR); + return conf != null && !conf.isEmpty(); + } + @Override public String toString() { diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java index ff8f9682ff28..3aef08e70d93 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java @@ -21,6 +21,7 @@ import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.SDKGlobalConfiguration; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @@ -223,6 +224,61 @@ public void testSerdeWithCloudConfigPropertiesWithKeyAndSecret() throws Exceptio EasyMock.verify(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER); } + @Test + public void testSerdeWithCloudConfigPropertiesWithIdentityFileConfigured() throws Exception + { + EasyMock.reset(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER); + EasyMock.expect(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER.getAmazonS3ClientBuilder()) + .andStubReturn(AMAZON_S3_CLIENT_BUILDER); + EasyMock.expect(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER.build()) + .andReturn(SERVICE); + EasyMock.replay(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER); + + // Mock setting the AWS's environment variables. + String oldRoleARN = System.getProperty(SDKGlobalConfiguration.AWS_ROLE_ARN_ENV_VAR); + if(oldRoleARN.equals("")) { + System.setProperty(SDKGlobalConfiguration.AWS_ROLE_ARN_ENV_VAR, "mockROLEARN"); + } + + String oldIdentityFile = System.getProperty(SDKGlobalConfiguration.AWS_WEB_IDENTITY_ENV_VAR); + if(oldIdentityFile.equals("")) { + System.setProperty(SDKGlobalConfiguration.AWS_WEB_IDENTITY_ENV_VAR, "mockIdentityFile"); + } + + S3InputSourceConfig cloudConfigProperties = new S3InputSourceConfig( + null, + null, + null, + null + ); + + final S3InputSource withPrefixes = new S3InputSource( + SERVICE, + SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER, + INPUT_DATA_CONFIG, + null, + null, + EXPECTED_LOCATION, + cloudConfigProperties + ); + final S3InputSource serdeWithPrefixes = + MAPPER.readValue(MAPPER.writeValueAsString(withPrefixes), S3InputSource.class); + + // This is to force the s3ClientSupplier to initialize the ServerSideEncryptingAmazonS3 + serdeWithPrefixes.createEntity(new CloudObjectLocation("bucket", "path")); + + Assert.assertEquals(withPrefixes, serdeWithPrefixes); + + if(oldRoleARN.equals("")) { + System.clearProperty(SDKGlobalConfiguration.AWS_ROLE_ARN_ENV_VAR); + } + if(oldIdentityFile.equals("")) { + System.clearProperty(SDKGlobalConfiguration.AWS_WEB_IDENTITY_ENV_VAR); + } + + EasyMock.verify(SERVER_SIDE_ENCRYPTING_AMAZON_S3_BUILDER); + } + @Test public void testS3InputSourceUseDefaultPasswordWhenCloudConfigPropertiesWithoutCrediential() {