From 5b4ca59d6f0dc0413f8918091e773a3e23a12d85 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 5 Dec 2025 10:53:52 +0800 Subject: [PATCH 1/4] add credentials_provider_type [feat](catalog)Support for Loading Catalog Credentials via AwsCredentialsProviderChain This update enables Catalogs to automatically load credentials for S3 or S3-compatible storage using AwsCredentialsProviderChain. Users no longer need to explicitly specify AK/SK in Catalog properties. The feature provides stronger support for cloud-native environments such as IRSA, containers, and EC2 Instance Profiles. All credential resolution is unified through DefaultDorisAwsCredentialsProviderChain, ensuring consistent and secure handling across the project. Key Behavior No need to configure AK/SK manually: Credentials can be automatically derived from IRSA, WebIdentity, container metadata, EC2 instance profile, environment variables, system properties, or AWS profile files. Simplified Catalog configuration: Users can set s3.auth_type = INSTANCE_PROFILE or rely on environment detection without providing any keys. Unified credential resolution: All modules use DefaultDorisAwsCredentialsProviderChain to avoid inconsistent custom implementations. Backwards compatible: If s3.access_key / s3.secret_key are provided explicitly, the previous behavior is preserved. --- .../ConfigurationAWSCredentialsProvider.java | 17 ++- .../glue/catalog/util/AWSGlueConfig.java | 1 + .../common/AwsCredentialsProviderMode.java | 99 ++++++++++++++ ...faultDorisAwsCredentialsProviderChain.java | 89 +++++++++++++ .../AWSGlueMetaStoreBaseProperties.java | 18 ++- .../HiveGlueMetaStoreProperties.java | 11 ++ .../IcebergGlueMetaStoreProperties.java | 1 + .../property/storage/S3Properties.java | 59 ++++----- .../property/storage/S3PropertiesTest.java | 20 ++- .../test_catalog_instance_profile.groovy | 125 ++++++++++++++++++ 10 files changed, 397 insertions(+), 43 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/DefaultDorisAwsCredentialsProviderChain.java create mode 100644 regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy diff --git a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java index 49bea54c38e5b0..30af6c8dc54a18 100644 --- a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java +++ b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java @@ -20,12 +20,20 @@ import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSCredentialsProviderChain; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.BasicSessionCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; +import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; +import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; +import com.amazonaws.auth.SystemPropertiesCredentialsProvider; +import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; import com.amazonaws.glue.catalog.util.AWSGlueConfig; import com.amazonaws.util.StringUtils; +import org.apache.doris.common.Config; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import org.apache.hadoop.conf.Configuration; public class ConfigurationAWSCredentialsProvider implements AWSCredentialsProvider { @@ -47,9 +55,9 @@ public AWSCredentials getCredentials() { return (StringUtils.isNullOrEmpty(sessionToken) ? new BasicAWSCredentials(accessKey, secretKey) : new BasicSessionCredentials(accessKey, secretKey, sessionToken)); } - - AWSCredentialsProvider longLivedProvider = new DefaultAWSCredentialsProviderChain(); - + String credentialsProviderModeString = StringUtils.lowerCase(conf.get(AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE)); + AwsCredentialsProviderMode credentialsProviderMode=AwsCredentialsProviderMode.fromString(credentialsProviderModeString); + AWSCredentialsProvider longLivedProvider = credentialsProviderMode.getCredentialsProviderV1(); if (!StringUtils.isNullOrEmpty(roleArn)) { STSAssumeRoleSessionCredentialsProvider.Builder builder = new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, "local-session") @@ -61,6 +69,9 @@ public AWSCredentials getCredentials() { STSAssumeRoleSessionCredentialsProvider provider = builder.build(); return provider.getCredentials(); } + if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) { + return longLivedProvider.getCredentials(); + } throw new SdkClientException("Unable to load AWS credentials from any provider in the chain"); } diff --git a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java index 4296eee0e94364..7b0bd3ef979130 100644 --- a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java +++ b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java @@ -63,4 +63,5 @@ private AWSGlueConfig() { public static final String AWS_GLUE_SESSION_TOKEN = "aws.glue.session-token"; public static final String AWS_GLUE_ROLE_ARN = "aws.glue.role-arn"; public static final String AWS_GLUE_EXTERNAL_ID = "aws.glue.external-id"; + public static final String AWS_CREDENTIALS_PROVIDER_MODE = "aws.credentials.provider.mode"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java new file mode 100644 index 00000000000000..fb8c296353d997 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.common; + +import lombok.Getter; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; +import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider; +import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; + +public enum AwsCredentialsProviderMode { + + DEFAULT("DEFAULT", DefaultDorisAwsCredentialsProviderChain.create(), new com.amazonaws.auth + .AWSCredentialsProviderChain( + new com.amazonaws.auth.InstanceProfileCredentialsProvider(), com.amazonaws.auth + .WebIdentityTokenCredentialsProvider.create(), + new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper(), + new com.amazonaws.auth.EnvironmentVariableCredentialsProvider(), new com.amazonaws.auth + .SystemPropertiesCredentialsProvider()), + DefaultDorisAwsCredentialsProviderChain.class.getName()), + ENV("ENV", EnvironmentVariableCredentialsProvider.create(), new com.amazonaws.auth + .EnvironmentVariableCredentialsProvider(), EnvironmentVariableCredentialsProvider.class.getName()), + SYSTEM_PROPERTIES("SYSTEM_PROPERTIES", SystemPropertyCredentialsProvider.create(), new com.amazonaws + .auth.SystemPropertiesCredentialsProvider(), SystemPropertyCredentialsProvider.class.getName()), + WEB_IDENTITY("WEB_IDENTITY", WebIdentityTokenFileCredentialsProvider.create(), new com.amazonaws + .auth.WebIdentityTokenCredentialsProvider(), WebIdentityTokenFileCredentialsProvider.class.getName()), + CONTAINER("CONTAINER", ContainerCredentialsProvider.create(), new com.amazonaws + .auth.ContainerCredentialsProvider(), ContainerCredentialsProvider.class.getName()), + INSTANCE_PROFILE("INSTANCE_PROFILE", InstanceProfileCredentialsProvider.create(), + new com.amazonaws.auth.InstanceProfileCredentialsProvider(), InstanceProfileCredentialsProvider + .class.getName()); + + private final String mode; + + @Getter + private final com.amazonaws.auth.AWSCredentialsProvider credentialsProviderV1; + @Getter + private final AwsCredentialsProvider credentialsProviderV2; + @Getter + private final String className; + + AwsCredentialsProviderMode(String mode, AwsCredentialsProvider credentialsProviderV2, + com.amazonaws.auth.AWSCredentialsProvider credentialsProviderV1, String className) { + this.mode = mode; + this.credentialsProviderV2 = credentialsProviderV2; + this.credentialsProviderV1 = credentialsProviderV1; + this.className = className; + } + + /** + * Parse from user-provided string (case-insensitive) + * Supported examples: + * "auto", "env", "system-properties", "system_properties", + * "web_identity", "web-identity", "profile", + * "container", "instance_profile", "instance-profile" + */ + public static AwsCredentialsProviderMode fromString(String value) { + if (value == null || value.isEmpty()) { + return DEFAULT; + } + + String normalized = value.trim().toUpperCase().replace('-', '_'); + + switch (normalized) { + case "ENV": + return ENV; + case "SYSTEM_PROPERTIES": + return SYSTEM_PROPERTIES; + case "WEB_IDENTITY": + return WEB_IDENTITY; + case "CONTAINER": + return CONTAINER; + case "INSTANCE_PROFILE": + return INSTANCE_PROFILE; + case "DEFAULT": + return DEFAULT; + default: + throw new IllegalArgumentException("Unsupported AWS credentials provider: " + value); + } + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/DefaultDorisAwsCredentialsProviderChain.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/DefaultDorisAwsCredentialsProviderChain.java new file mode 100644 index 00000000000000..15c644b4d75752 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/DefaultDorisAwsCredentialsProviderChain.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.common; + +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; +import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; +import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider; +import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; +import software.amazon.awssdk.utils.IoUtils; +import software.amazon.awssdk.utils.SdkAutoCloseable; + +/** + * DefaultDorisAwsCredentialsProviderChain + * + *

Default AWS credentials provider chain used across Doris (singleton).

+ * + * + *

+ * Usage example: + * {@code AwsCredentials creds = DefaultDorisAwsCredentialsProviderChain.create();} + */ +public final class DefaultDorisAwsCredentialsProviderChain implements AwsCredentialsProvider, SdkAutoCloseable { + private static final DefaultDorisAwsCredentialsProviderChain INSTANCE = + new DefaultDorisAwsCredentialsProviderChain(); + + private final AwsCredentialsProvider delegate; + + private DefaultDorisAwsCredentialsProviderChain() { + this.delegate = AwsCredentialsProviderChain.of( + WebIdentityTokenFileCredentialsProvider.create(), + ContainerCredentialsProvider.create(), + InstanceProfileCredentialsProvider.create(), + SystemPropertyCredentialsProvider.create(), + EnvironmentVariableCredentialsProvider.create() + ); + } + + public static DefaultDorisAwsCredentialsProviderChain create() { + return new DefaultDorisAwsCredentialsProviderChain(); + } + + public static DefaultDorisAwsCredentialsProviderChain getInstance() { + return INSTANCE; + } + + @Override + public AwsCredentials resolveCredentials() { + return delegate.resolveCredentials(); + } + + @Override + public void close() { + IoUtils.closeIfCloseable(delegate, null); + } + + @Override + public String toString() { + return "DefaultDorisAwsCredentialsProviderChain{delegate=" + delegate + "}"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java index 445b8311f85f67..b17504fbca7bb4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java @@ -114,8 +114,6 @@ private ParamRules buildRules() { return new ParamRules().requireTogether(new String[]{glueAccessKey, glueSecretKey}, "glue.access_key and glue.secret_key must be set together") - .requireAtLeastOne(new String[]{glueAccessKey, glueIAMRole}, - "At least one of glue.access_key or glue.role_arn must be set") .require(glueEndpoint, "glue.endpoint must be set") .check(() -> StringUtils.isNotBlank(glueEndpoint) && !glueEndpoint.startsWith("https://"), "glue.endpoint must use https protocol,please set glue.endpoint to https://..."); @@ -137,6 +135,22 @@ private void checkAndInit() { } } + /** + * Validate that at least one Glue credential (an access key or an IAM role) is explicitly provided. + * + * Purpose: Some catalog implementations (for example, Iceberg) do not support obtaining credentials + * from the default credential chain (instance metadata, environment variables, etc.). In addition, + * the configuration or UI may only expose two options: {@code glue.access_key} and {@code glue.role_arn}. + * In such cases, at least one of these must be explicitly set. If neither is provided, an + * {@link IllegalArgumentException} is thrown to prompt the user to complete the configuration. + */ + protected void requireExplicitGlueCredentials() { + if (StringUtils.isNotBlank(glueAccessKey) || StringUtils.isNotBlank(glueIAMRole)) { + return; + } + throw new IllegalArgumentException("At least one of glue.access_key or glue.role_arn must be set"); + } + private String extractRegionFromEndpoint(Matcher matcher) { for (int i = 1; i <= matcher.groupCount(); i++) { String group = matcher.group(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java index 5a90a3af7dc94f..06cd55b431b195 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property.metastore; import org.apache.doris.datasource.property.ConnectorProperty; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import com.amazonaws.ClientConfiguration; import com.amazonaws.glue.catalog.util.AWSGlueConfig; @@ -75,6 +76,14 @@ public class HiveGlueMetaStoreProperties extends AbstractHiveProperties { description = "Catalog separator character for AWS Glue.") protected String awsGlueCatalogSeparator = ""; + @ConnectorProperty(names = {"glue.credentials_provider_type"}, + required = false, + description = "The credentials provider type of S3. " + + "Options are: DEFAULT, ASSUME_ROLE, ANONYMOUS, ENVIRONMENT, SYSTEM_PROPERTIES, " + + "WEB_IDENTITY_TOKEN_FILE, INSTANCE_PROFILE. " + + "If not set, it will use the default provider chain of AWS SDK.") + protected String awsCredentialsProviderType = AwsCredentialsProviderMode.DEFAULT.name(); + // ========== Constructor ========== /** @@ -115,6 +124,8 @@ private void initHiveConf() { setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_SESSION_TOKEN, baseProperties.glueSessionToken); setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_ROLE_ARN, baseProperties.glueIAMRole); setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_EXTERNAL_ID, baseProperties.glueExternalId); + setHiveConfPropertiesIfNotNull(hiveConf, + AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE, awsCredentialsProviderType); } private static void setHiveConfPropertiesIfNotNull(HiveConf hiveConf, String key, String value) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java index ba40940005b5fa..3039a96ea9d3f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java @@ -55,6 +55,7 @@ public String getIcebergCatalogType() { public void initNormalizeAndCheckProps() { super.initNormalizeAndCheckProps(); glueProperties = AWSGlueMetaStoreBaseProperties.of(origProps); + glueProperties.requireExplicitGlueCredentials(); s3Properties = S3Properties.of(origProps); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index ffa6e6c7450dd4..a9c1bd308a366a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -24,6 +24,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.datasource.property.ConnectorPropertiesUtils; import org.apache.doris.datasource.property.ConnectorProperty; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import org.apache.doris.thrift.TCredProviderType; import org.apache.doris.thrift.TS3StorageParam; @@ -33,15 +34,10 @@ import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; -import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; -import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; -import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; -import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; -import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider; -import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; @@ -57,6 +53,8 @@ public class S3Properties extends AbstractS3CompatibleProperties { + private static final Logger LOG = LogManager.getLogger(S3Properties.class); + public static final String USE_PATH_STYLE = "use_path_style"; private static final String[] ENDPOINT_NAMES_FOR_GUESSING = { @@ -170,6 +168,16 @@ public class S3Properties extends AbstractS3CompatibleProperties { description = "The external id of S3.") protected String s3ExternalId = ""; + @ConnectorProperty(names = {"s3.credentials_provider_type", "glue.credentials_provider_type"}, + required = false, + description = "The credentials provider type of S3. " + + "Options are: DEFAULT, ASSUME_ROLE, ENVIRONMENT, SYSTEM_PROPERTIES, " + + "WEB_IDENTITY_TOKEN_FILE, INSTANCE_PROFILE. " + + "If not set, it will use the default provider chain of AWS SDK.") + protected String awsCredentialsProviderType = AwsCredentialsProviderMode.DEFAULT.name(); + + private AwsCredentialsProviderMode awsCredentialsProviderMode; + public static S3Properties of(Map properties) { S3Properties propertiesObj = new S3Properties(properties); ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj, properties); @@ -215,6 +223,7 @@ public void initNormalizeAndCheckProps() { throw new IllegalArgumentException("s3.external_id must be used with s3.role_arn"); } convertGlueToS3EndpointIfNeeded(); + awsCredentialsProviderMode = AwsCredentialsProviderMode.fromString(awsCredentialsProviderType); } /** @@ -297,7 +306,7 @@ private AwsCredentialsProvider getAwsCredentialsProviderV1() { if (StringUtils.isNotBlank(s3IAMRole)) { StsClient stsClient = StsClient.builder() .region(Region.of(region)) - .credentialsProvider(InstanceProfileCredentialsProvider.create()) + .credentialsProvider(awsCredentialsProviderMode.getCredentialsProviderV2()) .build(); return StsAssumeRoleCredentialsProvider.builder() @@ -310,15 +319,7 @@ private AwsCredentialsProvider getAwsCredentialsProviderV1() { }).build(); } // For anonymous access (no credentials required) - //fixme: should return AwsCredentialsProviderChain - if (StringUtils.isBlank(accessKey) && StringUtils.isBlank(secretKey)) { - return AnonymousCredentialsProvider.create(); - } - return AwsCredentialsProviderChain.of(SystemPropertyCredentialsProvider.create(), - EnvironmentVariableCredentialsProvider.create(), - WebIdentityTokenFileCredentialsProvider.create(), - ProfileCredentialsProvider.create(), - InstanceProfileCredentialsProvider.create()); + return AnonymousCredentialsProvider.create(); } private AwsCredentialsProvider getAwsCredentialsProviderV2() { @@ -329,13 +330,7 @@ private AwsCredentialsProvider getAwsCredentialsProviderV2() { if (StringUtils.isNotBlank(s3IAMRole)) { StsClient stsClient = StsClient.builder() .region(Region.of(region)) - .credentialsProvider(AwsCredentialsProviderChain.of( - WebIdentityTokenFileCredentialsProvider.create(), - ContainerCredentialsProvider.create(), - InstanceProfileCredentialsProvider.create(), - SystemPropertyCredentialsProvider.create(), - EnvironmentVariableCredentialsProvider.create(), - ProfileCredentialsProvider.create())) + .credentialsProvider(awsCredentialsProviderMode.getCredentialsProviderV2()) .build(); return StsAssumeRoleCredentialsProvider.builder() @@ -347,13 +342,7 @@ private AwsCredentialsProvider getAwsCredentialsProviderV2() { } }).build(); } - return AwsCredentialsProviderChain.of( - WebIdentityTokenFileCredentialsProvider.create(), - ContainerCredentialsProvider.create(), - InstanceProfileCredentialsProvider.create(), - SystemPropertyCredentialsProvider.create(), - EnvironmentVariableCredentialsProvider.create(), - ProfileCredentialsProvider.create()); + return awsCredentialsProviderMode.getCredentialsProviderV2(); } @Override @@ -375,11 +364,15 @@ public void initializeHadoopStorageConfig() { hadoopStorageConfig.set("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider"); hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider", - "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,com.amazonaws.auth.EnvironmentVar" - + "iableCredentialsProvider,com.amazonaws.auth.InstanceProfileCredentialsProvider"); + awsCredentialsProviderMode.getClassName()); if (StringUtils.isNotBlank(s3ExternalId)) { hadoopStorageConfig.set("fs.s3a.assumed.role.external.id", s3ExternalId); } + return; + } + if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) { + hadoopStorageConfig.set("fs.s3a.aws.credentials.provider", + awsCredentialsProviderMode.getClassName()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index f4ea2fcb381a0c..b2808af86d2628 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.common.DefaultDorisAwsCredentialsProviderChain; import com.google.common.collect.Maps; import mockit.Expectations; @@ -29,7 +30,6 @@ import org.junit.jupiter.api.Test; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.services.sts.StsClient; @@ -222,9 +222,10 @@ public void testS3IamRoleWithExternalId() throws UserException { Assertions.assertEquals("arn:aws:iam::123456789012:role/MyTestRole", backendProperties.get("AWS_ROLE_ARN")); } + @Test public void testGetAwsCredentialsProviderWithIamRoleAndExternalId(@Mocked StsClientBuilder mockBuilder, - @Mocked StsClient mockStsClient, @Mocked InstanceProfileCredentialsProvider mockInstanceCreds) { + @Mocked StsClient mockStsClient) { new Expectations() { { @@ -234,8 +235,6 @@ public void testGetAwsCredentialsProviderWithIamRoleAndExternalId(@Mocked StsCli result = mockBuilder; mockBuilder.build(); result = mockStsClient; - InstanceProfileCredentialsProvider.create(); - result = mockInstanceCreds; } }; @@ -247,6 +246,17 @@ public void testGetAwsCredentialsProviderWithIamRoleAndExternalId(@Mocked StsCli AwsCredentialsProvider provider = s3Props.getAwsCredentialsProvider(); Assertions.assertNotNull(provider); Assertions.assertTrue(provider instanceof StsAssumeRoleCredentialsProvider); + origProps.remove("s3.external_id"); + origProps.remove("s3.role_arn"); + s3Props = (S3Properties) StorageProperties.createPrimary(origProps); + provider = s3Props.getAwsCredentialsProvider(); + Assertions.assertNotNull(provider); + Assertions.assertTrue(provider instanceof DefaultDorisAwsCredentialsProviderChain); + origProps.put("s3.credentials_provider_type", "instance_profile"); + s3Props = (S3Properties) StorageProperties.createPrimary(origProps); + provider = s3Props.getAwsCredentialsProvider(); + Assertions.assertNotNull(provider); + Assertions.assertTrue(provider instanceof InstanceProfileCredentialsProvider); } @Test @@ -420,7 +430,7 @@ public void testS3PropertiesAwsAnonymousCredentialsProvider() { Assertions.assertEquals(AnonymousCredentialsProvider.class, provider.getClass()); Config.aws_credentials_provider_version = "v2"; provider = s3Properties.getAwsCredentialsProvider(); - Assertions.assertEquals(AwsCredentialsProviderChain.class, provider.getClass()); + Assertions.assertEquals(DefaultDorisAwsCredentialsProviderChain.class, provider.getClass()); Config.aws_credentials_provider_version = "v2"; } diff --git a/regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy b/regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy new file mode 100644 index 00000000000000..f8877929b969b5 --- /dev/null +++ b/regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.google.common.base.Strings; + +suite("test_catalog_instance_profile_with_role") { + + if (Strings.isNullOrEmpty(context.config.otherConfigs.get("hiveGlueInstanceProfileQueryTableName"))) { + return + } + + String hiveGlueQueryTableName = context.config.otherConfigs.get("hiveGlueInstanceProfileQueryTableName") + String hiveGlueExpectCounts = context.config.otherConfigs.get("hiveGlueInstanceProfileExpectCounts") + String icebergFsQueryTableName = context.config.otherConfigs.get("icebergFsInstanceProfileQueryTableName") + String icebergFsExpectCounts = context.config.otherConfigs.get("icebergFsInstanceProfileExpectCounts") + String icebergFsWarehouse = context.config.otherConfigs.get("icebergFsInstanceProfileWarehouse") + String region = context.config.otherConfigs.get("awsInstanceProfileRegion") + //query method + def createCatalogAndQuery = { catalogProps, catalogName, queryTableName, expectCounts -> + sql """drop catalog if exists ${catalogName}""" + sql """ + ${catalogProps} + """ + def result = sql """ + select count(1) from ${catalogName}.${queryTableName}; + """ + println("result: ${result}") + def countValue = result[0][0] + assertTrue(countValue == expectCounts.toInteger()) + sql """drop catalog if exists ${catalogName}""" + } + def assertCatalogAndQueryException = { catalogProps, catalogName, errMsg -> + sql """drop catalog if exists ${catalogName}""" + sql """ + ${catalogProps} + """ + try { + sql """ + switch ${catalogName}; + """ + sql """ + show databases; + """ + throw new Exception("Expected exception was not thrown") + }catch (Exception e){ + assertTrue(e.getMessage().contains(errMsg)) + } + } + String hiveGlueCatalogProps = """ + create catalog hive_glue_catalog properties( + "type"="hms", + "hive.metastore.type"="glue", + "glue.region"="${region}", + "glue.endpoint" = "https://glue.${region}.amazonaws.com" + ); + """ + createCatalogAndQuery(hiveGlueCatalogProps, "hive_glue_catalog", hiveGlueQueryTableName, hiveGlueExpectCounts) + hiveGlueCatalogProps = """ + create catalog hive_glue_catalog properties( + "type"="hms", + "hive.metastore.type"="glue", + "glue.credentials_provider_type"="INSTANCE_PROFILE", + "glue.region"="${region}", + "glue.endpoint" = "https://glue.${region}.amazonaws.com" + ); + """ + createCatalogAndQuery(hiveGlueCatalogProps, "hive_glue_catalog", hiveGlueQueryTableName, hiveGlueExpectCounts) + hiveGlueCatalogProps = """ + create catalog hive_glue_catalog properties( + "type"="hms", + "hive.metastore.type"="glue", + "glue.credentials_provider_type"="CONTAINER", + "glue.region"="${region}", + "glue.endpoint" = "https://glue.${region}.amazonaws.com" + ); + """ + assertCatalogAndQueryException(hiveGlueCatalogProps,"hive_glue_catalog", "The environment variable AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + String icebergFsCatalogProps = """ + create catalog iceberg_fs_catalog properties( + "type"="iceberg", + "warehouse"="${icebergFsWarehouse}", + "iceberg.catalog.type"="hadoop", + "s3.region" = "${region}", + "s3.endpoint" = "https://s3.${region}.amazonaws.com" + ); + """ + createCatalogAndQuery(icebergFsCatalogProps, "iceberg_fs_catalog", icebergFsQueryTableName, icebergFsExpectCounts) + icebergFsCatalogProps = """ + create catalog iceberg_fs_catalog properties( + "type"="iceberg", + "warehouse"="${icebergFsWarehouse}", + "iceberg.catalog.type"="hadoop", + "s3.credentials_provider_type"="INSTANCE_PROFILE", + "s3.region" = "${region}", + "s3.endpoint" = "https://s3.${region}.amazonaws.com" + ); + """ + createCatalogAndQuery(icebergFsCatalogProps, "iceberg_fs_catalog", icebergFsQueryTableName, icebergFsExpectCounts) + icebergFsCatalogProps = """ + create catalog iceberg_fs_catalog properties( + "type"="iceberg", + "warehouse"="${icebergFsWarehouse}", + "iceberg.catalog.type"="hadoop", + "s3.credentials_provider_type"="CONTAINER", + "s3.region" = "${region}", + "s3.endpoint" = "https://s3.${region}.amazonaws.com" + ); + """ + assertCatalogAndQueryException(icebergFsCatalogProps,"iceberg_fs_catalog", "No AWS Credentials provided by ContainerCredentialsProvider") + +} From 27faaa3a1e08827249e81bed7a144d839d69199c Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Wed, 10 Dec 2025 10:53:30 +0800 Subject: [PATCH 2/4] remove gcs --- fe/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/pom.xml b/fe/pom.xml index a648bef6e1832c..794d32ae01d74b 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -230,7 +230,7 @@ under the License. 3.1.9 compile compile - compile + provided 2.0.1 8.2.7 5.6.211 From dd340bf110f418eebf95f1bccb3c02260aa69cbd Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sat, 13 Dec 2025 14:50:53 +0800 Subject: [PATCH 3/4] fix --- .../ConfigurationAWSCredentialsProvider.java | 10 +- .../common/AwsCredentialsProviderFactory.java | 158 ++++++++++++++++++ .../common/AwsCredentialsProviderMode.java | 71 +++----- .../property/storage/S3Properties.java | 29 +++- .../property/storage/S3PropertiesTest.java | 15 +- 5 files changed, 219 insertions(+), 64 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java diff --git a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java index 30af6c8dc54a18..23ba972fdfe1aa 100644 --- a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java +++ b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java @@ -20,19 +20,13 @@ import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSCredentialsProviderChain; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.BasicSessionCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; -import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; -import com.amazonaws.auth.SystemPropertiesCredentialsProvider; -import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; import com.amazonaws.glue.catalog.util.AWSGlueConfig; import com.amazonaws.util.StringUtils; import org.apache.doris.common.Config; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory; import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import org.apache.hadoop.conf.Configuration; @@ -57,7 +51,7 @@ public AWSCredentials getCredentials() { } String credentialsProviderModeString = StringUtils.lowerCase(conf.get(AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE)); AwsCredentialsProviderMode credentialsProviderMode=AwsCredentialsProviderMode.fromString(credentialsProviderModeString); - AWSCredentialsProvider longLivedProvider = credentialsProviderMode.getCredentialsProviderV1(); + AWSCredentialsProvider longLivedProvider = AwsCredentialsProviderFactory.createV1(credentialsProviderMode); if (!StringUtils.isNullOrEmpty(roleArn)) { STSAssumeRoleSessionCredentialsProvider.Builder builder = new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, "local-session") diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java new file mode 100644 index 00000000000000..f56aed0533e619 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Copied from +// https://github.com/awslabs/aws-glue-data-catalog-client-for-apache-hive-metastore/blob/branch-3.4.0/ +// + +package org.apache.doris.datasource.property.common; + + +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; +import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; +import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider; +import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; + +import java.util.ArrayList; +import java.util.List; + +public final class AwsCredentialsProviderFactory { + + private AwsCredentialsProviderFactory() { + } + + /* ========================= + * AWS SDK V1 + * ========================= */ + + public static com.amazonaws.auth.AWSCredentialsProvider createV1( + AwsCredentialsProviderMode mode) { + + switch (mode) { + case ENV: + return new com.amazonaws.auth.EnvironmentVariableCredentialsProvider(); + case SYSTEM_PROPERTIES: + return new com.amazonaws.auth.SystemPropertiesCredentialsProvider(); + case WEB_IDENTITY: + return com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create(); + case CONTAINER: + return new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper(); + case ANONYMOUS: + throw new UnsupportedOperationException( + "AWS SDK V1 does not support anonymous credentials provider."); + case INSTANCE_PROFILE: + return new com.amazonaws.auth.InstanceProfileCredentialsProvider(); + case DEFAULT: + return createDefaultV1(); + default: + throw new UnsupportedOperationException( + "AWS SDK V1 does not support credentials provider mode: " + mode); + } + } + + private static com.amazonaws.auth.AWSCredentialsProvider createDefaultV1() { + List providers = new ArrayList<>(); + providers.add(new com.amazonaws.auth.InstanceProfileCredentialsProvider()); + //lazy + env + providers.add(com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create()); + providers.add(new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper()); + providers.add(new com.amazonaws.auth.EnvironmentVariableCredentialsProvider()); + providers.add(new com.amazonaws.auth.SystemPropertiesCredentialsProvider()); + return new com.amazonaws.auth.AWSCredentialsProviderChain( + providers.toArray(new com.amazonaws.auth.AWSCredentialsProvider[0])); + } + + /* ========================= + * AWS SDK V2 + * ========================= */ + + public static AwsCredentialsProvider createV2( + AwsCredentialsProviderMode mode, + boolean includeAnonymousInDefault) { + switch (mode) { + case ENV: + return EnvironmentVariableCredentialsProvider.create(); + case SYSTEM_PROPERTIES: + return SystemPropertyCredentialsProvider.create(); + case WEB_IDENTITY: + return WebIdentityTokenFileCredentialsProvider.create(); + case CONTAINER: + return ContainerCredentialsProvider.create(); + case INSTANCE_PROFILE: + return InstanceProfileCredentialsProvider.create(); + case ANONYMOUS: + return AnonymousCredentialsProvider.create(); + case DEFAULT: + return createDefaultV2(includeAnonymousInDefault); + default: + throw new UnsupportedOperationException( + "AWS SDK V2 does not support credentials provider mode: " + mode); + } + } + + private static AwsCredentialsProvider createDefaultV2( + boolean includeAnonymous) { + + List providers = new ArrayList<>(); + providers.add(InstanceProfileCredentialsProvider.create()); + providers.add(WebIdentityTokenFileCredentialsProvider.create()); + providers.add(ContainerCredentialsProvider.create()); + providers.add(EnvironmentVariableCredentialsProvider.create()); + providers.add(SystemPropertyCredentialsProvider.create()); + if (includeAnonymous) { + providers.add(AnonymousCredentialsProvider.create()); + } + return AwsCredentialsProviderChain.builder() + .credentialsProviders(providers) + .build(); + } + + public static String getV2ClassName(AwsCredentialsProviderMode mode, boolean includeAnonymousInDefault) { + switch (mode) { + case ENV: + return EnvironmentVariableCredentialsProvider.class.getName(); + case SYSTEM_PROPERTIES: + return SystemPropertyCredentialsProvider.class.getName(); + case WEB_IDENTITY: + return WebIdentityTokenFileCredentialsProvider.class.getName(); + case CONTAINER: + return ContainerCredentialsProvider.class.getName(); + case INSTANCE_PROFILE: + return InstanceProfileCredentialsProvider.class.getName(); + case ANONYMOUS: + return AnonymousCredentialsProvider.class.getName(); + case DEFAULT: + List providers = new ArrayList<>(); + providers.add(EnvironmentVariableCredentialsProvider.class.getName()); + providers.add(SystemPropertyCredentialsProvider.class.getName()); + providers.add(WebIdentityTokenFileCredentialsProvider.class.getName()); + providers.add(ContainerCredentialsProvider.class.getName()); + providers.add(InstanceProfileCredentialsProvider.class.getName()); + if (includeAnonymousInDefault) { + providers.add(AnonymousCredentialsProvider.class.getName()); + } + return String.join("+", providers); + default: + throw new UnsupportedOperationException( + "AWS SDK V2 does not support credentials provider mode: " + mode); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java index fb8c296353d997..63481ca5a7a482 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java @@ -17,60 +17,33 @@ package org.apache.doris.datasource.property.common; -import lombok.Getter; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; -import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; -import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; -import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider; -import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; - public enum AwsCredentialsProviderMode { - DEFAULT("DEFAULT", DefaultDorisAwsCredentialsProviderChain.create(), new com.amazonaws.auth - .AWSCredentialsProviderChain( - new com.amazonaws.auth.InstanceProfileCredentialsProvider(), com.amazonaws.auth - .WebIdentityTokenCredentialsProvider.create(), - new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper(), - new com.amazonaws.auth.EnvironmentVariableCredentialsProvider(), new com.amazonaws.auth - .SystemPropertiesCredentialsProvider()), - DefaultDorisAwsCredentialsProviderChain.class.getName()), - ENV("ENV", EnvironmentVariableCredentialsProvider.create(), new com.amazonaws.auth - .EnvironmentVariableCredentialsProvider(), EnvironmentVariableCredentialsProvider.class.getName()), - SYSTEM_PROPERTIES("SYSTEM_PROPERTIES", SystemPropertyCredentialsProvider.create(), new com.amazonaws - .auth.SystemPropertiesCredentialsProvider(), SystemPropertyCredentialsProvider.class.getName()), - WEB_IDENTITY("WEB_IDENTITY", WebIdentityTokenFileCredentialsProvider.create(), new com.amazonaws - .auth.WebIdentityTokenCredentialsProvider(), WebIdentityTokenFileCredentialsProvider.class.getName()), - CONTAINER("CONTAINER", ContainerCredentialsProvider.create(), new com.amazonaws - .auth.ContainerCredentialsProvider(), ContainerCredentialsProvider.class.getName()), - INSTANCE_PROFILE("INSTANCE_PROFILE", InstanceProfileCredentialsProvider.create(), - new com.amazonaws.auth.InstanceProfileCredentialsProvider(), InstanceProfileCredentialsProvider - .class.getName()); + DEFAULT("DEFAULT"), - private final String mode; + ENV("ENV"), + + SYSTEM_PROPERTIES("SYSTEM_PROPERTIES"), + + WEB_IDENTITY("WEB_IDENTITY"), + + CONTAINER("CONTAINER"), + + INSTANCE_PROFILE("INSTANCE_PROFILE"), - @Getter - private final com.amazonaws.auth.AWSCredentialsProvider credentialsProviderV1; - @Getter - private final AwsCredentialsProvider credentialsProviderV2; - @Getter - private final String className; + ANONYMOUS("ANONYMOUS"); - AwsCredentialsProviderMode(String mode, AwsCredentialsProvider credentialsProviderV2, - com.amazonaws.auth.AWSCredentialsProvider credentialsProviderV1, String className) { + private final String mode; + + AwsCredentialsProviderMode(String mode) { this.mode = mode; - this.credentialsProviderV2 = credentialsProviderV2; - this.credentialsProviderV1 = credentialsProviderV1; - this.className = className; } - /** - * Parse from user-provided string (case-insensitive) - * Supported examples: - * "auto", "env", "system-properties", "system_properties", - * "web_identity", "web-identity", "profile", - * "container", "instance_profile", "instance-profile" - */ + public String getMode() { + return mode; + } + + public static AwsCredentialsProviderMode fromString(String value) { if (value == null || value.isEmpty()) { return DEFAULT; @@ -89,11 +62,13 @@ public static AwsCredentialsProviderMode fromString(String value) { return CONTAINER; case "INSTANCE_PROFILE": return INSTANCE_PROFILE; + case "ANONYMOUS": + return ANONYMOUS; case "DEFAULT": return DEFAULT; default: - throw new IllegalArgumentException("Unsupported AWS credentials provider: " + value); + throw new IllegalArgumentException( + "Unsupported AWS credentials provider mode: " + value); } } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index a9c1bd308a366a..3452d759395b2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -24,6 +24,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.datasource.property.ConnectorPropertiesUtils; import org.apache.doris.datasource.property.ConnectorProperty; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory; import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import org.apache.doris.thrift.TCredProviderType; import org.apache.doris.thrift.TS3StorageParam; @@ -38,6 +39,7 @@ import org.apache.logging.log4j.Logger; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; @@ -306,7 +308,7 @@ private AwsCredentialsProvider getAwsCredentialsProviderV1() { if (StringUtils.isNotBlank(s3IAMRole)) { StsClient stsClient = StsClient.builder() .region(Region.of(region)) - .credentialsProvider(awsCredentialsProviderMode.getCredentialsProviderV2()) + .credentialsProvider(InstanceProfileCredentialsProvider.create()) .build(); return StsAssumeRoleCredentialsProvider.builder() @@ -330,9 +332,10 @@ private AwsCredentialsProvider getAwsCredentialsProviderV2() { if (StringUtils.isNotBlank(s3IAMRole)) { StsClient stsClient = StsClient.builder() .region(Region.of(region)) - .credentialsProvider(awsCredentialsProviderMode.getCredentialsProviderV2()) + .credentialsProvider(AwsCredentialsProviderFactory.createV2( + awsCredentialsProviderMode, + false)) .build(); - return StsAssumeRoleCredentialsProvider.builder() .stsClient(stsClient) .refreshRequest(builder -> { @@ -342,7 +345,9 @@ private AwsCredentialsProvider getAwsCredentialsProviderV2() { } }).build(); } - return awsCredentialsProviderMode.getCredentialsProviderV2(); + return AwsCredentialsProviderFactory.createV2( + awsCredentialsProviderMode, + true); } @Override @@ -363,8 +368,16 @@ public void initializeHadoopStorageConfig() { hadoopStorageConfig.set("fs.s3a.assumed.role.arn", s3IAMRole); hadoopStorageConfig.set("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider"); - hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider", - awsCredentialsProviderMode.getClassName()); + if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) { + hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider", + AwsCredentialsProviderFactory.getV2ClassName( + awsCredentialsProviderMode, + false)); + } else { + hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider", + InstanceProfileCredentialsProvider.class.getName()); + } + if (StringUtils.isNotBlank(s3ExternalId)) { hadoopStorageConfig.set("fs.s3a.assumed.role.external.id", s3ExternalId); } @@ -372,7 +385,9 @@ public void initializeHadoopStorageConfig() { } if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) { hadoopStorageConfig.set("fs.s3a.aws.credentials.provider", - awsCredentialsProviderMode.getClassName()); + AwsCredentialsProviderFactory.createV2( + awsCredentialsProviderMode, + true).getClass().getName()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index b2808af86d2628..25353e5119b72c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -30,6 +30,7 @@ import org.junit.jupiter.api.Test; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.services.sts.StsClient; @@ -246,17 +247,29 @@ public void testGetAwsCredentialsProviderWithIamRoleAndExternalId(@Mocked StsCli AwsCredentialsProvider provider = s3Props.getAwsCredentialsProvider(); Assertions.assertNotNull(provider); Assertions.assertTrue(provider instanceof StsAssumeRoleCredentialsProvider); + origProps.put("s3.credentials_provider_type", "instance_profile"); + s3Props = (S3Properties) StorageProperties.createPrimary(origProps); + provider = s3Props.getAwsCredentialsProvider(); + Assertions.assertEquals(StsAssumeRoleCredentialsProvider.class.getName(), provider.getClass().getName()); + Assertions.assertEquals(InstanceProfileCredentialsProvider.class.getName(), s3Props.getHadoopStorageConfig().get("fs.s3a.assumed.role.credentials.provider")); origProps.remove("s3.external_id"); origProps.remove("s3.role_arn"); + origProps.remove("s3.credentials_provider_type"); s3Props = (S3Properties) StorageProperties.createPrimary(origProps); provider = s3Props.getAwsCredentialsProvider(); Assertions.assertNotNull(provider); - Assertions.assertTrue(provider instanceof DefaultDorisAwsCredentialsProviderChain); + Assertions.assertTrue(provider instanceof AwsCredentialsProviderChain); origProps.put("s3.credentials_provider_type", "instance_profile"); s3Props = (S3Properties) StorageProperties.createPrimary(origProps); provider = s3Props.getAwsCredentialsProvider(); Assertions.assertNotNull(provider); Assertions.assertTrue(provider instanceof InstanceProfileCredentialsProvider); + Assertions.assertEquals(InstanceProfileCredentialsProvider.class.getName(), s3Props.getHadoopStorageConfig().get("fs.s3a.aws.credentials.provider")); + origProps.put("s3.credentials_provider_type", "static"); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, "Unsupported AWS credentials provider mode: static", () -> StorageProperties.createPrimary(origProps)); + origProps.put("s3.credentials_provider_type", "anonymous"); + Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps)); + } @Test From 2f5468a03581291b4ca9e67b78dcda30f3eacbe8 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sat, 13 Dec 2025 15:15:47 +0800 Subject: [PATCH 4/4] fix --- ...faultDorisAwsCredentialsProviderChain.java | 89 ------------------- .../property/storage/S3PropertiesTest.java | 3 +- 2 files changed, 1 insertion(+), 91 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/DefaultDorisAwsCredentialsProviderChain.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/DefaultDorisAwsCredentialsProviderChain.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/DefaultDorisAwsCredentialsProviderChain.java deleted file mode 100644 index 15c644b4d75752..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/DefaultDorisAwsCredentialsProviderChain.java +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.datasource.property.common; - -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; -import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; -import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; -import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; -import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider; -import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; -import software.amazon.awssdk.utils.IoUtils; -import software.amazon.awssdk.utils.SdkAutoCloseable; - -/** - * DefaultDorisAwsCredentialsProviderChain - * - *

Default AWS credentials provider chain used across Doris (singleton).

- * - * - *

- * Usage example: - * {@code AwsCredentials creds = DefaultDorisAwsCredentialsProviderChain.create();} - */ -public final class DefaultDorisAwsCredentialsProviderChain implements AwsCredentialsProvider, SdkAutoCloseable { - private static final DefaultDorisAwsCredentialsProviderChain INSTANCE = - new DefaultDorisAwsCredentialsProviderChain(); - - private final AwsCredentialsProvider delegate; - - private DefaultDorisAwsCredentialsProviderChain() { - this.delegate = AwsCredentialsProviderChain.of( - WebIdentityTokenFileCredentialsProvider.create(), - ContainerCredentialsProvider.create(), - InstanceProfileCredentialsProvider.create(), - SystemPropertyCredentialsProvider.create(), - EnvironmentVariableCredentialsProvider.create() - ); - } - - public static DefaultDorisAwsCredentialsProviderChain create() { - return new DefaultDorisAwsCredentialsProviderChain(); - } - - public static DefaultDorisAwsCredentialsProviderChain getInstance() { - return INSTANCE; - } - - @Override - public AwsCredentials resolveCredentials() { - return delegate.resolveCredentials(); - } - - @Override - public void close() { - IoUtils.closeIfCloseable(delegate, null); - } - - @Override - public String toString() { - return "DefaultDorisAwsCredentialsProviderChain{delegate=" + delegate + "}"; - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index 25353e5119b72c..b6a0285f6dae32 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -20,7 +20,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.UserException; -import org.apache.doris.datasource.property.common.DefaultDorisAwsCredentialsProviderChain; import com.google.common.collect.Maps; import mockit.Expectations; @@ -443,7 +442,7 @@ public void testS3PropertiesAwsAnonymousCredentialsProvider() { Assertions.assertEquals(AnonymousCredentialsProvider.class, provider.getClass()); Config.aws_credentials_provider_version = "v2"; provider = s3Properties.getAwsCredentialsProvider(); - Assertions.assertEquals(DefaultDorisAwsCredentialsProviderChain.class, provider.getClass()); + Assertions.assertEquals(AwsCredentialsProviderChain.class, provider.getClass()); Config.aws_credentials_provider_version = "v2"; }