Skip to content

Commit e93eaab

Browse files
zhaoyunjiongJohn Zhao
zhaoyunjiong
and
John Zhao
authored
Aliyun: Add RRSA support for OSS authentication (#14443)
* Aliyun: Add RRSA support for OSS authentication * Add more description about RRSA and use Strings.isNullOrEmpty to simplify the code. --------- Co-authored-by: John Zhao <yunjiong_zhao@apple.com>
1 parent bf54961 commit e93eaab

4 files changed

Lines changed: 142 additions & 1 deletion

File tree

‎aliyun/src/main/java/org/apache/iceberg/aliyun/AliyunClientFactories.java‎

Lines changed: 74 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,20 @@
1818
*/
1919
package org.apache.iceberg.aliyun;
2020

21+
import com.aliyun.credentials.models.CredentialModel;
22+
import com.aliyun.credentials.provider.OIDCRoleArnCredentialProvider;
2123
import com.aliyun.oss.OSS;
2224
import com.aliyun.oss.OSSClientBuilder;
25+
import com.aliyun.oss.common.auth.BasicCredentials;
26+
import com.aliyun.oss.common.auth.Credentials;
27+
import com.aliyun.oss.common.auth.CredentialsProvider;
2328
import java.util.Map;
2429
import org.apache.iceberg.common.DynConstructors;
2530
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
2631
import org.apache.iceberg.relocated.com.google.common.base.Strings;
2732
import org.apache.iceberg.util.PropertyUtil;
33+
import org.slf4j.Logger;
34+
import org.slf4j.LoggerFactory;
2835

2936
public class AliyunClientFactories {
3037

@@ -81,17 +88,83 @@ private static AliyunClientFactory loadClientFactory(
8188
}
8289

8390
static class DefaultAliyunClientFactory implements AliyunClientFactory {
91+
private static final Logger LOG = LoggerFactory.getLogger(DefaultAliyunClientFactory.class);
8492
private AliyunProperties aliyunProperties;
8593

8694
DefaultAliyunClientFactory() {}
8795

96+
/**
97+
* Check if RRSA environment variables are present. RRSA requires
98+
* ALIBABA_CLOUD_OIDC_PROVIDER_ARN, ALIBABA_CLOUD_ROLE_ARN and ALIBABA_CLOUD_OIDC_TOKEN_FILE to
99+
* be set. RRSA stands for RAM Roles for Service Accounts. It works by letting pods assume
100+
* specific RAM (Resource Access Management) roles when they need to call cloud APIs — no
101+
* hard-coded credentials are needed, which reduces risk of credential leaks. Here is the
102+
* document for RRSA:
103+
* https://www.alibabacloud.com/help/en/ack/ack-managed-and-ack-dedicated/user-guide/use-rrsa-to-authorize-pods-to-access-different-cloud-services
104+
*/
105+
boolean isRrsaEnvironmentAvailable() {
106+
String oidcProviderArn = System.getenv("ALIBABA_CLOUD_OIDC_PROVIDER_ARN");
107+
String roleArn = System.getenv("ALIBABA_CLOUD_ROLE_ARN");
108+
String oidcTokenFile = System.getenv("ALIBABA_CLOUD_OIDC_TOKEN_FILE");
109+
return !Strings.isNullOrEmpty(oidcProviderArn)
110+
&& !Strings.isNullOrEmpty(roleArn)
111+
&& !Strings.isNullOrEmpty(oidcTokenFile);
112+
}
113+
88114
@Override
89115
public OSS newOSSClient() {
90116
Preconditions.checkNotNull(
91117
aliyunProperties,
92118
"Cannot create aliyun oss client before initializing the AliyunClientFactory.");
93119

94-
if (Strings.isNullOrEmpty(aliyunProperties.securityToken())) {
120+
String endpoint = aliyunProperties.ossEndpoint();
121+
122+
// Check if RRSA environment is available
123+
if (isRrsaEnvironmentAvailable()) {
124+
try {
125+
LOG.info(
126+
"Detected RRSA environment variables, creating OSS client with RRSA credentials for endpoint: {}",
127+
endpoint);
128+
129+
// Use OIDCRoleArnCredentialProvider directly with built-in caching and auto-refresh
130+
final OIDCRoleArnCredentialProvider oidcProvider =
131+
OIDCRoleArnCredentialProvider.builder().build();
132+
133+
CredentialsProvider ossCredProvider =
134+
new CredentialsProvider() {
135+
private volatile Credentials currentCredentials;
136+
137+
@Override
138+
public void setCredentials(Credentials credentials) {}
139+
140+
@Override
141+
public Credentials getCredentials() {
142+
try {
143+
LOG.debug("Getting credentials using RRSA");
144+
// getCredentials() returns cached credentials and auto-refreshes when needed
145+
CredentialModel cred = oidcProvider.getCredentials();
146+
long expirationSeconds = 0;
147+
if (cred.getExpiration() > 0) {
148+
expirationSeconds =
149+
(cred.getExpiration() - System.currentTimeMillis()) / 1000;
150+
}
151+
this.currentCredentials =
152+
new BasicCredentials(
153+
cred.getAccessKeyId(),
154+
cred.getAccessKeySecret(),
155+
cred.getSecurityToken(),
156+
expirationSeconds);
157+
return this.currentCredentials;
158+
} catch (Exception e) {
159+
throw new RuntimeException("Failed to get RRSA credentials", e);
160+
}
161+
}
162+
};
163+
return new OSSClientBuilder().build(endpoint, ossCredProvider);
164+
} catch (Exception e) {
165+
throw new RuntimeException("Failed to create RRSA OSS client", e);
166+
}
167+
} else if (Strings.isNullOrEmpty(aliyunProperties.securityToken())) {
95168
return new OSSClientBuilder()
96169
.build(
97170
aliyunProperties.ossEndpoint(),

‎aliyun/src/test/java/org/apache/iceberg/aliyun/TestAliyunClientFactories.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
2626
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
2727
import org.junit.jupiter.api.Test;
28+
import org.junitpioneer.jupiter.SetEnvironmentVariable;
2829

2930
public class TestAliyunClientFactories {
3031

@@ -76,6 +77,64 @@ public void testLoadCustom() {
7677
.isInstanceOf(CustomFactory.class);
7778
}
7879

80+
/**
81+
* Test RRSA environment detection.
82+
*
83+
* <p>This test requires the following environment variables to be set:
84+
*
85+
* <ul>
86+
* <li>ALIBABA_CLOUD_OIDC_PROVIDER_ARN
87+
* <li>ALIBABA_CLOUD_ROLE_ARN
88+
* <li>ALIBABA_CLOUD_OIDC_TOKEN_FILE
89+
* </ul>
90+
*/
91+
@Test
92+
@SetEnvironmentVariable(
93+
key = "ALIBABA_CLOUD_OIDC_PROVIDER_ARN",
94+
value = "acs:ram::123456789:oidc-provider/ack-rrsa-test")
95+
@SetEnvironmentVariable(
96+
key = "ALIBABA_CLOUD_ROLE_ARN",
97+
value = "acs:ram::123456789:role/test-rrsa-role")
98+
@SetEnvironmentVariable(key = "ALIBABA_CLOUD_OIDC_TOKEN_FILE", value = "/tmp/oidc-token")
99+
public void testRRSAEnvironmentDetection() {
100+
Map<String, String> properties = Maps.newHashMap();
101+
properties.put(AliyunProperties.OSS_ENDPOINT, "https://oss-cn-hangzhou.aliyuncs.com");
102+
103+
AliyunClientFactories.DefaultAliyunClientFactory factory =
104+
new AliyunClientFactories.DefaultAliyunClientFactory();
105+
factory.initialize(properties);
106+
assertThat(factory.isRrsaEnvironmentAvailable()).isTrue();
107+
108+
OSS client = factory.newOSSClient();
109+
assertThat(client).as("OSS client should be created with RRSA").isNotNull();
110+
111+
// Try to actually use the client - this should trigger credential retrieval
112+
// With fake credentials, this should fail
113+
try {
114+
client.doesBucketExist("test-bucket");
115+
// If we get here with fake creds, something is wrong
116+
throw new AssertionError(
117+
"Expected operation to fail with fake RRSA credentials, but it succeeded");
118+
} catch (Exception e) {
119+
// Expected - fake RRSA credentials should cause failure
120+
assertThat(e).isNotNull();
121+
} finally {
122+
client.shutdown();
123+
}
124+
}
125+
126+
@Test
127+
public void testIsRrsaEnvironmentAvailableWithoutEnvVars() {
128+
// Verify that isRrsaEnvironmentAvailable returns false when env vars are not set
129+
AliyunClientFactories.DefaultAliyunClientFactory factory =
130+
new AliyunClientFactories.DefaultAliyunClientFactory();
131+
132+
// Assuming RRSA env vars are not set in test environment
133+
assertThat(factory.isRrsaEnvironmentAvailable())
134+
.as("RRSA should not be available without environment variables")
135+
.isFalse();
136+
}
137+
79138
public static class CustomFactory implements AliyunClientFactory {
80139

81140
AliyunProperties aliyunProperties;

‎build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -454,6 +454,8 @@ project(':iceberg-aliyun') {
454454
implementation project(':iceberg-common')
455455

456456
compileOnly libs.aliyun.sdk.oss
457+
implementation libs.aliyun.credentials.java
458+
implementation libs.aliyun.tea
457459
compileOnly libs.jaxb.api
458460
compileOnly libs.activation
459461
compileOnly libs.jaxb.runtime
@@ -467,6 +469,7 @@ project(':iceberg-aliyun') {
467469
testImplementation platform(libs.jackson.bom)
468470
testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml"
469471
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
472+
testImplementation libs.junit.pioneer
470473
}
471474
}
472475

‎gradle/libs.versions.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@
2121

2222
[versions]
2323
activation = "1.1.1"
24+
aliyun-credentials-java = "0.3.2"
2425
aliyun-sdk-oss = "3.18.4"
26+
aliyun-tea = "1.2.1"
2527
analyticsaccelerator = "1.3.1"
2628
antlr = "4.9.3"
2729
antlr413 = "4.13.1" # For Spark 4.0 support
@@ -70,6 +72,7 @@ jaxb-runtime = "2.3.9"
7072
jetty = "11.0.26"
7173
junit = "5.14.2"
7274
junit-platform = "1.14.2"
75+
junit-pioneer = "2.3.0"
7376
kafka = "3.9.1"
7477
kryo-shaded = "4.0.3"
7578
microprofile-openapi-api = "3.1.2"
@@ -95,7 +98,9 @@ tez08 = { strictly = "0.8.4"} # see rich version usage explanation above
9598
[libraries]
9699
activation = { module = "javax.activation:activation", version.ref = "activation" }
97100
aircompressor = { module = "io.airlift:aircompressor", version.ref = "aircompressor" }
101+
aliyun-credentials-java = { module = "com.aliyun:credentials-java", version.ref = "aliyun-credentials-java" }
98102
aliyun-sdk-oss = { module = "com.aliyun.oss:aliyun-sdk-oss", version.ref = "aliyun-sdk-oss" }
103+
aliyun-tea = { module = "com.aliyun:tea", version.ref = "aliyun-tea" }
99104
analyticsaccelerator-s3 = { module = "software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3", version.ref = "analyticsaccelerator" }
100105
antlr-antlr4 = { module = "org.antlr:antlr4", version.ref = "antlr" }
101106
antlr-runtime = { module = "org.antlr:antlr4-runtime", version.ref = "antlr" }
@@ -206,6 +211,7 @@ jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty
206211
jetty-servlet = { module = "org.eclipse.jetty:jetty-servlet", version.ref = "jetty" }
207212
junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" }
208213
junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" }
214+
junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" }
209215
junit-platform-launcher = { module = "org.junit.platform:junit-platform-launcher", version.ref = "junit-platform" }
210216
junit-suite-api = { module = "org.junit.platform:junit-platform-suite-api", version.ref = "junit-platform" }
211217
junit-suite-engine = { module = "org.junit.platform:junit-platform-suite-engine", version.ref = "junit-platform" }

0 commit comments

Comments
 (0)