From e5ce67b69f9f5c4ad26dc94375586ea0011e6ee7 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 2 Dec 2024 11:13:33 +0800 Subject: [PATCH 01/26] [core] Add basic auth implementation to support REST Catalog --- .../org/apache/paimon/rest/RESTCatalog.java | 56 ++++++- .../apache/paimon/rest/auth/AuthConfig.java | 56 +++++++ .../apache/paimon/rest/auth/AuthSession.java | 142 ++++++++++++++++++ .../org/apache/paimon/rest/auth/AuthUtil.java | 36 +++++ .../apache/paimon/rest/HttpClientTest.java | 7 +- 5 files changed, 287 insertions(+), 10 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index c96400831370..39b8e579bca7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -18,7 +18,6 @@ package org.apache.paimon.rest; -import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Database; import org.apache.paimon.catalog.Identifier; @@ -27,18 +26,24 @@ import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.auth.AuthConfig; +import org.apache.paimon.rest.auth.AuthSession; +import org.apache.paimon.rest.auth.AuthUtil; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.Table; -import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; +import org.apache.paimon.shade.guava30.com.google.common.annotations.VisibleForTesting; +import org.apache.paimon.shade.guava30.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; /** A catalog implementation for REST. */ public class RESTCatalog implements Catalog { @@ -47,10 +52,12 @@ public class RESTCatalog implements Catalog { private ResourcePaths resourcePaths; private Map options; private Map baseHeader; + // a lazy thread pool for token refresh + private AuthSession catalogAuth = null; + private volatile ScheduledExecutorService refreshExecutor = null; + private boolean keepTokenRefreshed = true; private static final ObjectMapper objectMapper = RESTObjectMapper.create(); - static final String AUTH_HEADER = "Authorization"; - static final String AUTH_HEADER_VALUE_FORMAT = "Bearer %s"; public RESTCatalog(Options options) { if (options.getOptional(CatalogOptions.WAREHOUSE).isPresent()) { @@ -71,8 +78,7 @@ public RESTCatalog(Options options) { threadPoolSize, DefaultErrorHandler.getInstance()); this.client = new HttpClient(httpClientOptions); - Map authHeaders = - ImmutableMap.of(AUTH_HEADER, String.format(AUTH_HEADER_VALUE_FORMAT, token)); + Map authHeaders = AuthUtil.authHeaders(token); Map initHeaders = RESTUtil.merge(configHeaders(options.toMap()), authHeaders); this.options = fetchOptionsFromServer(initHeaders, options.toMap()); @@ -80,6 +86,16 @@ public RESTCatalog(Options options) { this.resourcePaths = ResourcePaths.forCatalogProperties( this.options.get(RESTCatalogInternalOptions.PREFIX)); + this.keepTokenRefreshed = false; + this.catalogAuth = + AuthSession.fromAccessToken( + client, + tokenRefreshExecutor(), + token, + this.baseHeader, + // todo: update,fix null value + new AuthConfig(token, keepTokenRefreshed, null, null), + null); } @Override @@ -194,4 +210,32 @@ Map fetchOptionsFromServer( private static Map configHeaders(Map properties) { return RESTUtil.extractPrefixMap(properties, "header."); } + + private Map headers() { + catalogAuth.refresh(client); + return catalogAuth.getHeaders(); + } + + private ScheduledExecutorService tokenRefreshExecutor() { + if (!keepTokenRefreshed) { + return null; + } + + if (refreshExecutor == null) { + synchronized (this) { + if (refreshExecutor == null) { + this.refreshExecutor = + // todo: move to ThreadPoolUtil + new ScheduledThreadPoolExecutor( + 1, + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("token-refresh-thread") + .build()); + } + } + } + + return refreshExecutor; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java new file mode 100644 index 000000000000..74348cd1b475 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import javax.annotation.Nullable; + +public class AuthConfig { + private final @Nullable String token; + private final Boolean keepRefreshed; + private final @Nullable Long expiresAtMillis; + private final @Nullable Long expiresInMills; + + public AuthConfig( + @Nullable String token, + boolean keepRefreshed, + Long expiresAtMillis, + Long expiresInMills) { + this.token = token; + this.keepRefreshed = keepRefreshed; + this.expiresAtMillis = expiresAtMillis; + this.expiresInMills = expiresInMills; + } + + @Nullable + public String token() { + return token; + } + + public boolean keepRefreshed() { + return keepRefreshed; + } + + public Long expiresAtMillis() { + return expiresAtMillis; + } + + public Long expiresInMills() { + return expiresInMills; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java new file mode 100644 index 000000000000..ff0fc5d87beb --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.rest.RESTClient; +import org.apache.paimon.rest.RESTUtil; +import org.apache.paimon.utils.Pair; + +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.apache.paimon.rest.auth.AuthUtil.authHeaders; + +/** Auth session. */ +public class AuthSession { + private static int tokenRefreshNumRetries = 5; + private static final long MAX_REFRESH_WINDOW_MILLIS = 300_000; // 5 minutes + private static final long MIN_REFRESH_WAIT_MILLIS = 10; + private volatile Map headers; + private volatile AuthConfig config; + + public AuthSession(Map headers, AuthConfig config) { + this.headers = headers; + this.config = config; + } + + public static AuthSession fromAccessToken( + RESTClient client, + ScheduledExecutorService executor, + String token, + Map headers, + AuthConfig config, + Long defaultExpiresAtMillis) { + AuthSession session = new AuthSession(headers, config); + + long startTimeMillis = System.currentTimeMillis(); + Long expiresAtMillis = session.config.expiresAtMillis(); + + if (null != expiresAtMillis && expiresAtMillis <= startTimeMillis) { + Pair expiration = session.refresh(client); + // if expiration is non-null, then token refresh was successful + if (expiration != null) { + if (null != config.expiresAtMillis()) { + // use the new expiration time from the refreshed token + expiresAtMillis = config.expiresAtMillis(); + } else { + // otherwise use the expiration time from the token response + expiresAtMillis = startTimeMillis + expiration.getKey(); + } + } else { + // token refresh failed, don't reattempt with the original expiration + expiresAtMillis = null; + } + } else if (null == expiresAtMillis && defaultExpiresAtMillis != null) { + expiresAtMillis = defaultExpiresAtMillis; + } + + if (null != executor && null != expiresAtMillis) { + scheduleTokenRefresh(client, executor, session, expiresAtMillis); + } + + return session; + } + + public Map getHeaders() { + return headers; + } + + private static void scheduleTokenRefresh( + RESTClient client, + ScheduledExecutorService executor, + AuthSession session, + long expiresAtMillis) { + long expiresInMillis = expiresAtMillis - System.currentTimeMillis(); + // how much ahead of time to start the request to allow it to complete + long refreshWindowMillis = Math.min(expiresInMillis / 10, MAX_REFRESH_WINDOW_MILLIS); + // how much time to wait before expiration + long waitIntervalMillis = expiresInMillis - refreshWindowMillis; + // how much time to actually wait + long timeToWait = Math.max(waitIntervalMillis, MIN_REFRESH_WAIT_MILLIS); + + executor.schedule( + () -> { + long refreshStartTime = System.currentTimeMillis(); + Pair expiration = session.refresh(client); + if (expiration != null) { + scheduleTokenRefresh( + client, executor, session, refreshStartTime + expiration.getKey()); + } + }, + timeToWait, + TimeUnit.MILLISECONDS); + } + + public Pair refresh(RESTClient client) { + if (config.token() != null && config.keepRefreshed()) { + long startTimeMillis = System.currentTimeMillis(); + AuthConfig authConfig = refreshExpiredToken(client); + boolean isSuccessful = authConfig.token() != null; + if (!isSuccessful) { + return null; + } + long expiresAtMillis = startTimeMillis + authConfig.expiresInMills(); + this.config = + new AuthConfig( + authConfig.token(), + config.keepRefreshed(), + expiresAtMillis, + authConfig.expiresInMills()); + Map currentHeaders = this.headers; + this.headers = RESTUtil.merge(currentHeaders, authHeaders(config.token())); + + if (authConfig.expiresInMills() != null) { + return Pair.of(authConfig.expiresInMills(), TimeUnit.SECONDS); + } + } + + return null; + } + + private AuthConfig refreshExpiredToken(RESTClient client) { + // todo: update the token + return new AuthConfig("token", config.keepRefreshed(), null, this.config.expiresInMills()); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java new file mode 100644 index 000000000000..c8db55683a64 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public class AuthUtil { + private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final String BEARER_PREFIX = "Bearer "; + + public static Map authHeaders(String token) { + if (token != null) { + return ImmutableMap.of(AUTHORIZATION_HEADER, BEARER_PREFIX + token); + } else { + return ImmutableMap.of(); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java index 1140e399824c..639c125dfd1d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java @@ -18,7 +18,8 @@ package org.apache.paimon.rest; -import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; +import org.apache.paimon.rest.auth.AuthUtil; + import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import okhttp3.mockwebserver.MockResponse; @@ -33,8 +34,6 @@ import java.util.Map; import java.util.Optional; -import static org.apache.paimon.rest.RESTCatalog.AUTH_HEADER; -import static org.apache.paimon.rest.RESTCatalog.AUTH_HEADER_VALUE_FORMAT; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -70,7 +69,7 @@ public void setUp() throws IOException { mockResponseData = new MockRESTData(MOCK_PATH); mockResponseDataStr = objectMapper.writeValueAsString(mockResponseData); httpClient = new HttpClient(httpClientOptions); - headers = ImmutableMap.of(AUTH_HEADER, String.format(AUTH_HEADER_VALUE_FORMAT, TOKEN)); + headers = AuthUtil.authHeaders(TOKEN); } @After From d2e120b5629824c0a353f81c8f28f2aa5cf8a4c8 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 2 Dec 2024 11:26:06 +0800 Subject: [PATCH 02/26] fix checkstyle fail --- .../org/apache/paimon/rest/RESTCatalog.java | 4 +-- .../{AuthConfig.java => AuthOptions.java} | 5 ++-- .../apache/paimon/rest/auth/AuthSession.java | 26 +++++++++---------- .../org/apache/paimon/rest/auth/AuthUtil.java | 1 + 4 files changed, 19 insertions(+), 17 deletions(-) rename paimon-core/src/main/java/org/apache/paimon/rest/auth/{AuthConfig.java => AuthOptions.java} (95%) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 39b8e579bca7..19f5047e986c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -26,7 +26,7 @@ import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.rest.auth.AuthConfig; +import org.apache.paimon.rest.auth.AuthOptions; import org.apache.paimon.rest.auth.AuthSession; import org.apache.paimon.rest.auth.AuthUtil; import org.apache.paimon.rest.responses.ConfigResponse; @@ -94,7 +94,7 @@ public RESTCatalog(Options options) { token, this.baseHeader, // todo: update,fix null value - new AuthConfig(token, keepTokenRefreshed, null, null), + new AuthOptions(token, keepTokenRefreshed, null, null), null); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java similarity index 95% rename from paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java rename to paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java index 74348cd1b475..14c1e2a541a5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java @@ -20,13 +20,14 @@ import javax.annotation.Nullable; -public class AuthConfig { +/** Auth options. */ +public class AuthOptions { private final @Nullable String token; private final Boolean keepRefreshed; private final @Nullable Long expiresAtMillis; private final @Nullable Long expiresInMills; - public AuthConfig( + public AuthOptions( @Nullable String token, boolean keepRefreshed, Long expiresAtMillis, diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index ff0fc5d87beb..279b14380e72 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -34,9 +34,9 @@ public class AuthSession { private static final long MAX_REFRESH_WINDOW_MILLIS = 300_000; // 5 minutes private static final long MIN_REFRESH_WAIT_MILLIS = 10; private volatile Map headers; - private volatile AuthConfig config; + private volatile AuthOptions config; - public AuthSession(Map headers, AuthConfig config) { + public AuthSession(Map headers, AuthOptions config) { this.headers = headers; this.config = config; } @@ -46,7 +46,7 @@ public static AuthSession fromAccessToken( ScheduledExecutorService executor, String token, Map headers, - AuthConfig config, + AuthOptions config, Long defaultExpiresAtMillis) { AuthSession session = new AuthSession(headers, config); @@ -112,31 +112,31 @@ private static void scheduleTokenRefresh( public Pair refresh(RESTClient client) { if (config.token() != null && config.keepRefreshed()) { long startTimeMillis = System.currentTimeMillis(); - AuthConfig authConfig = refreshExpiredToken(client); - boolean isSuccessful = authConfig.token() != null; + AuthOptions authOptions = refreshExpiredToken(client); + boolean isSuccessful = authOptions.token() != null; if (!isSuccessful) { return null; } - long expiresAtMillis = startTimeMillis + authConfig.expiresInMills(); + long expiresAtMillis = startTimeMillis + authOptions.expiresInMills(); this.config = - new AuthConfig( - authConfig.token(), + new AuthOptions( + authOptions.token(), config.keepRefreshed(), expiresAtMillis, - authConfig.expiresInMills()); + authOptions.expiresInMills()); Map currentHeaders = this.headers; this.headers = RESTUtil.merge(currentHeaders, authHeaders(config.token())); - if (authConfig.expiresInMills() != null) { - return Pair.of(authConfig.expiresInMills(), TimeUnit.SECONDS); + if (authOptions.expiresInMills() != null) { + return Pair.of(authOptions.expiresInMills(), TimeUnit.SECONDS); } } return null; } - private AuthConfig refreshExpiredToken(RESTClient client) { + private AuthOptions refreshExpiredToken(RESTClient client) { // todo: update the token - return new AuthConfig("token", config.keepRefreshed(), null, this.config.expiresInMills()); + return new AuthOptions("token", config.keepRefreshed(), null, this.config.expiresInMills()); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java index c8db55683a64..09f9c8e08070 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java @@ -22,6 +22,7 @@ import java.util.Map; +/** Auth util. */ public class AuthUtil { private static final String AUTHORIZATION_HEADER = "Authorization"; private static final String BEARER_PREFIX = "Bearer "; From 1cc75d95a01d959aa325f350deaefaf01902c199 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 2 Dec 2024 14:16:35 +0800 Subject: [PATCH 03/26] add auth options for conf --- .../org/apache/paimon/rest/RESTCatalog.java | 4 +- .../apache/paimon/rest/auth/AuthConfig.java | 57 +++++++++++++++++++ .../apache/paimon/rest/auth/AuthOptions.java | 57 ++++++++----------- .../apache/paimon/rest/auth/AuthSession.java | 26 ++++----- 4 files changed, 96 insertions(+), 48 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 19f5047e986c..39b8e579bca7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -26,7 +26,7 @@ import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.rest.auth.AuthOptions; +import org.apache.paimon.rest.auth.AuthConfig; import org.apache.paimon.rest.auth.AuthSession; import org.apache.paimon.rest.auth.AuthUtil; import org.apache.paimon.rest.responses.ConfigResponse; @@ -94,7 +94,7 @@ public RESTCatalog(Options options) { token, this.baseHeader, // todo: update,fix null value - new AuthOptions(token, keepTokenRefreshed, null, null), + new AuthConfig(token, keepTokenRefreshed, null, null), null); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java new file mode 100644 index 000000000000..53f363b58598 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import javax.annotation.Nullable; + +/** Auth options. */ +public class AuthConfig { + private final @Nullable String token; + private final Boolean keepRefreshed; + private final @Nullable Long expiresAtMillis; + private final @Nullable Long expiresInMills; + + public AuthConfig( + @Nullable String token, + boolean keepRefreshed, + Long expiresAtMillis, + Long expiresInMills) { + this.token = token; + this.keepRefreshed = keepRefreshed; + this.expiresAtMillis = expiresAtMillis; + this.expiresInMills = expiresInMills; + } + + @Nullable + public String token() { + return token; + } + + public boolean keepRefreshed() { + return keepRefreshed; + } + + public Long expiresAtMillis() { + return expiresAtMillis; + } + + public Long expiresInMills() { + return expiresInMills; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java index 14c1e2a541a5..3697e54034fd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java @@ -18,40 +18,31 @@ package org.apache.paimon.rest.auth; -import javax.annotation.Nullable; +import org.apache.paimon.options.ConfigOption; +import org.apache.paimon.options.ConfigOptions; + +import java.time.Duration; /** Auth options. */ public class AuthOptions { - private final @Nullable String token; - private final Boolean keepRefreshed; - private final @Nullable Long expiresAtMillis; - private final @Nullable Long expiresInMills; - - public AuthOptions( - @Nullable String token, - boolean keepRefreshed, - Long expiresAtMillis, - Long expiresInMills) { - this.token = token; - this.keepRefreshed = keepRefreshed; - this.expiresAtMillis = expiresAtMillis; - this.expiresInMills = expiresInMills; - } - - @Nullable - public String token() { - return token; - } - - public boolean keepRefreshed() { - return keepRefreshed; - } - - public Long expiresAtMillis() { - return expiresAtMillis; - } - - public Long expiresInMills() { - return expiresInMills; - } + public static final ConfigOption TOKEN = + ConfigOptions.key("token") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth token."); + public static final ConfigOption TOKEN_EXPIRES = + ConfigOptions.key("token-expires") + .durationType() + .defaultValue(Duration.ofHours(1)) + .withDescription("REST Catalog auth token expires duration."); + public static final ConfigOption TOKEN_REFRESH_ENABLED = + ConfigOptions.key("token-refresh-enabled") + .booleanType() + .defaultValue(false) + .withDescription("REST Catalog auth token refresh enable."); + public static final ConfigOption TOKEN_FILE_PATH = + ConfigOptions.key("token-file-path") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth token file path."); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index 279b14380e72..ff0fc5d87beb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -34,9 +34,9 @@ public class AuthSession { private static final long MAX_REFRESH_WINDOW_MILLIS = 300_000; // 5 minutes private static final long MIN_REFRESH_WAIT_MILLIS = 10; private volatile Map headers; - private volatile AuthOptions config; + private volatile AuthConfig config; - public AuthSession(Map headers, AuthOptions config) { + public AuthSession(Map headers, AuthConfig config) { this.headers = headers; this.config = config; } @@ -46,7 +46,7 @@ public static AuthSession fromAccessToken( ScheduledExecutorService executor, String token, Map headers, - AuthOptions config, + AuthConfig config, Long defaultExpiresAtMillis) { AuthSession session = new AuthSession(headers, config); @@ -112,31 +112,31 @@ private static void scheduleTokenRefresh( public Pair refresh(RESTClient client) { if (config.token() != null && config.keepRefreshed()) { long startTimeMillis = System.currentTimeMillis(); - AuthOptions authOptions = refreshExpiredToken(client); - boolean isSuccessful = authOptions.token() != null; + AuthConfig authConfig = refreshExpiredToken(client); + boolean isSuccessful = authConfig.token() != null; if (!isSuccessful) { return null; } - long expiresAtMillis = startTimeMillis + authOptions.expiresInMills(); + long expiresAtMillis = startTimeMillis + authConfig.expiresInMills(); this.config = - new AuthOptions( - authOptions.token(), + new AuthConfig( + authConfig.token(), config.keepRefreshed(), expiresAtMillis, - authOptions.expiresInMills()); + authConfig.expiresInMills()); Map currentHeaders = this.headers; this.headers = RESTUtil.merge(currentHeaders, authHeaders(config.token())); - if (authOptions.expiresInMills() != null) { - return Pair.of(authOptions.expiresInMills(), TimeUnit.SECONDS); + if (authConfig.expiresInMills() != null) { + return Pair.of(authConfig.expiresInMills(), TimeUnit.SECONDS); } } return null; } - private AuthOptions refreshExpiredToken(RESTClient client) { + private AuthConfig refreshExpiredToken(RESTClient client) { // todo: update the token - return new AuthOptions("token", config.keepRefreshed(), null, this.config.expiresInMills()); + return new AuthConfig("token", config.keepRefreshed(), null, this.config.expiresInMills()); } } From 9ee1aa174389aad29ba4be47e389a72c54897c9c Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 2 Dec 2024 18:01:56 +0800 Subject: [PATCH 04/26] support read token from file --- .../org/apache/paimon/rest/RESTCatalog.java | 23 +++++---- .../apache/paimon/rest/auth/AuthOptions.java | 8 +-- .../apache/paimon/rest/auth/AuthSession.java | 51 +++++++++++-------- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 39b8e579bca7..8e72f648703f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -27,6 +27,7 @@ import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.rest.auth.AuthConfig; +import org.apache.paimon.rest.auth.AuthOptions; import org.apache.paimon.rest.auth.AuthSession; import org.apache.paimon.rest.auth.AuthUtil; import org.apache.paimon.rest.responses.ConfigResponse; @@ -54,8 +55,9 @@ public class RESTCatalog implements Catalog { private Map baseHeader; // a lazy thread pool for token refresh private AuthSession catalogAuth = null; + private String tokenFilePath = null; private volatile ScheduledExecutorService refreshExecutor = null; - private boolean keepTokenRefreshed = true; + private boolean keepTokenRefreshed; private static final ObjectMapper objectMapper = RESTObjectMapper.create(); @@ -64,7 +66,6 @@ public RESTCatalog(Options options) { throw new IllegalArgumentException("Can not config warehouse in RESTCatalog."); } String uri = options.get(RESTCatalogOptions.URI); - token = options.get(RESTCatalogOptions.TOKEN); Optional connectTimeout = options.getOptional(RESTCatalogOptions.CONNECTION_TIMEOUT); Optional readTimeout = options.getOptional(RESTCatalogOptions.READ_TIMEOUT); @@ -78,6 +79,8 @@ public RESTCatalog(Options options) { threadPoolSize, DefaultErrorHandler.getInstance()); this.client = new HttpClient(httpClientOptions); + token = options.get(RESTCatalogOptions.TOKEN); + this.keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); Map authHeaders = AuthUtil.authHeaders(token); Map initHeaders = RESTUtil.merge(configHeaders(options.toMap()), authHeaders); @@ -86,16 +89,18 @@ public RESTCatalog(Options options) { this.resourcePaths = ResourcePaths.forCatalogProperties( this.options.get(RESTCatalogInternalOptions.PREFIX)); - this.keepTokenRefreshed = false; + long tokenExpireInMills = options.get(AuthOptions.TOKEN_EXPIRES_IN).toMillis(); + long tokenExpireAtMills = System.currentTimeMillis() + tokenExpireInMills; + this.tokenFilePath = options.get(AuthOptions.TOKEN_FILE_PATH); this.catalogAuth = - AuthSession.fromAccessToken( - client, + AuthSession.fromTokenPath( + tokenFilePath, tokenRefreshExecutor(), - token, this.baseHeader, // todo: update,fix null value - new AuthConfig(token, keepTokenRefreshed, null, null), - null); + new AuthConfig( + token, keepTokenRefreshed, tokenExpireAtMills, tokenExpireInMills), + tokenExpireAtMills); } @Override @@ -212,7 +217,7 @@ private static Map configHeaders(Map properties) } private Map headers() { - catalogAuth.refresh(client); + catalogAuth.refresh(this.tokenFilePath); return catalogAuth.getHeaders(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java index 3697e54034fd..28118b2b50f1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java @@ -23,18 +23,18 @@ import java.time.Duration; -/** Auth options. */ +/** Options for REST Catalog Auth. */ public class AuthOptions { public static final ConfigOption TOKEN = ConfigOptions.key("token") .stringType() .noDefaultValue() .withDescription("REST Catalog auth token."); - public static final ConfigOption TOKEN_EXPIRES = - ConfigOptions.key("token-expires") + public static final ConfigOption TOKEN_EXPIRES_IN = + ConfigOptions.key("token-expires-in") .durationType() .defaultValue(Duration.ofHours(1)) - .withDescription("REST Catalog auth token expires duration."); + .withDescription("REST Catalog auth token expires in."); public static final ConfigOption TOKEN_REFRESH_ENABLED = ConfigOptions.key("token-refresh-enabled") .booleanType() diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index ff0fc5d87beb..e2d130720b51 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -18,10 +18,13 @@ package org.apache.paimon.rest.auth; -import org.apache.paimon.rest.RESTClient; import org.apache.paimon.rest.RESTUtil; import org.apache.paimon.utils.Pair; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Map; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -41,10 +44,9 @@ public AuthSession(Map headers, AuthConfig config) { this.config = config; } - public static AuthSession fromAccessToken( - RESTClient client, + public static AuthSession fromTokenPath( + String tokeFilePath, ScheduledExecutorService executor, - String token, Map headers, AuthConfig config, Long defaultExpiresAtMillis) { @@ -54,7 +56,7 @@ public static AuthSession fromAccessToken( Long expiresAtMillis = session.config.expiresAtMillis(); if (null != expiresAtMillis && expiresAtMillis <= startTimeMillis) { - Pair expiration = session.refresh(client); + Pair expiration = session.refresh(tokeFilePath); // if expiration is non-null, then token refresh was successful if (expiration != null) { if (null != config.expiresAtMillis()) { @@ -73,7 +75,7 @@ public static AuthSession fromAccessToken( } if (null != executor && null != expiresAtMillis) { - scheduleTokenRefresh(client, executor, session, expiresAtMillis); + scheduleTokenRefresh(tokeFilePath, executor, session, expiresAtMillis); } return session; @@ -84,7 +86,7 @@ public Map getHeaders() { } private static void scheduleTokenRefresh( - RESTClient client, + String tokeFilePath, ScheduledExecutorService executor, AuthSession session, long expiresAtMillis) { @@ -99,31 +101,27 @@ private static void scheduleTokenRefresh( executor.schedule( () -> { long refreshStartTime = System.currentTimeMillis(); - Pair expiration = session.refresh(client); + Pair expiration = session.refresh(tokeFilePath); if (expiration != null) { scheduleTokenRefresh( - client, executor, session, refreshStartTime + expiration.getKey()); + tokeFilePath, + executor, + session, + refreshStartTime + expiration.getKey()); } }, timeToWait, TimeUnit.MILLISECONDS); } - public Pair refresh(RESTClient client) { + public Pair refresh(String tokenFilePath) { if (config.token() != null && config.keepRefreshed()) { - long startTimeMillis = System.currentTimeMillis(); - AuthConfig authConfig = refreshExpiredToken(client); + AuthConfig authConfig = refreshExpiredToken(tokenFilePath, System.currentTimeMillis()); boolean isSuccessful = authConfig.token() != null; if (!isSuccessful) { return null; } - long expiresAtMillis = startTimeMillis + authConfig.expiresInMills(); - this.config = - new AuthConfig( - authConfig.token(), - config.keepRefreshed(), - expiresAtMillis, - authConfig.expiresInMills()); + this.config = authConfig; Map currentHeaders = this.headers; this.headers = RESTUtil.merge(currentHeaders, authHeaders(config.token())); @@ -135,8 +133,17 @@ public Pair refresh(RESTClient client) { return null; } - private AuthConfig refreshExpiredToken(RESTClient client) { - // todo: update the token - return new AuthConfig("token", config.keepRefreshed(), null, this.config.expiresInMills()); + private AuthConfig refreshExpiredToken(String tokenFilePath, long startTimeMillis) { + try { + // todo: handle exception + String token = + new String( + Files.readAllBytes(Paths.get(tokenFilePath)), StandardCharsets.UTF_8); + long expiresAtMillis = startTimeMillis + this.config.expiresInMills(); + return new AuthConfig( + token, config.keepRefreshed(), expiresAtMillis, this.config.expiresInMills()); + } catch (IOException e) { + throw new RuntimeException(e); + } } } From 94c1ab355a46d87d66aac57ca0d90d1ec1f155b3 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 2 Dec 2024 18:18:17 +0800 Subject: [PATCH 05/26] support diff auth session when config is diff --- .../org/apache/paimon/rest/RESTCatalog.java | 34 +++++++++++-------- .../apache/paimon/rest/auth/AuthConfig.java | 8 +++++ .../apache/paimon/rest/auth/AuthSession.java | 12 +++++-- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 8e72f648703f..4a4512e1bce5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -55,7 +55,6 @@ public class RESTCatalog implements Catalog { private Map baseHeader; // a lazy thread pool for token refresh private AuthSession catalogAuth = null; - private String tokenFilePath = null; private volatile ScheduledExecutorService refreshExecutor = null; private boolean keepTokenRefreshed; @@ -89,18 +88,25 @@ public RESTCatalog(Options options) { this.resourcePaths = ResourcePaths.forCatalogProperties( this.options.get(RESTCatalogInternalOptions.PREFIX)); - long tokenExpireInMills = options.get(AuthOptions.TOKEN_EXPIRES_IN).toMillis(); - long tokenExpireAtMills = System.currentTimeMillis() + tokenExpireInMills; - this.tokenFilePath = options.get(AuthOptions.TOKEN_FILE_PATH); - this.catalogAuth = - AuthSession.fromTokenPath( - tokenFilePath, - tokenRefreshExecutor(), - this.baseHeader, - // todo: update,fix null value - new AuthConfig( - token, keepTokenRefreshed, tokenExpireAtMills, tokenExpireInMills), - tokenExpireAtMills); + this.catalogAuth = new AuthSession(this.baseHeader, null); + if (options.getOptional(AuthOptions.TOKEN_FILE_PATH).isPresent()) { + long tokenExpireInMills = options.get(AuthOptions.TOKEN_EXPIRES_IN).toMillis(); + String tokenFilePath = options.getOptional(AuthOptions.TOKEN_FILE_PATH).orElse(null); + long tokenExpireAtMills = System.currentTimeMillis() + tokenExpireInMills; + this.catalogAuth = + AuthSession.fromTokenPath( + tokenFilePath, + tokenRefreshExecutor(), + this.baseHeader, + // todo: update,fix null value + new AuthConfig( + token, + tokenFilePath, + keepTokenRefreshed, + tokenExpireAtMills, + tokenExpireInMills), + tokenExpireAtMills); + } } @Override @@ -217,7 +223,7 @@ private static Map configHeaders(Map properties) } private Map headers() { - catalogAuth.refresh(this.tokenFilePath); + catalogAuth.refresh(); return catalogAuth.getHeaders(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java index 53f363b58598..3daab89a2fc1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java @@ -23,16 +23,19 @@ /** Auth options. */ public class AuthConfig { private final @Nullable String token; + private final @Nullable String tokenFilePath; private final Boolean keepRefreshed; private final @Nullable Long expiresAtMillis; private final @Nullable Long expiresInMills; public AuthConfig( @Nullable String token, + @Nullable String tokenFilePath, boolean keepRefreshed, Long expiresAtMillis, Long expiresInMills) { this.token = token; + this.tokenFilePath = tokenFilePath; this.keepRefreshed = keepRefreshed; this.expiresAtMillis = expiresAtMillis; this.expiresInMills = expiresInMills; @@ -43,6 +46,11 @@ public String token() { return token; } + @Nullable + public String tokenFilePath() { + return tokenFilePath; + } + public boolean keepRefreshed() { return keepRefreshed; } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index e2d130720b51..ec6756546a0a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -114,7 +114,11 @@ private static void scheduleTokenRefresh( TimeUnit.MILLISECONDS); } - public Pair refresh(String tokenFilePath) { + public Pair refresh() { + return refresh(config.tokenFilePath()); + } + + private Pair refresh(String tokenFilePath) { if (config.token() != null && config.keepRefreshed()) { AuthConfig authConfig = refreshExpiredToken(tokenFilePath, System.currentTimeMillis()); boolean isSuccessful = authConfig.token() != null; @@ -141,7 +145,11 @@ private AuthConfig refreshExpiredToken(String tokenFilePath, long startTimeMilli Files.readAllBytes(Paths.get(tokenFilePath)), StandardCharsets.UTF_8); long expiresAtMillis = startTimeMillis + this.config.expiresInMills(); return new AuthConfig( - token, config.keepRefreshed(), expiresAtMillis, this.config.expiresInMills()); + token, + config.tokenFilePath(), + config.keepRefreshed(), + expiresAtMillis, + this.config.expiresInMills()); } catch (IOException e) { throw new RuntimeException(e); } From c4e98e8f4df479ddcabbbe1ecaf76c54989d12c6 Mon Sep 17 00:00:00 2001 From: yantian Date: Wed, 4 Dec 2024 13:49:48 +0800 Subject: [PATCH 06/26] add CredentialsProvider to support diff credential --- .../org/apache/paimon/rest/RESTCatalog.java | 7 ++-- .../apache/paimon/rest/auth/AuthSession.java | 8 +++-- ...java => BearTokenCredentialsProvider.java} | 27 ++++++++++++---- .../paimon/rest/auth/CredentialsProvider.java | 32 +++++++++++++++++++ .../apache/paimon/rest/HttpClientTest.java | 6 ++-- 5 files changed, 67 insertions(+), 13 deletions(-) rename paimon-core/src/main/java/org/apache/paimon/rest/auth/{AuthUtil.java => BearTokenCredentialsProvider.java} (64%) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 4a4512e1bce5..6d4ec8c72898 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -29,7 +29,8 @@ import org.apache.paimon.rest.auth.AuthConfig; import org.apache.paimon.rest.auth.AuthOptions; import org.apache.paimon.rest.auth.AuthSession; -import org.apache.paimon.rest.auth.AuthUtil; +import org.apache.paimon.rest.auth.BearTokenCredentialsProvider; +import org.apache.paimon.rest.auth.CredentialsProvider; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; @@ -80,7 +81,9 @@ public RESTCatalog(Options options) { this.client = new HttpClient(httpClientOptions); token = options.get(RESTCatalogOptions.TOKEN); this.keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); - Map authHeaders = AuthUtil.authHeaders(token); + // todo: update + CredentialsProvider credentialsProvider = new BearTokenCredentialsProvider(token); + Map authHeaders = credentialsProvider.authHeader(); Map initHeaders = RESTUtil.merge(configHeaders(options.toMap()), authHeaders); this.options = fetchOptionsFromServer(initHeaders, options.toMap()); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index ec6756546a0a..d4ba9df2c085 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -29,8 +29,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import static org.apache.paimon.rest.auth.AuthUtil.authHeaders; - /** Auth session. */ public class AuthSession { private static int tokenRefreshNumRetries = 5; @@ -127,7 +125,11 @@ private Pair refresh(String tokenFilePath) { } this.config = authConfig; Map currentHeaders = this.headers; - this.headers = RESTUtil.merge(currentHeaders, authHeaders(config.token())); + // todo: fixme + this.headers = + RESTUtil.merge( + currentHeaders, + new BearTokenCredentialsProvider(authConfig.token()).authHeader()); if (authConfig.expiresInMills() != null) { return Pair.of(authConfig.expiresInMills(), TimeUnit.SECONDS); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java similarity index 64% rename from paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java rename to paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java index 09f9c8e08070..2e47a04b60c6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthUtil.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java @@ -18,20 +18,35 @@ package org.apache.paimon.rest.auth; +import org.apache.paimon.utils.StringUtils; + import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; import java.util.Map; -/** Auth util. */ -public class AuthUtil { +/** credentials provider for bear token. */ +public class BearTokenCredentialsProvider implements CredentialsProvider { + private static final String AUTHORIZATION_HEADER = "Authorization"; private static final String BEARER_PREFIX = "Bearer "; - public static Map authHeaders(String token) { - if (token != null) { - return ImmutableMap.of(AUTHORIZATION_HEADER, BEARER_PREFIX + token); + private final String token; + + public BearTokenCredentialsProvider(String token) { + if (StringUtils.isNullOrWhitespaceOnly(token)) { + throw new IllegalArgumentException("token is null"); } else { - return ImmutableMap.of(); + this.token = token; } } + + @Override + public Map authHeader() { + return ImmutableMap.of(AUTHORIZATION_HEADER, BEARER_PREFIX + token); + } + + @Override + public void refresh() { + // do nothing + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java new file mode 100644 index 000000000000..c369c2f51a19 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import java.util.Map; + +/** Credentials provider. */ +public interface CredentialsProvider { + Map authHeader(); + + void refresh(); + + default boolean supportRefresh() { + return false; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java index 639c125dfd1d..17c13b932fd2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java @@ -18,7 +18,8 @@ package org.apache.paimon.rest; -import org.apache.paimon.rest.auth.AuthUtil; +import org.apache.paimon.rest.auth.BearTokenCredentialsProvider; +import org.apache.paimon.rest.auth.CredentialsProvider; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -69,7 +70,8 @@ public void setUp() throws IOException { mockResponseData = new MockRESTData(MOCK_PATH); mockResponseDataStr = objectMapper.writeValueAsString(mockResponseData); httpClient = new HttpClient(httpClientOptions); - headers = AuthUtil.authHeaders(TOKEN); + CredentialsProvider credentialsProvider = new BearTokenCredentialsProvider(TOKEN); + headers = credentialsProvider.authHeader(); } @After From 6f3391a441fa603bdbe9f6812d7f84bf00a5c1c5 Mon Sep 17 00:00:00 2001 From: yantian Date: Wed, 4 Dec 2024 14:44:38 +0800 Subject: [PATCH 07/26] support credentials provider --- .../org/apache/paimon/rest/RESTCatalog.java | 17 ++-- .../apache/paimon/rest/auth/AuthConfig.java | 65 ------------- .../apache/paimon/rest/auth/AuthSession.java | 93 ++++++------------ .../BaseBearTokenCredentialsProvider.java | 36 +++++++ .../auth/BearTokenCredentialsProvider.java | 17 +--- .../BearTokenFileCredentialsProvider.java | 97 +++++++++++++++++++ .../paimon/rest/auth/CredentialsProvider.java | 15 ++- 7 files changed, 189 insertions(+), 151 deletions(-) delete mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 6d4ec8c72898..9b27281ad534 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -26,10 +26,10 @@ import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.rest.auth.AuthConfig; import org.apache.paimon.rest.auth.AuthOptions; import org.apache.paimon.rest.auth.AuthSession; import org.apache.paimon.rest.auth.BearTokenCredentialsProvider; +import org.apache.paimon.rest.auth.BearTokenFileCredentialsProvider; import org.apache.paimon.rest.auth.CredentialsProvider; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.schema.Schema; @@ -96,18 +96,17 @@ public RESTCatalog(Options options) { long tokenExpireInMills = options.get(AuthOptions.TOKEN_EXPIRES_IN).toMillis(); String tokenFilePath = options.getOptional(AuthOptions.TOKEN_FILE_PATH).orElse(null); long tokenExpireAtMills = System.currentTimeMillis() + tokenExpireInMills; + credentialsProvider = + new BearTokenFileCredentialsProvider( + tokenFilePath, + keepTokenRefreshed, + tokenExpireAtMills, + tokenExpireInMills); this.catalogAuth = AuthSession.fromTokenPath( - tokenFilePath, tokenRefreshExecutor(), this.baseHeader, - // todo: update,fix null value - new AuthConfig( - token, - tokenFilePath, - keepTokenRefreshed, - tokenExpireAtMills, - tokenExpireInMills), + credentialsProvider, tokenExpireAtMills); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java deleted file mode 100644 index 3daab89a2fc1..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthConfig.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.auth; - -import javax.annotation.Nullable; - -/** Auth options. */ -public class AuthConfig { - private final @Nullable String token; - private final @Nullable String tokenFilePath; - private final Boolean keepRefreshed; - private final @Nullable Long expiresAtMillis; - private final @Nullable Long expiresInMills; - - public AuthConfig( - @Nullable String token, - @Nullable String tokenFilePath, - boolean keepRefreshed, - Long expiresAtMillis, - Long expiresInMills) { - this.token = token; - this.tokenFilePath = tokenFilePath; - this.keepRefreshed = keepRefreshed; - this.expiresAtMillis = expiresAtMillis; - this.expiresInMills = expiresInMills; - } - - @Nullable - public String token() { - return token; - } - - @Nullable - public String tokenFilePath() { - return tokenFilePath; - } - - public boolean keepRefreshed() { - return keepRefreshed; - } - - public Long expiresAtMillis() { - return expiresAtMillis; - } - - public Long expiresInMills() { - return expiresInMills; - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index d4ba9df2c085..25a3a1c142e4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -21,11 +21,8 @@ import org.apache.paimon.rest.RESTUtil; import org.apache.paimon.utils.Pair; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -35,45 +32,47 @@ public class AuthSession { private static final long MAX_REFRESH_WINDOW_MILLIS = 300_000; // 5 minutes private static final long MIN_REFRESH_WAIT_MILLIS = 10; private volatile Map headers; - private volatile AuthConfig config; + private final CredentialsProvider credentialsProvider; - public AuthSession(Map headers, AuthConfig config) { + public AuthSession(Map headers, CredentialsProvider credentialsProvider) { this.headers = headers; - this.config = config; + this.credentialsProvider = credentialsProvider; } public static AuthSession fromTokenPath( - String tokeFilePath, ScheduledExecutorService executor, Map headers, - AuthConfig config, + CredentialsProvider credentialsProvider, Long defaultExpiresAtMillis) { - AuthSession session = new AuthSession(headers, config); + AuthSession session = new AuthSession(headers, credentialsProvider); long startTimeMillis = System.currentTimeMillis(); - Long expiresAtMillis = session.config.expiresAtMillis(); + Optional expiresAtMillisOpt = credentialsProvider.expiresAtMillis(); + + if (expiresAtMillisOpt.isPresent() + && null != expiresAtMillisOpt.get() + && expiresAtMillisOpt.get() <= startTimeMillis) { + Pair expiration = session.refresh(); - if (null != expiresAtMillis && expiresAtMillis <= startTimeMillis) { - Pair expiration = session.refresh(tokeFilePath); // if expiration is non-null, then token refresh was successful if (expiration != null) { - if (null != config.expiresAtMillis()) { + if (session.credentialsProvider.expiresAtMillis().isPresent()) { // use the new expiration time from the refreshed token - expiresAtMillis = config.expiresAtMillis(); + expiresAtMillisOpt = session.credentialsProvider.expiresAtMillis(); } else { // otherwise use the expiration time from the token response - expiresAtMillis = startTimeMillis + expiration.getKey(); + expiresAtMillisOpt = Optional.of(startTimeMillis + expiration.getKey()); } } else { // token refresh failed, don't reattempt with the original expiration - expiresAtMillis = null; + expiresAtMillisOpt = Optional.empty(); } - } else if (null == expiresAtMillis && defaultExpiresAtMillis != null) { - expiresAtMillis = defaultExpiresAtMillis; + } else if (expiresAtMillisOpt.isPresent() && defaultExpiresAtMillis != null) { + expiresAtMillisOpt = Optional.of(defaultExpiresAtMillis); } - if (null != executor && null != expiresAtMillis) { - scheduleTokenRefresh(tokeFilePath, executor, session, expiresAtMillis); + if (null != executor && expiresAtMillisOpt.isPresent()) { + scheduleTokenRefresh(executor, session, expiresAtMillisOpt.get()); } return session; @@ -84,10 +83,7 @@ public Map getHeaders() { } private static void scheduleTokenRefresh( - String tokeFilePath, - ScheduledExecutorService executor, - AuthSession session, - long expiresAtMillis) { + ScheduledExecutorService executor, AuthSession session, long expiresAtMillis) { long expiresInMillis = expiresAtMillis - System.currentTimeMillis(); // how much ahead of time to start the request to allow it to complete long refreshWindowMillis = Math.min(expiresInMillis / 10, MAX_REFRESH_WINDOW_MILLIS); @@ -99,13 +95,10 @@ private static void scheduleTokenRefresh( executor.schedule( () -> { long refreshStartTime = System.currentTimeMillis(); - Pair expiration = session.refresh(tokeFilePath); + Pair expiration = session.refresh(); if (expiration != null) { scheduleTokenRefresh( - tokeFilePath, - executor, - session, - refreshStartTime + expiration.getKey()); + executor, session, refreshStartTime + expiration.getKey()); } }, timeToWait, @@ -113,47 +106,19 @@ private static void scheduleTokenRefresh( } public Pair refresh() { - return refresh(config.tokenFilePath()); - } - - private Pair refresh(String tokenFilePath) { - if (config.token() != null && config.keepRefreshed()) { - AuthConfig authConfig = refreshExpiredToken(tokenFilePath, System.currentTimeMillis()); - boolean isSuccessful = authConfig.token() != null; + if (this.credentialsProvider.supportRefresh() && this.credentialsProvider.keepRefreshed()) { + boolean isSuccessful = this.credentialsProvider.refresh(); if (!isSuccessful) { return null; } - this.config = authConfig; Map currentHeaders = this.headers; - // todo: fixme - this.headers = - RESTUtil.merge( - currentHeaders, - new BearTokenCredentialsProvider(authConfig.token()).authHeader()); - - if (authConfig.expiresInMills() != null) { - return Pair.of(authConfig.expiresInMills(), TimeUnit.SECONDS); + this.headers = RESTUtil.merge(currentHeaders, this.credentialsProvider.authHeader()); + + if (credentialsProvider.expiresInMills().isPresent()) { + return Pair.of(credentialsProvider.expiresInMills().get(), TimeUnit.SECONDS); } } return null; } - - private AuthConfig refreshExpiredToken(String tokenFilePath, long startTimeMillis) { - try { - // todo: handle exception - String token = - new String( - Files.readAllBytes(Paths.get(tokenFilePath)), StandardCharsets.UTF_8); - long expiresAtMillis = startTimeMillis + this.config.expiresInMills(); - return new AuthConfig( - token, - config.tokenFilePath(), - config.keepRefreshed(), - expiresAtMillis, - this.config.expiresInMills()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java new file mode 100644 index 000000000000..a7f1a2b459f1 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; + +import java.util.Map; + +/** Base bear token credentials provider. */ +public abstract class BaseBearTokenCredentialsProvider implements CredentialsProvider { + private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final String BEARER_PREFIX = "Bearer "; + + @Override + public Map authHeader() { + return ImmutableMap.of(AUTHORIZATION_HEADER, BEARER_PREFIX + token()); + } + + abstract String token(); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java index 2e47a04b60c6..dfe9725fbf39 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java @@ -20,15 +20,8 @@ import org.apache.paimon.utils.StringUtils; -import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; - -import java.util.Map; - /** credentials provider for bear token. */ -public class BearTokenCredentialsProvider implements CredentialsProvider { - - private static final String AUTHORIZATION_HEADER = "Authorization"; - private static final String BEARER_PREFIX = "Bearer "; +public class BearTokenCredentialsProvider extends BaseBearTokenCredentialsProvider { private final String token; @@ -41,12 +34,12 @@ public BearTokenCredentialsProvider(String token) { } @Override - public Map authHeader() { - return ImmutableMap.of(AUTHORIZATION_HEADER, BEARER_PREFIX + token); + String token() { + return this.token; } @Override - public void refresh() { - // do nothing + public boolean refresh() { + return true; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java new file mode 100644 index 000000000000..ea5d5a650970 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.utils.StringUtils; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Optional; + +/** credentials provider for get bear token from file. */ +public class BearTokenFileCredentialsProvider extends BaseBearTokenCredentialsProvider { + private final String tokenFilePath; + private String token; + private boolean keepRefreshed = false; + private Long expiresAtMillis = null; + private Long expiresInMills = null; + + public BearTokenFileCredentialsProvider(String tokenFilePath) { + this.tokenFilePath = tokenFilePath; + this.token = getTokenFromFile(); + } + + public BearTokenFileCredentialsProvider( + String tokenFilePath, + boolean keepRefreshed, + Long expiresAtMillis, + Long expiresInMills) { + this(tokenFilePath); + this.keepRefreshed = keepRefreshed; + this.expiresAtMillis = expiresAtMillis; + this.expiresInMills = expiresInMills; + } + + @Override + String token() { + return this.token; + } + + @Override + public boolean refresh() { + long start = System.currentTimeMillis(); + this.token = getTokenFromFile(); + this.expiresAtMillis = start + this.expiresInMills; + if (StringUtils.isNullOrWhitespaceOnly(this.token)) { + return false; + } + return true; + } + + @Override + public boolean supportRefresh() { + return true; + } + + @Override + public boolean keepRefreshed() { + return this.keepRefreshed; + } + + @Override + public Optional expiresAtMillis() { + return Optional.ofNullable(this.expiresAtMillis); + } + + @Override + public Optional expiresInMills() { + return Optional.ofNullable(this.expiresInMills); + } + + private String getTokenFromFile() { + try { + // todo: handle exception + return new String(Files.readAllBytes(Paths.get(tokenFilePath)), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java index c369c2f51a19..b6bfa589282f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java @@ -19,14 +19,27 @@ package org.apache.paimon.rest.auth; import java.util.Map; +import java.util.Optional; /** Credentials provider. */ public interface CredentialsProvider { Map authHeader(); - void refresh(); + boolean refresh(); default boolean supportRefresh() { return false; } + + default boolean keepRefreshed() { + return false; + } + + default Optional expiresAtMillis() { + return Optional.empty(); + } + + default Optional expiresInMills() { + return Optional.empty(); + } } From f733f277d0e6143a1df406d5020afaa5e0cad85b Mon Sep 17 00:00:00 2001 From: yantian Date: Wed, 4 Dec 2024 15:05:45 +0800 Subject: [PATCH 08/26] update AuthSession and RESTCatalog --- .../org/apache/paimon/rest/RESTCatalog.java | 35 ++++++++++--------- .../apache/paimon/rest/auth/AuthSession.java | 4 +-- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 9b27281ad534..c4a461e04dfc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -50,7 +50,6 @@ /** A catalog implementation for REST. */ public class RESTCatalog implements Catalog { private RESTClient client; - private String token; private ResourcePaths resourcePaths; private Map options; private Map baseHeader; @@ -79,24 +78,18 @@ public RESTCatalog(Options options) { threadPoolSize, DefaultErrorHandler.getInstance()); this.client = new HttpClient(httpClientOptions); - token = options.get(RESTCatalogOptions.TOKEN); - this.keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); - // todo: update - CredentialsProvider credentialsProvider = new BearTokenCredentialsProvider(token); - Map authHeaders = credentialsProvider.authHeader(); - Map initHeaders = - RESTUtil.merge(configHeaders(options.toMap()), authHeaders); - this.options = fetchOptionsFromServer(initHeaders, options.toMap()); - this.baseHeader = configHeaders(this.options()); - this.resourcePaths = - ResourcePaths.forCatalogProperties( - this.options.get(RESTCatalogInternalOptions.PREFIX)); - this.catalogAuth = new AuthSession(this.baseHeader, null); - if (options.getOptional(AuthOptions.TOKEN_FILE_PATH).isPresent()) { + this.baseHeader = configHeaders(options.toMap()); + // todo: support create CredentialsProvider by conf + if (options.getOptional(RESTCatalogOptions.TOKEN).isPresent()) { + CredentialsProvider credentialsProvider = + new BearTokenCredentialsProvider(options.get(RESTCatalogOptions.TOKEN)); + this.catalogAuth = new AuthSession(this.baseHeader, credentialsProvider); + } else if (options.getOptional(AuthOptions.TOKEN_FILE_PATH).isPresent()) { + this.keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); long tokenExpireInMills = options.get(AuthOptions.TOKEN_EXPIRES_IN).toMillis(); String tokenFilePath = options.getOptional(AuthOptions.TOKEN_FILE_PATH).orElse(null); long tokenExpireAtMills = System.currentTimeMillis() + tokenExpireInMills; - credentialsProvider = + CredentialsProvider credentialsProvider = new BearTokenFileCredentialsProvider( tokenFilePath, keepTokenRefreshed, @@ -109,6 +102,16 @@ public RESTCatalog(Options options) { credentialsProvider, tokenExpireAtMills); } + if (this.catalogAuth != null) { + Map initHeaders = + RESTUtil.merge(configHeaders(options.toMap()), this.catalogAuth.getHeaders()); + this.options = fetchOptionsFromServer(initHeaders, options.toMap()); + this.resourcePaths = + ResourcePaths.forCatalogProperties( + this.options.get(RESTCatalogInternalOptions.PREFIX)); + } else { + throw new IllegalArgumentException("No auth provider provided."); + } } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index 25a3a1c142e4..25038ad5e41e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -49,9 +49,7 @@ public static AuthSession fromTokenPath( long startTimeMillis = System.currentTimeMillis(); Optional expiresAtMillisOpt = credentialsProvider.expiresAtMillis(); - if (expiresAtMillisOpt.isPresent() - && null != expiresAtMillisOpt.get() - && expiresAtMillisOpt.get() <= startTimeMillis) { + if (expiresAtMillisOpt.isPresent() && expiresAtMillisOpt.get() <= startTimeMillis) { Pair expiration = session.refresh(); // if expiration is non-null, then token refresh was successful From e27991d2fb5bb15357366d51b9eb16825f886f78 Mon Sep 17 00:00:00 2001 From: yantian Date: Thu, 5 Dec 2024 16:16:34 +0800 Subject: [PATCH 09/26] update RESTCatalog create auth --- .../apache/paimon/utils/ThreadPoolUtils.java | 10 ++++++ .../org/apache/paimon/rest/RESTCatalog.java | 32 +++++++------------ .../paimon/rest/RESTCatalogOptions.java | 5 --- .../apache/paimon/rest/RESTCatalogTest.java | 3 +- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/ThreadPoolUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/ThreadPoolUtils.java index f8959def67d1..c64b9e26ea6e 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/ThreadPoolUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/ThreadPoolUtils.java @@ -20,6 +20,7 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; +import org.apache.paimon.shade.guava30.com.google.common.util.concurrent.ThreadFactoryBuilder; import javax.annotation.Nullable; @@ -36,6 +37,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -76,6 +79,13 @@ public static ThreadPoolExecutor createCachedThreadPool( return executor; } + public static ScheduledExecutorService createScheduledThreadPool( + int threadNum, String namePrefix) { + return new ScheduledThreadPoolExecutor( + threadNum, + new ThreadFactoryBuilder().setDaemon(true).setNameFormat(namePrefix).build()); + } + /** This method aims to parallel process tasks with memory control and sequentially. */ public static Iterable sequentialBatchedExecute( ThreadPoolExecutor executor, diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index c4a461e04dfc..9bcf10230005 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -37,7 +37,6 @@ import org.apache.paimon.table.Table; import org.apache.paimon.shade.guava30.com.google.common.annotations.VisibleForTesting; -import org.apache.paimon.shade.guava30.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import java.time.Duration; @@ -45,7 +44,8 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; + +import static org.apache.paimon.utils.ThreadPoolUtils.createScheduledThreadPool; /** A catalog implementation for REST. */ public class RESTCatalog implements Catalog { @@ -80,9 +80,9 @@ public RESTCatalog(Options options) { this.client = new HttpClient(httpClientOptions); this.baseHeader = configHeaders(options.toMap()); // todo: support create CredentialsProvider by conf - if (options.getOptional(RESTCatalogOptions.TOKEN).isPresent()) { + if (options.getOptional(AuthOptions.TOKEN).isPresent()) { CredentialsProvider credentialsProvider = - new BearTokenCredentialsProvider(options.get(RESTCatalogOptions.TOKEN)); + new BearTokenCredentialsProvider(options.get(AuthOptions.TOKEN)); this.catalogAuth = new AuthSession(this.baseHeader, credentialsProvider); } else if (options.getOptional(AuthOptions.TOKEN_FILE_PATH).isPresent()) { this.keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); @@ -102,16 +102,16 @@ public RESTCatalog(Options options) { credentialsProvider, tokenExpireAtMills); } + Map configHeaders = configHeaders(options.toMap()); + Map initHeaders = configHeaders; if (this.catalogAuth != null) { - Map initHeaders = + initHeaders = RESTUtil.merge(configHeaders(options.toMap()), this.catalogAuth.getHeaders()); - this.options = fetchOptionsFromServer(initHeaders, options.toMap()); - this.resourcePaths = - ResourcePaths.forCatalogProperties( - this.options.get(RESTCatalogInternalOptions.PREFIX)); - } else { - throw new IllegalArgumentException("No auth provider provided."); } + this.options = fetchOptionsFromServer(initHeaders, options.toMap()); + this.resourcePaths = + ResourcePaths.forCatalogProperties( + this.options.get(RESTCatalogInternalOptions.PREFIX)); } @Override @@ -228,7 +228,6 @@ private static Map configHeaders(Map properties) } private Map headers() { - catalogAuth.refresh(); return catalogAuth.getHeaders(); } @@ -240,14 +239,7 @@ private ScheduledExecutorService tokenRefreshExecutor() { if (refreshExecutor == null) { synchronized (this) { if (refreshExecutor == null) { - this.refreshExecutor = - // todo: move to ThreadPoolUtil - new ScheduledThreadPoolExecutor( - 1, - new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat("token-refresh-thread") - .build()); + this.refreshExecutor = createScheduledThreadPool(1, "token-refresh-thread"); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index 6155b893751b..d49fa38ab85e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -30,11 +30,6 @@ public class RESTCatalogOptions { .stringType() .noDefaultValue() .withDescription("REST Catalog server's uri."); - public static final ConfigOption TOKEN = - ConfigOptions.key("token") - .stringType() - .noDefaultValue() - .withDescription("REST Catalog server's auth token."); public static final ConfigOption CONNECTION_TIMEOUT = ConfigOptions.key("rest.client.connection-timeout") .durationType() diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 3ed8730862ee..779b40a2af7a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -20,6 +20,7 @@ import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.auth.AuthOptions; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -47,7 +48,7 @@ public void setUp() throws IOException { String baseUrl = mockWebServer.url("").toString(); Options options = new Options(); options.set(RESTCatalogOptions.URI, baseUrl); - options.set(RESTCatalogOptions.TOKEN, initToken); + options.set(AuthOptions.TOKEN, initToken); options.set(RESTCatalogOptions.THREAD_POOL_SIZE, 1); mockOptions(RESTCatalogInternalOptions.PREFIX.key(), "prefix"); restCatalog = new RESTCatalog(options); From 8478865d100bdf0554b80a29fcfafd04adce5330 Mon Sep 17 00:00:00 2001 From: yantian Date: Thu, 5 Dec 2024 17:07:25 +0800 Subject: [PATCH 10/26] support factory to create credentials provider --- .../org/apache/paimon/rest/RESTCatalog.java | 39 +++++---------- .../apache/paimon/rest/auth/AuthOptions.java | 5 ++ .../BearTokenCredentialsProviderFactory.java | 34 +++++++++++++ ...arTokenFileCredentialsProviderFactory.java | 40 +++++++++++++++ .../rest/auth/CredentialsProviderFactory.java | 49 +++++++++++++++++++ .../rest/auth/CredentialsProviderType.java | 24 +++++++++ .../org.apache.paimon.factories.Factory | 2 + .../apache/paimon/rest/RESTCatalogTest.java | 3 ++ 8 files changed, 170 insertions(+), 26 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 9bcf10230005..bb48bf1924b8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -28,9 +28,8 @@ import org.apache.paimon.options.Options; import org.apache.paimon.rest.auth.AuthOptions; import org.apache.paimon.rest.auth.AuthSession; -import org.apache.paimon.rest.auth.BearTokenCredentialsProvider; -import org.apache.paimon.rest.auth.BearTokenFileCredentialsProvider; import org.apache.paimon.rest.auth.CredentialsProvider; +import org.apache.paimon.rest.auth.CredentialsProviderFactory; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; @@ -54,7 +53,7 @@ public class RESTCatalog implements Catalog { private Map options; private Map baseHeader; // a lazy thread pool for token refresh - private AuthSession catalogAuth = null; + private final AuthSession catalogAuth; private volatile ScheduledExecutorService refreshExecutor = null; private boolean keepTokenRefreshed; @@ -79,35 +78,23 @@ public RESTCatalog(Options options) { DefaultErrorHandler.getInstance()); this.client = new HttpClient(httpClientOptions); this.baseHeader = configHeaders(options.toMap()); - // todo: support create CredentialsProvider by conf - if (options.getOptional(AuthOptions.TOKEN).isPresent()) { - CredentialsProvider credentialsProvider = - new BearTokenCredentialsProvider(options.get(AuthOptions.TOKEN)); - this.catalogAuth = new AuthSession(this.baseHeader, credentialsProvider); - } else if (options.getOptional(AuthOptions.TOKEN_FILE_PATH).isPresent()) { - this.keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); - long tokenExpireInMills = options.get(AuthOptions.TOKEN_EXPIRES_IN).toMillis(); - String tokenFilePath = options.getOptional(AuthOptions.TOKEN_FILE_PATH).orElse(null); - long tokenExpireAtMills = System.currentTimeMillis() + tokenExpireInMills; - CredentialsProvider credentialsProvider = - new BearTokenFileCredentialsProvider( - tokenFilePath, - keepTokenRefreshed, - tokenExpireAtMills, - tokenExpireInMills); + CredentialsProvider credentialsProvider = + CredentialsProviderFactory.createCredentialsProvider( + options, RESTCatalog.class.getClassLoader()); + this.keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); + if (keepTokenRefreshed) { this.catalogAuth = AuthSession.fromTokenPath( tokenRefreshExecutor(), this.baseHeader, credentialsProvider, - tokenExpireAtMills); - } - Map configHeaders = configHeaders(options.toMap()); - Map initHeaders = configHeaders; - if (this.catalogAuth != null) { - initHeaders = - RESTUtil.merge(configHeaders(options.toMap()), this.catalogAuth.getHeaders()); + credentialsProvider.expiresAtMillis().get()); + + } else { + this.catalogAuth = new AuthSession(this.baseHeader, credentialsProvider); } + Map initHeaders = + RESTUtil.merge(configHeaders(options.toMap()), this.catalogAuth.getHeaders()); this.options = fetchOptionsFromServer(initHeaders, options.toMap()); this.resourcePaths = ResourcePaths.forCatalogProperties( diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java index 28118b2b50f1..fd2a1cb2df89 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java @@ -45,4 +45,9 @@ public class AuthOptions { .stringType() .noDefaultValue() .withDescription("REST Catalog auth token file path."); + public static final ConfigOption CREDENTIALS_PROVIDER = + ConfigOptions.key("credentials_provider") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth credentials provider."); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java new file mode 100644 index 000000000000..a647198c5c9a --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.options.Options; + +/** factory for create {@link BearTokenCredentialsProvider}. */ +public class BearTokenCredentialsProviderFactory implements CredentialsProviderFactory { + @Override + public String identifier() { + return CredentialsProviderType.BEAR_TOKEN.name(); + } + + @Override + public CredentialsProvider create(Options options) { + return new BearTokenCredentialsProvider(options.get(AuthOptions.TOKEN)); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java new file mode 100644 index 000000000000..547bcf2783fa --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.options.Options; + +/** factory for create {@link BearTokenCredentialsProvider}. */ +public class BearTokenFileCredentialsProviderFactory implements CredentialsProviderFactory { + + @Override + public String identifier() { + return CredentialsProviderType.BEAR_TOKEN_FILE.name(); + } + + @Override + public CredentialsProvider create(Options options) { + boolean keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); + long tokenExpireInMills = options.get(AuthOptions.TOKEN_EXPIRES_IN).toMillis(); + String tokenFilePath = options.getOptional(AuthOptions.TOKEN_FILE_PATH).orElse(null); + long tokenExpireAtMills = System.currentTimeMillis() + tokenExpireInMills; + return new BearTokenFileCredentialsProvider( + tokenFilePath, keepTokenRefreshed, tokenExpireAtMills, tokenExpireInMills); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java new file mode 100644 index 000000000000..a20607c679ed --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.factories.Factory; +import org.apache.paimon.factories.FactoryUtil; +import org.apache.paimon.options.Options; + +import static org.apache.paimon.rest.auth.AuthOptions.CREDENTIALS_PROVIDER; + +/** Factory for creating {@link CredentialsProvider}. */ +public interface CredentialsProviderFactory extends Factory { + + default CredentialsProvider create(Options options) { + throw new UnsupportedOperationException( + "Use create(context) for " + this.getClass().getSimpleName()); + } + + static CredentialsProvider createCredentialsProvider(Options options, ClassLoader classLoader) { + String credentialsProviderIdentifier = options.get(CREDENTIALS_PROVIDER); + CredentialsProviderFactory credentialsProviderFactory = + FactoryUtil.discoverFactory( + classLoader, + CredentialsProviderFactory.class, + credentialsProviderIdentifier); + + try { + return credentialsProviderFactory.create(options); + } catch (UnsupportedOperationException ignore) { + } + return new BearTokenCredentialsProvider(options.get(AuthOptions.TOKEN)); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java new file mode 100644 index 000000000000..5166e1e84767 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; +/** Credentials provider type. */ +public enum CredentialsProviderType { + BEAR_TOKEN, + BEAR_TOKEN_FILE +} diff --git a/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory b/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory index 3b98eef52c85..6416edd720f8 100644 --- a/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory +++ b/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory @@ -37,3 +37,5 @@ org.apache.paimon.mergetree.compact.aggregate.factory.FieldRoaringBitmap64AggFac org.apache.paimon.mergetree.compact.aggregate.factory.FieldSumAggFactory org.apache.paimon.mergetree.compact.aggregate.factory.FieldThetaSketchAggFactory org.apache.paimon.rest.RESTCatalogFactory +org.apache.paimon.rest.auth.BearTokenCredentialsProviderFactory +org.apache.paimon.rest.auth.BearTokenFileCredentialsProviderFactory diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 779b40a2af7a..5c40f435fbff 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.rest.auth.AuthOptions; +import org.apache.paimon.rest.auth.CredentialsProviderType; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -51,6 +52,7 @@ public void setUp() throws IOException { options.set(AuthOptions.TOKEN, initToken); options.set(RESTCatalogOptions.THREAD_POOL_SIZE, 1); mockOptions(RESTCatalogInternalOptions.PREFIX.key(), "prefix"); + options.set(AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); restCatalog = new RESTCatalog(options); } @@ -63,6 +65,7 @@ public void tearDown() throws IOException { public void testInitFailWhenDefineWarehouse() { Options options = new Options(); options.set(CatalogOptions.WAREHOUSE, "/a/b/c"); + options.set(AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); assertThrows(IllegalArgumentException.class, () -> new RESTCatalog(options)); } From ebe0bb538a9d5a2d814282dd6d3b6d590a961b63 Mon Sep 17 00:00:00 2001 From: yantian Date: Thu, 5 Dec 2024 17:36:28 +0800 Subject: [PATCH 11/26] support factory to create credentials provider --- .../org/apache/paimon/rest/auth/CredentialsProviderType.java | 1 + 1 file changed, 1 insertion(+) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java index 5166e1e84767..28c344d70eee 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java @@ -17,6 +17,7 @@ */ package org.apache.paimon.rest.auth; + /** Credentials provider type. */ public enum CredentialsProviderType { BEAR_TOKEN, From bb46ae3afc242827f27f2d40e42676691dbd03e0 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 6 Dec 2024 14:29:51 +0800 Subject: [PATCH 12/26] update token refresh --- .../org/apache/paimon/rest/RESTCatalog.java | 7 +- .../apache/paimon/rest/auth/AuthSession.java | 92 +++++++++++-------- 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index bb48bf1924b8..2e764294f9d2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -84,11 +84,8 @@ public RESTCatalog(Options options) { this.keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); if (keepTokenRefreshed) { this.catalogAuth = - AuthSession.fromTokenPath( - tokenRefreshExecutor(), - this.baseHeader, - credentialsProvider, - credentialsProvider.expiresAtMillis().get()); + AuthSession.fromRefreshCredentialsProvider( + tokenRefreshExecutor(), this.baseHeader, credentialsProvider); } else { this.catalogAuth = new AuthSession(this.baseHeader, credentialsProvider); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index 25038ad5e41e..613c652d0e10 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -21,6 +21,9 @@ import org.apache.paimon.rest.RESTUtil; import org.apache.paimon.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.Map; import java.util.Optional; import java.util.concurrent.ScheduledExecutorService; @@ -28,7 +31,8 @@ /** Auth session. */ public class AuthSession { - private static int tokenRefreshNumRetries = 5; + private static final Logger log = LoggerFactory.getLogger(AuthSession.class); + private static final int TOKEN_REFRESH_NUM_RETRIES = 5; private static final long MAX_REFRESH_WINDOW_MILLIS = 300_000; // 5 minutes private static final long MIN_REFRESH_WAIT_MILLIS = 10; private volatile Map headers; @@ -39,34 +43,32 @@ public AuthSession(Map headers, CredentialsProvider credentialsP this.credentialsProvider = credentialsProvider; } - public static AuthSession fromTokenPath( + public static AuthSession fromRefreshCredentialsProvider( ScheduledExecutorService executor, Map headers, - CredentialsProvider credentialsProvider, - Long defaultExpiresAtMillis) { + CredentialsProvider credentialsProvider) { AuthSession session = new AuthSession(headers, credentialsProvider); long startTimeMillis = System.currentTimeMillis(); Optional expiresAtMillisOpt = credentialsProvider.expiresAtMillis(); if (expiresAtMillisOpt.isPresent() && expiresAtMillisOpt.get() <= startTimeMillis) { - Pair expiration = session.refresh(); + Pair refreshResult = session.refresh(); // if expiration is non-null, then token refresh was successful - if (expiration != null) { + boolean isSuccessful = refreshResult.getKey(); + if (isSuccessful) { if (session.credentialsProvider.expiresAtMillis().isPresent()) { // use the new expiration time from the refreshed token expiresAtMillisOpt = session.credentialsProvider.expiresAtMillis(); } else { // otherwise use the expiration time from the token response - expiresAtMillisOpt = Optional.of(startTimeMillis + expiration.getKey()); + expiresAtMillisOpt = Optional.of(startTimeMillis + refreshResult.getValue()); } } else { // token refresh failed, don't reattempt with the original expiration expiresAtMillisOpt = Optional.empty(); } - } else if (expiresAtMillisOpt.isPresent() && defaultExpiresAtMillis != null) { - expiresAtMillisOpt = Optional.of(defaultExpiresAtMillis); } if (null != executor && expiresAtMillisOpt.isPresent()) { @@ -82,41 +84,59 @@ public Map getHeaders() { private static void scheduleTokenRefresh( ScheduledExecutorService executor, AuthSession session, long expiresAtMillis) { - long expiresInMillis = expiresAtMillis - System.currentTimeMillis(); - // how much ahead of time to start the request to allow it to complete - long refreshWindowMillis = Math.min(expiresInMillis / 10, MAX_REFRESH_WINDOW_MILLIS); - // how much time to wait before expiration - long waitIntervalMillis = expiresInMillis - refreshWindowMillis; - // how much time to actually wait - long timeToWait = Math.max(waitIntervalMillis, MIN_REFRESH_WAIT_MILLIS); - - executor.schedule( - () -> { - long refreshStartTime = System.currentTimeMillis(); - Pair expiration = session.refresh(); - if (expiration != null) { - scheduleTokenRefresh( - executor, session, refreshStartTime + expiration.getKey()); - } - }, - timeToWait, - TimeUnit.MILLISECONDS); + scheduleTokenRefresh(executor, session, expiresAtMillis, 0); + } + + private static void scheduleTokenRefresh( + ScheduledExecutorService executor, + AuthSession session, + long expiresAtMillis, + int retryTimes) { + if (retryTimes < TOKEN_REFRESH_NUM_RETRIES) { + long expiresInMillis = expiresAtMillis - System.currentTimeMillis(); + // how much ahead of time to start the request to allow it to complete + long refreshWindowMillis = Math.min(expiresInMillis / 10, MAX_REFRESH_WINDOW_MILLIS); + // how much time to wait before expiration + long waitIntervalMillis = expiresInMillis - refreshWindowMillis; + // how much time to actually wait + long timeToWait = Math.max(waitIntervalMillis, MIN_REFRESH_WAIT_MILLIS); + + executor.schedule( + () -> { + long refreshStartTime = System.currentTimeMillis(); + Pair refreshResult = session.refresh(); + boolean isSuccessful = refreshResult.getKey(); + if (isSuccessful) { + scheduleTokenRefresh( + executor, + session, + refreshStartTime + refreshResult.getValue(), + 0); + } else { + scheduleTokenRefresh( + executor, session, expiresAtMillis, retryTimes + 1); + } + }, + timeToWait, + TimeUnit.MILLISECONDS); + } else { + log.warn("Failed to refresh token after {} retries.", TOKEN_REFRESH_NUM_RETRIES); + } } - public Pair refresh() { - if (this.credentialsProvider.supportRefresh() && this.credentialsProvider.keepRefreshed()) { + public Pair refresh() { + if (this.credentialsProvider.supportRefresh() + && this.credentialsProvider.keepRefreshed() + && this.credentialsProvider.expiresInMills().isPresent()) { boolean isSuccessful = this.credentialsProvider.refresh(); if (!isSuccessful) { - return null; + return Pair.of(false, 0L); } Map currentHeaders = this.headers; this.headers = RESTUtil.merge(currentHeaders, this.credentialsProvider.authHeader()); - - if (credentialsProvider.expiresInMills().isPresent()) { - return Pair.of(credentialsProvider.expiresInMills().get(), TimeUnit.SECONDS); - } + return Pair.of(true, credentialsProvider.expiresInMills().get()); } - return null; + return Pair.of(false, 0L); } } From b91d797788f2434d4088c1dd737f0c7f90ed9c0c Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 6 Dec 2024 15:19:24 +0800 Subject: [PATCH 13/26] add test for AuthSession --- .../apache/paimon/rest/auth/AuthSession.java | 2 +- ...arTokenFileCredentialsProviderFactory.java | 3 +- .../paimon/rest/auth/AuthSessionTest.java | 69 +++++++++++++++++++ 3 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index 613c652d0e10..f3971d645951 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -35,8 +35,8 @@ public class AuthSession { private static final int TOKEN_REFRESH_NUM_RETRIES = 5; private static final long MAX_REFRESH_WINDOW_MILLIS = 300_000; // 5 minutes private static final long MIN_REFRESH_WAIT_MILLIS = 10; - private volatile Map headers; private final CredentialsProvider credentialsProvider; + private volatile Map headers; public AuthSession(Map headers, CredentialsProvider credentialsProvider) { this.headers = headers; diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java index 547bcf2783fa..5fe11bf97d89 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java @@ -33,8 +33,7 @@ public CredentialsProvider create(Options options) { boolean keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); long tokenExpireInMills = options.get(AuthOptions.TOKEN_EXPIRES_IN).toMillis(); String tokenFilePath = options.getOptional(AuthOptions.TOKEN_FILE_PATH).orElse(null); - long tokenExpireAtMills = System.currentTimeMillis() + tokenExpireInMills; return new BearTokenFileCredentialsProvider( - tokenFilePath, keepTokenRefreshed, tokenExpireAtMills, tokenExpireInMills); + tokenFilePath, keepTokenRefreshed, -1L, tokenExpireInMills); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java new file mode 100644 index 000000000000..1afbcb44b56d --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.utils.ThreadPoolUtils; + +import org.apache.commons.io.FileUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; + +import static org.junit.Assert.assertEquals; + +/** Test for {@link AuthSession}. */ +public class AuthSessionTest { + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testRefreshBearTokenFileCredentialsProvider() + throws IOException, InterruptedException { + String fileName = "token"; + File tokenFile = folder.newFile(fileName); + String token = UUID.randomUUID().toString(); + FileUtils.writeStringToFile(tokenFile, token); + Map initialHeaders = new HashMap<>(); + long expiresInMillis = 1000L; + CredentialsProvider credentialsProvider = + new BearTokenFileCredentialsProvider( + tokenFile.getPath(), true, -1L, expiresInMillis); + ScheduledExecutorService executor = + ThreadPoolUtils.createScheduledThreadPool(1, "refresh-token"); + AuthSession session = + AuthSession.fromRefreshCredentialsProvider( + executor, initialHeaders, credentialsProvider); + Map header = session.getHeaders(); + assertEquals(header.get("Authorization"), "Bearer " + token); + tokenFile.delete(); + tokenFile = folder.newFile(fileName); + token = UUID.randomUUID().toString(); + FileUtils.writeStringToFile(tokenFile, token); + Thread.sleep(expiresInMillis + 500L); + header = session.getHeaders(); + assertEquals(header.get("Authorization"), "Bearer " + token); + } +} From ec074c21635d4d738a4f1bbe522139e2e051d706 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 6 Dec 2024 15:29:32 +0800 Subject: [PATCH 14/26] get header from auth --- .../src/main/java/org/apache/paimon/rest/RESTCatalog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 2e764294f9d2..d45cf8b8a38c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -203,7 +203,7 @@ public void close() throws Exception {} Map fetchOptionsFromServer( Map headers, Map clientProperties) { ConfigResponse response = - client.get(ResourcePaths.V1_CONFIG, ConfigResponse.class, headers); + client.get(ResourcePaths.V1_CONFIG, ConfigResponse.class, headers()); return response.merge(clientProperties); } From 9aafd86d38dc5b624b789f888647580f79e0e7ba Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 6 Dec 2024 15:54:35 +0800 Subject: [PATCH 15/26] when CredentialsProvider is soon expire force refresh --- .../apache/paimon/rest/auth/AuthSession.java | 3 ++ .../BearTokenFileCredentialsProvider.java | 13 +++++ .../paimon/rest/auth/CredentialsProvider.java | 4 ++ .../paimon/rest/auth/AuthSessionTest.java | 48 ++++++++++++++++--- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index f3971d645951..62405113f509 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -79,6 +79,9 @@ public static AuthSession fromRefreshCredentialsProvider( } public Map getHeaders() { + if (this.credentialsProvider.keepRefreshed() && this.credentialsProvider.willSoonExpire()) { + refresh(); + } return headers; } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java index ea5d5a650970..a400c5af6e3a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java @@ -28,6 +28,9 @@ /** credentials provider for get bear token from file. */ public class BearTokenFileCredentialsProvider extends BaseBearTokenCredentialsProvider { + + public static final double EXPIRED_FACTOR = 0.4; + private final String tokenFilePath; private String token; private boolean keepRefreshed = false; @@ -76,6 +79,16 @@ public boolean keepRefreshed() { return this.keepRefreshed; } + @Override + public boolean willSoonExpire() { + if (keepRefreshed()) { + return expiresAtMillis().get() - System.currentTimeMillis() + < expiresInMills().get() * EXPIRED_FACTOR; + } else { + return false; + } + } + @Override public Optional expiresAtMillis() { return Optional.ofNullable(this.expiresAtMillis); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java index b6bfa589282f..b927ff980baa 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java @@ -35,6 +35,10 @@ default boolean keepRefreshed() { return false; } + default boolean willSoonExpire() { + return false; + } + default Optional expiresAtMillis() { return Optional.empty(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java index 1afbcb44b56d..b149530b8c47 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java @@ -18,6 +18,7 @@ package org.apache.paimon.rest.auth; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.ThreadPoolUtils; import org.apache.commons.io.FileUtils; @@ -43,9 +44,9 @@ public class AuthSessionTest { public void testRefreshBearTokenFileCredentialsProvider() throws IOException, InterruptedException { String fileName = "token"; - File tokenFile = folder.newFile(fileName); - String token = UUID.randomUUID().toString(); - FileUtils.writeStringToFile(tokenFile, token); + Pair tokenFile2Token = generateTokenAndWriteToFile(fileName); + String token = tokenFile2Token.getRight(); + File tokenFile = tokenFile2Token.getLeft(); Map initialHeaders = new HashMap<>(); long expiresInMillis = 1000L; CredentialsProvider credentialsProvider = @@ -59,11 +60,46 @@ public void testRefreshBearTokenFileCredentialsProvider() Map header = session.getHeaders(); assertEquals(header.get("Authorization"), "Bearer " + token); tokenFile.delete(); - tokenFile = folder.newFile(fileName); - token = UUID.randomUUID().toString(); - FileUtils.writeStringToFile(tokenFile, token); + tokenFile2Token = generateTokenAndWriteToFile(fileName); + token = tokenFile2Token.getRight(); Thread.sleep(expiresInMillis + 500L); header = session.getHeaders(); assertEquals(header.get("Authorization"), "Bearer " + token); } + + @Test + public void testRefreshCredentialsProviderIsSoonExpire() + throws IOException, InterruptedException { + String fileName = "token"; + Pair tokenFile2Token = generateTokenAndWriteToFile(fileName); + String token = tokenFile2Token.getRight(); + File tokenFile = tokenFile2Token.getLeft(); + Map initialHeaders = new HashMap<>(); + long expiresInMillis = 1000L; + CredentialsProvider credentialsProvider = + new BearTokenFileCredentialsProvider( + tokenFile.getPath(), true, -1L, expiresInMillis); + AuthSession session = + AuthSession.fromRefreshCredentialsProvider( + null, initialHeaders, credentialsProvider); + Map header = session.getHeaders(); + assertEquals(header.get("Authorization"), "Bearer " + token); + tokenFile.delete(); + tokenFile2Token = generateTokenAndWriteToFile(fileName); + token = tokenFile2Token.getRight(); + tokenFile = tokenFile2Token.getLeft(); + FileUtils.writeStringToFile(tokenFile, token); + Thread.sleep( + (long) (expiresInMillis * (1 - BearTokenFileCredentialsProvider.EXPIRED_FACTOR)) + + 10L); + header = session.getHeaders(); + assertEquals(header.get("Authorization"), "Bearer " + token); + } + + private Pair generateTokenAndWriteToFile(String fileName) throws IOException { + File tokenFile = folder.newFile(fileName); + String token = UUID.randomUUID().toString(); + FileUtils.writeStringToFile(tokenFile, token); + return Pair.of(tokenFile, token); + } } From 6e7f2cd0a2e24a213a33531149b089a157f310e9 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 6 Dec 2024 16:20:49 +0800 Subject: [PATCH 16/26] add check when create BearTokenFileCredentialsProvider --- .../BearTokenFileCredentialsProvider.java | 10 +++------ ...arTokenFileCredentialsProviderFactory.java | 22 +++++++++++++++---- .../paimon/rest/auth/AuthSessionTest.java | 6 ++--- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java index a400c5af6e3a..4e8246498fb2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java @@ -42,14 +42,10 @@ public BearTokenFileCredentialsProvider(String tokenFilePath) { this.token = getTokenFromFile(); } - public BearTokenFileCredentialsProvider( - String tokenFilePath, - boolean keepRefreshed, - Long expiresAtMillis, - Long expiresInMills) { + public BearTokenFileCredentialsProvider(String tokenFilePath, Long expiresInMills) { this(tokenFilePath); - this.keepRefreshed = keepRefreshed; - this.expiresAtMillis = expiresAtMillis; + this.keepRefreshed = true; + this.expiresAtMillis = -1L; this.expiresInMills = expiresInMills; } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java index 5fe11bf97d89..448d279cd887 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java @@ -20,6 +20,9 @@ import org.apache.paimon.options.Options; +import static org.apache.paimon.rest.auth.AuthOptions.TOKEN_EXPIRES_IN; +import static org.apache.paimon.rest.auth.AuthOptions.TOKEN_FILE_PATH; + /** factory for create {@link BearTokenCredentialsProvider}. */ public class BearTokenFileCredentialsProviderFactory implements CredentialsProviderFactory { @@ -30,10 +33,21 @@ public String identifier() { @Override public CredentialsProvider create(Options options) { + if (!options.getOptional(TOKEN_FILE_PATH).isPresent()) { + throw new IllegalArgumentException(TOKEN_FILE_PATH.key() + " is required"); + } + String tokenFilePath = options.get(TOKEN_FILE_PATH); boolean keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); - long tokenExpireInMills = options.get(AuthOptions.TOKEN_EXPIRES_IN).toMillis(); - String tokenFilePath = options.getOptional(AuthOptions.TOKEN_FILE_PATH).orElse(null); - return new BearTokenFileCredentialsProvider( - tokenFilePath, keepTokenRefreshed, -1L, tokenExpireInMills); + if (keepTokenRefreshed) { + if (!options.getOptional(TOKEN_EXPIRES_IN).isPresent()) { + throw new IllegalArgumentException( + TOKEN_EXPIRES_IN.key() + " is required when token refresh enabled"); + } + long tokenExpireInMills = options.get(TOKEN_EXPIRES_IN).toMillis(); + return new BearTokenFileCredentialsProvider(tokenFilePath, tokenExpireInMills); + + } else { + return new BearTokenFileCredentialsProvider(tokenFilePath); + } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java index b149530b8c47..a614f83283d5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java @@ -50,8 +50,7 @@ public void testRefreshBearTokenFileCredentialsProvider() Map initialHeaders = new HashMap<>(); long expiresInMillis = 1000L; CredentialsProvider credentialsProvider = - new BearTokenFileCredentialsProvider( - tokenFile.getPath(), true, -1L, expiresInMillis); + new BearTokenFileCredentialsProvider(tokenFile.getPath(), expiresInMillis); ScheduledExecutorService executor = ThreadPoolUtils.createScheduledThreadPool(1, "refresh-token"); AuthSession session = @@ -77,8 +76,7 @@ public void testRefreshCredentialsProviderIsSoonExpire() Map initialHeaders = new HashMap<>(); long expiresInMillis = 1000L; CredentialsProvider credentialsProvider = - new BearTokenFileCredentialsProvider( - tokenFile.getPath(), true, -1L, expiresInMillis); + new BearTokenFileCredentialsProvider(tokenFile.getPath(), expiresInMillis); AuthSession session = AuthSession.fromRefreshCredentialsProvider( null, initialHeaders, credentialsProvider); From 2a422b5aec6f11fe49eb7e48f4a55182e830bd50 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 6 Dec 2024 17:19:52 +0800 Subject: [PATCH 17/26] add test for CredentialsProviderFactory --- .../BearTokenCredentialsProviderFactory.java | 7 + .../auth/CredentialsProviderFactoryTest.java | 131 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java index a647198c5c9a..f009ad8442cf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest.auth; import org.apache.paimon.options.Options; +import org.apache.paimon.utils.StringUtils; /** factory for create {@link BearTokenCredentialsProvider}. */ public class BearTokenCredentialsProviderFactory implements CredentialsProviderFactory { @@ -29,6 +30,12 @@ public String identifier() { @Override public CredentialsProvider create(Options options) { + if (options.getOptional(AuthOptions.TOKEN) + .map(StringUtils::isNullOrWhitespaceOnly) + .orElse(true)) { + throw new IllegalArgumentException( + AuthOptions.TOKEN.key() + " is required and not empty"); + } return new BearTokenCredentialsProvider(options.get(AuthOptions.TOKEN)); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java new file mode 100644 index 000000000000..1816c59212a7 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.options.Options; + +import org.apache.commons.io.FileUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.time.Duration; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +/** Test for {@link CredentialsProviderFactory}. */ +public class CredentialsProviderFactoryTest { + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testCreateBearTokenCredentialsProviderSuccess() { + Options options = new Options(); + String token = UUID.randomUUID().toString(); + options.set(AuthOptions.TOKEN, token); + options.set(AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); + BearTokenCredentialsProvider credentialsProvider = + (BearTokenCredentialsProvider) + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader()); + assertEquals(token, credentialsProvider.token()); + } + + @Test + public void testCreateBearTokenCredentialsProviderFail() { + Options options = new Options(); + options.set(AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); + assertThrows( + IllegalArgumentException.class, + () -> + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader())); + } + + @Test + public void testCreateBearTokenFileCredentialsProviderSuccess() throws Exception { + Options options = new Options(); + String fileName = "token"; + File tokenFile = folder.newFile(fileName); + String token = UUID.randomUUID().toString(); + FileUtils.writeStringToFile(tokenFile, token); + options.set(AuthOptions.TOKEN_FILE_PATH, tokenFile.getPath()); + options.set( + AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + BearTokenFileCredentialsProvider credentialsProvider = + (BearTokenFileCredentialsProvider) + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader()); + assertEquals(token, credentialsProvider.token()); + } + + @Test + public void testCreateBearTokenFileCredentialsProviderFail() throws Exception { + Options options = new Options(); + options.set( + AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + assertThrows( + IllegalArgumentException.class, + () -> + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader())); + } + + @Test + public void testCreateRefreshBearTokenFileCredentialsProviderSuccess() throws Exception { + Options options = new Options(); + String fileName = "token"; + File tokenFile = folder.newFile(fileName); + String token = UUID.randomUUID().toString(); + FileUtils.writeStringToFile(tokenFile, token); + options.set( + AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + options.set(AuthOptions.TOKEN_FILE_PATH, tokenFile.getPath()); + options.set(AuthOptions.TOKEN_REFRESH_ENABLED, true); + options.set(AuthOptions.TOKEN_EXPIRES_IN, Duration.ofSeconds(10L)); + BearTokenFileCredentialsProvider credentialsProvider = + (BearTokenFileCredentialsProvider) + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader()); + assertEquals(token, credentialsProvider.token()); + } + + @Test + public void testCreateRefreshBearTokenFileCredentialsProviderFail() throws Exception { + Options options = new Options(); + String fileName = "token"; + File tokenFile = folder.newFile(fileName); + String token = UUID.randomUUID().toString(); + FileUtils.writeStringToFile(tokenFile, token); + options.set( + AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + options.set(AuthOptions.TOKEN_FILE_PATH, tokenFile.getPath()); + options.set(AuthOptions.TOKEN_REFRESH_ENABLED, true); + options.set( + AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + assertThrows( + IllegalArgumentException.class, + () -> + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader())); + } +} From b754f0b326c3dd6a5b370c88da4809772e703612 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 6 Dec 2024 17:29:38 +0800 Subject: [PATCH 18/26] update credentials provider conf key --- .../src/main/java/org/apache/paimon/rest/auth/AuthOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java index fd2a1cb2df89..c68bb29d12e9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java @@ -46,7 +46,7 @@ public class AuthOptions { .noDefaultValue() .withDescription("REST Catalog auth token file path."); public static final ConfigOption CREDENTIALS_PROVIDER = - ConfigOptions.key("credentials_provider") + ConfigOptions.key("credentials-provider") .stringType() .noDefaultValue() .withDescription("REST Catalog auth credentials provider."); From 858f13bc017bd46613aa3b4fd6e76efd8f53b68d Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 9 Dec 2024 13:45:50 +0800 Subject: [PATCH 19/26] move AuthOptions to RESTCatalogOptions and update option name about auth --- .../org/apache/paimon/rest/RESTCatalog.java | 3 +- .../paimon/rest/RESTCatalogOptions.java | 25 +++++++++ .../apache/paimon/rest/auth/AuthOptions.java | 53 ------------------- .../BearTokenCredentialsProviderFactory.java | 7 +-- .../BearTokenFileCredentialsProvider.java | 18 +++---- ...arTokenFileCredentialsProviderFactory.java | 19 +++---- .../rest/auth/CredentialsProviderFactory.java | 5 +- .../apache/paimon/rest/RESTCatalogTest.java | 9 ++-- .../auth/CredentialsProviderFactoryTest.java | 36 ++++++++----- 9 files changed, 79 insertions(+), 96 deletions(-) delete mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index d45cf8b8a38c..ab1799a56367 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -26,7 +26,6 @@ import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.rest.auth.AuthOptions; import org.apache.paimon.rest.auth.AuthSession; import org.apache.paimon.rest.auth.CredentialsProvider; import org.apache.paimon.rest.auth.CredentialsProviderFactory; @@ -81,7 +80,7 @@ public RESTCatalog(Options options) { CredentialsProvider credentialsProvider = CredentialsProviderFactory.createCredentialsProvider( options, RESTCatalog.class.getClassLoader()); - this.keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); + this.keepTokenRefreshed = options.get(RESTCatalogOptions.TOKEN_REFRESH_ENABLED); if (keepTokenRefreshed) { this.catalogAuth = AuthSession.fromRefreshCredentialsProvider( diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index d49fa38ab85e..bc405b996a94 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -45,4 +45,29 @@ public class RESTCatalogOptions { .intType() .defaultValue(1) .withDescription("REST Catalog http client thread num."); + public static final ConfigOption TOKEN = + ConfigOptions.key("token") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth token."); + public static final ConfigOption TOKEN_EXPIRATION_TIME = + ConfigOptions.key("token.expiration-time") + .durationType() + .defaultValue(Duration.ofHours(1)) + .withDescription("REST Catalog auth token expires in."); + public static final ConfigOption TOKEN_REFRESH_ENABLED = + ConfigOptions.key("token-refresh-enabled") + .booleanType() + .defaultValue(false) + .withDescription("REST Catalog auth token refresh enable."); + public static final ConfigOption TOKEN_PROVIDER_PATH = + ConfigOptions.key("token.provider.path") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth token file path."); + public static final ConfigOption CREDENTIALS_PROVIDER = + ConfigOptions.key("credentials-provider") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth credentials provider."); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java deleted file mode 100644 index c68bb29d12e9..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthOptions.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.auth; - -import org.apache.paimon.options.ConfigOption; -import org.apache.paimon.options.ConfigOptions; - -import java.time.Duration; - -/** Options for REST Catalog Auth. */ -public class AuthOptions { - public static final ConfigOption TOKEN = - ConfigOptions.key("token") - .stringType() - .noDefaultValue() - .withDescription("REST Catalog auth token."); - public static final ConfigOption TOKEN_EXPIRES_IN = - ConfigOptions.key("token-expires-in") - .durationType() - .defaultValue(Duration.ofHours(1)) - .withDescription("REST Catalog auth token expires in."); - public static final ConfigOption TOKEN_REFRESH_ENABLED = - ConfigOptions.key("token-refresh-enabled") - .booleanType() - .defaultValue(false) - .withDescription("REST Catalog auth token refresh enable."); - public static final ConfigOption TOKEN_FILE_PATH = - ConfigOptions.key("token-file-path") - .stringType() - .noDefaultValue() - .withDescription("REST Catalog auth token file path."); - public static final ConfigOption CREDENTIALS_PROVIDER = - ConfigOptions.key("credentials-provider") - .stringType() - .noDefaultValue() - .withDescription("REST Catalog auth credentials provider."); -} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java index f009ad8442cf..cf3864025107 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest.auth; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTCatalogOptions; import org.apache.paimon.utils.StringUtils; /** factory for create {@link BearTokenCredentialsProvider}. */ @@ -30,12 +31,12 @@ public String identifier() { @Override public CredentialsProvider create(Options options) { - if (options.getOptional(AuthOptions.TOKEN) + if (options.getOptional(RESTCatalogOptions.TOKEN) .map(StringUtils::isNullOrWhitespaceOnly) .orElse(true)) { throw new IllegalArgumentException( - AuthOptions.TOKEN.key() + " is required and not empty"); + RESTCatalogOptions.TOKEN.key() + " is required and not empty"); } - return new BearTokenCredentialsProvider(options.get(AuthOptions.TOKEN)); + return new BearTokenCredentialsProvider(options.get(RESTCatalogOptions.TOKEN)); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java index 4e8246498fb2..d479caa67fd0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java @@ -18,12 +18,12 @@ package org.apache.paimon.rest.auth; +import org.apache.paimon.utils.FileIOUtils; import org.apache.paimon.utils.StringUtils; +import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; +import java.io.UncheckedIOException; import java.util.Optional; /** credentials provider for get bear token from file. */ @@ -57,11 +57,12 @@ String token() { @Override public boolean refresh() { long start = System.currentTimeMillis(); - this.token = getTokenFromFile(); - this.expiresAtMillis = start + this.expiresInMills; - if (StringUtils.isNullOrWhitespaceOnly(this.token)) { + String newToken = getTokenFromFile(); + if (StringUtils.isNullOrWhitespaceOnly(newToken)) { return false; } + this.expiresAtMillis = start + this.expiresInMills; + this.token = newToken; return true; } @@ -97,10 +98,9 @@ public Optional expiresInMills() { private String getTokenFromFile() { try { - // todo: handle exception - return new String(Files.readAllBytes(Paths.get(tokenFilePath)), StandardCharsets.UTF_8); + return FileIOUtils.readFileUtf8(new File(tokenFilePath)); } catch (IOException e) { - throw new RuntimeException(e); + throw new UncheckedIOException(e); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java index 448d279cd887..b7a99639289e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java @@ -19,9 +19,10 @@ package org.apache.paimon.rest.auth; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTCatalogOptions; -import static org.apache.paimon.rest.auth.AuthOptions.TOKEN_EXPIRES_IN; -import static org.apache.paimon.rest.auth.AuthOptions.TOKEN_FILE_PATH; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_EXPIRATION_TIME; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER_PATH; /** factory for create {@link BearTokenCredentialsProvider}. */ public class BearTokenFileCredentialsProviderFactory implements CredentialsProviderFactory { @@ -33,17 +34,17 @@ public String identifier() { @Override public CredentialsProvider create(Options options) { - if (!options.getOptional(TOKEN_FILE_PATH).isPresent()) { - throw new IllegalArgumentException(TOKEN_FILE_PATH.key() + " is required"); + if (!options.getOptional(TOKEN_PROVIDER_PATH).isPresent()) { + throw new IllegalArgumentException(TOKEN_PROVIDER_PATH.key() + " is required"); } - String tokenFilePath = options.get(TOKEN_FILE_PATH); - boolean keepTokenRefreshed = options.get(AuthOptions.TOKEN_REFRESH_ENABLED); + String tokenFilePath = options.get(TOKEN_PROVIDER_PATH); + boolean keepTokenRefreshed = options.get(RESTCatalogOptions.TOKEN_REFRESH_ENABLED); if (keepTokenRefreshed) { - if (!options.getOptional(TOKEN_EXPIRES_IN).isPresent()) { + if (!options.getOptional(TOKEN_EXPIRATION_TIME).isPresent()) { throw new IllegalArgumentException( - TOKEN_EXPIRES_IN.key() + " is required when token refresh enabled"); + TOKEN_EXPIRATION_TIME.key() + " is required when token refresh enabled"); } - long tokenExpireInMills = options.get(TOKEN_EXPIRES_IN).toMillis(); + long tokenExpireInMills = options.get(TOKEN_EXPIRATION_TIME).toMillis(); return new BearTokenFileCredentialsProvider(tokenFilePath, tokenExpireInMills); } else { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java index a20607c679ed..cf918805cbae 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java @@ -21,8 +21,9 @@ import org.apache.paimon.factories.Factory; import org.apache.paimon.factories.FactoryUtil; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTCatalogOptions; -import static org.apache.paimon.rest.auth.AuthOptions.CREDENTIALS_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.CREDENTIALS_PROVIDER; /** Factory for creating {@link CredentialsProvider}. */ public interface CredentialsProviderFactory extends Factory { @@ -44,6 +45,6 @@ static CredentialsProvider createCredentialsProvider(Options options, ClassLoade return credentialsProviderFactory.create(options); } catch (UnsupportedOperationException ignore) { } - return new BearTokenCredentialsProvider(options.get(AuthOptions.TOKEN)); + return new BearTokenCredentialsProvider(options.get(RESTCatalogOptions.TOKEN)); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 5c40f435fbff..40983d82d6dd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -20,7 +20,6 @@ import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.rest.auth.AuthOptions; import org.apache.paimon.rest.auth.CredentialsProviderType; import okhttp3.mockwebserver.MockResponse; @@ -49,10 +48,11 @@ public void setUp() throws IOException { String baseUrl = mockWebServer.url("").toString(); Options options = new Options(); options.set(RESTCatalogOptions.URI, baseUrl); - options.set(AuthOptions.TOKEN, initToken); + options.set(RESTCatalogOptions.TOKEN, initToken); options.set(RESTCatalogOptions.THREAD_POOL_SIZE, 1); mockOptions(RESTCatalogInternalOptions.PREFIX.key(), "prefix"); - options.set(AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); + options.set( + RESTCatalogOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); restCatalog = new RESTCatalog(options); } @@ -65,7 +65,8 @@ public void tearDown() throws IOException { public void testInitFailWhenDefineWarehouse() { Options options = new Options(); options.set(CatalogOptions.WAREHOUSE, "/a/b/c"); - options.set(AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); + options.set( + RESTCatalogOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); assertThrows(IllegalArgumentException.class, () -> new RESTCatalog(options)); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java index 1816c59212a7..c2ef4923175e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest.auth; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTCatalogOptions; import org.apache.commons.io.FileUtils; import org.junit.Rule; @@ -41,8 +42,9 @@ public class CredentialsProviderFactoryTest { public void testCreateBearTokenCredentialsProviderSuccess() { Options options = new Options(); String token = UUID.randomUUID().toString(); - options.set(AuthOptions.TOKEN, token); - options.set(AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); + options.set(RESTCatalogOptions.TOKEN, token); + options.set( + RESTCatalogOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); BearTokenCredentialsProvider credentialsProvider = (BearTokenCredentialsProvider) CredentialsProviderFactory.createCredentialsProvider( @@ -53,7 +55,8 @@ public void testCreateBearTokenCredentialsProviderSuccess() { @Test public void testCreateBearTokenCredentialsProviderFail() { Options options = new Options(); - options.set(AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); + options.set( + RESTCatalogOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); assertThrows( IllegalArgumentException.class, () -> @@ -68,9 +71,10 @@ public void testCreateBearTokenFileCredentialsProviderSuccess() throws Exception File tokenFile = folder.newFile(fileName); String token = UUID.randomUUID().toString(); FileUtils.writeStringToFile(tokenFile, token); - options.set(AuthOptions.TOKEN_FILE_PATH, tokenFile.getPath()); + options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); options.set( - AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + RESTCatalogOptions.CREDENTIALS_PROVIDER, + CredentialsProviderType.BEAR_TOKEN_FILE.name()); BearTokenFileCredentialsProvider credentialsProvider = (BearTokenFileCredentialsProvider) CredentialsProviderFactory.createCredentialsProvider( @@ -82,7 +86,8 @@ public void testCreateBearTokenFileCredentialsProviderSuccess() throws Exception public void testCreateBearTokenFileCredentialsProviderFail() throws Exception { Options options = new Options(); options.set( - AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + RESTCatalogOptions.CREDENTIALS_PROVIDER, + CredentialsProviderType.BEAR_TOKEN_FILE.name()); assertThrows( IllegalArgumentException.class, () -> @@ -98,10 +103,11 @@ public void testCreateRefreshBearTokenFileCredentialsProviderSuccess() throws Ex String token = UUID.randomUUID().toString(); FileUtils.writeStringToFile(tokenFile, token); options.set( - AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); - options.set(AuthOptions.TOKEN_FILE_PATH, tokenFile.getPath()); - options.set(AuthOptions.TOKEN_REFRESH_ENABLED, true); - options.set(AuthOptions.TOKEN_EXPIRES_IN, Duration.ofSeconds(10L)); + RESTCatalogOptions.CREDENTIALS_PROVIDER, + CredentialsProviderType.BEAR_TOKEN_FILE.name()); + options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); + options.set(RESTCatalogOptions.TOKEN_REFRESH_ENABLED, true); + options.set(RESTCatalogOptions.TOKEN_EXPIRATION_TIME, Duration.ofSeconds(10L)); BearTokenFileCredentialsProvider credentialsProvider = (BearTokenFileCredentialsProvider) CredentialsProviderFactory.createCredentialsProvider( @@ -117,11 +123,13 @@ public void testCreateRefreshBearTokenFileCredentialsProviderFail() throws Excep String token = UUID.randomUUID().toString(); FileUtils.writeStringToFile(tokenFile, token); options.set( - AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); - options.set(AuthOptions.TOKEN_FILE_PATH, tokenFile.getPath()); - options.set(AuthOptions.TOKEN_REFRESH_ENABLED, true); + RESTCatalogOptions.CREDENTIALS_PROVIDER, + CredentialsProviderType.BEAR_TOKEN_FILE.name()); + options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); + options.set(RESTCatalogOptions.TOKEN_REFRESH_ENABLED, true); options.set( - AuthOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + RESTCatalogOptions.CREDENTIALS_PROVIDER, + CredentialsProviderType.BEAR_TOKEN_FILE.name()); assertThrows( IllegalArgumentException.class, () -> From c2c3046cab96390fb08466595f4f30c04ee65791 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 9 Dec 2024 14:18:21 +0800 Subject: [PATCH 20/26] create credentials provider by conf about auth and move CREDENTIALS_PROVIDER to RESTCatalogInternalOptions --- .../rest/RESTCatalogInternalOptions.java | 5 ++ .../paimon/rest/RESTCatalogOptions.java | 5 -- .../rest/auth/CredentialsProviderFactory.java | 16 +++--- .../apache/paimon/rest/RESTCatalogTest.java | 5 -- .../auth/CredentialsProviderFactoryTest.java | 51 ++++++++++++------- 5 files changed, 47 insertions(+), 35 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java index cf61caa20e88..62a8bf134ae5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java @@ -28,4 +28,9 @@ public class RESTCatalogInternalOptions { .stringType() .noDefaultValue() .withDescription("REST Catalog uri's prefix."); + public static final ConfigOption CREDENTIALS_PROVIDER = + ConfigOptions.key("credentials-provider") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth credentials provider."); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index bc405b996a94..85fac413bba6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -65,9 +65,4 @@ public class RESTCatalogOptions { .stringType() .noDefaultValue() .withDescription("REST Catalog auth token file path."); - public static final ConfigOption CREDENTIALS_PROVIDER = - ConfigOptions.key("credentials-provider") - .stringType() - .noDefaultValue() - .withDescription("REST Catalog auth credentials provider."); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java index cf918805cbae..50c3564ad8c6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java @@ -23,7 +23,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.rest.RESTCatalogOptions; -import static org.apache.paimon.rest.RESTCatalogOptions.CREDENTIALS_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogInternalOptions.CREDENTIALS_PROVIDER; /** Factory for creating {@link CredentialsProvider}. */ public interface CredentialsProviderFactory extends Factory { @@ -34,17 +34,21 @@ default CredentialsProvider create(Options options) { } static CredentialsProvider createCredentialsProvider(Options options, ClassLoader classLoader) { - String credentialsProviderIdentifier = options.get(CREDENTIALS_PROVIDER); + String credentialsProviderIdentifier = getCredentialsProviderTypeByConf(options).name(); CredentialsProviderFactory credentialsProviderFactory = FactoryUtil.discoverFactory( classLoader, CredentialsProviderFactory.class, credentialsProviderIdentifier); + return credentialsProviderFactory.create(options); + } - try { - return credentialsProviderFactory.create(options); - } catch (UnsupportedOperationException ignore) { + static CredentialsProviderType getCredentialsProviderTypeByConf(Options options) { + if (options.getOptional(CREDENTIALS_PROVIDER).isPresent()) { + return CredentialsProviderType.valueOf(options.get(CREDENTIALS_PROVIDER)); + } else if (options.getOptional(RESTCatalogOptions.TOKEN_PROVIDER_PATH).isPresent()) { + return CredentialsProviderType.BEAR_TOKEN_FILE; } - return new BearTokenCredentialsProvider(options.get(RESTCatalogOptions.TOKEN)); + return CredentialsProviderType.BEAR_TOKEN; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 40983d82d6dd..3ed8730862ee 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -20,7 +20,6 @@ import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; -import org.apache.paimon.rest.auth.CredentialsProviderType; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -51,8 +50,6 @@ public void setUp() throws IOException { options.set(RESTCatalogOptions.TOKEN, initToken); options.set(RESTCatalogOptions.THREAD_POOL_SIZE, 1); mockOptions(RESTCatalogInternalOptions.PREFIX.key(), "prefix"); - options.set( - RESTCatalogOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); restCatalog = new RESTCatalog(options); } @@ -65,8 +62,6 @@ public void tearDown() throws IOException { public void testInitFailWhenDefineWarehouse() { Options options = new Options(); options.set(CatalogOptions.WAREHOUSE, "/a/b/c"); - options.set( - RESTCatalogOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); assertThrows(IllegalArgumentException.class, () -> new RESTCatalog(options)); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java index c2ef4923175e..c676064e1cf6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java @@ -30,6 +30,7 @@ import java.time.Duration; import java.util.UUID; +import static org.apache.paimon.rest.RESTCatalogInternalOptions.CREDENTIALS_PROVIDER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; @@ -43,8 +44,6 @@ public void testCreateBearTokenCredentialsProviderSuccess() { Options options = new Options(); String token = UUID.randomUUID().toString(); options.set(RESTCatalogOptions.TOKEN, token); - options.set( - RESTCatalogOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); BearTokenCredentialsProvider credentialsProvider = (BearTokenCredentialsProvider) CredentialsProviderFactory.createCredentialsProvider( @@ -55,8 +54,6 @@ public void testCreateBearTokenCredentialsProviderSuccess() { @Test public void testCreateBearTokenCredentialsProviderFail() { Options options = new Options(); - options.set( - RESTCatalogOptions.CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN.name()); assertThrows( IllegalArgumentException.class, () -> @@ -72,9 +69,7 @@ public void testCreateBearTokenFileCredentialsProviderSuccess() throws Exception String token = UUID.randomUUID().toString(); FileUtils.writeStringToFile(tokenFile, token); options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); - options.set( - RESTCatalogOptions.CREDENTIALS_PROVIDER, - CredentialsProviderType.BEAR_TOKEN_FILE.name()); + options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); BearTokenFileCredentialsProvider credentialsProvider = (BearTokenFileCredentialsProvider) CredentialsProviderFactory.createCredentialsProvider( @@ -85,9 +80,7 @@ public void testCreateBearTokenFileCredentialsProviderSuccess() throws Exception @Test public void testCreateBearTokenFileCredentialsProviderFail() throws Exception { Options options = new Options(); - options.set( - RESTCatalogOptions.CREDENTIALS_PROVIDER, - CredentialsProviderType.BEAR_TOKEN_FILE.name()); + options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); assertThrows( IllegalArgumentException.class, () -> @@ -102,9 +95,7 @@ public void testCreateRefreshBearTokenFileCredentialsProviderSuccess() throws Ex File tokenFile = folder.newFile(fileName); String token = UUID.randomUUID().toString(); FileUtils.writeStringToFile(tokenFile, token); - options.set( - RESTCatalogOptions.CREDENTIALS_PROVIDER, - CredentialsProviderType.BEAR_TOKEN_FILE.name()); + options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); options.set(RESTCatalogOptions.TOKEN_REFRESH_ENABLED, true); options.set(RESTCatalogOptions.TOKEN_EXPIRATION_TIME, Duration.ofSeconds(10L)); @@ -122,18 +113,40 @@ public void testCreateRefreshBearTokenFileCredentialsProviderFail() throws Excep File tokenFile = folder.newFile(fileName); String token = UUID.randomUUID().toString(); FileUtils.writeStringToFile(tokenFile, token); - options.set( - RESTCatalogOptions.CREDENTIALS_PROVIDER, - CredentialsProviderType.BEAR_TOKEN_FILE.name()); + options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); options.set(RESTCatalogOptions.TOKEN_REFRESH_ENABLED, true); - options.set( - RESTCatalogOptions.CREDENTIALS_PROVIDER, - CredentialsProviderType.BEAR_TOKEN_FILE.name()); + options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); assertThrows( IllegalArgumentException.class, () -> CredentialsProviderFactory.createCredentialsProvider( options, this.getClass().getClassLoader())); } + + @Test + public void getCredentialsProviderTypeByConfWhenDefineTokenPath() { + Options options = new Options(); + options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, "/a/b/c"); + assertEquals( + CredentialsProviderType.BEAR_TOKEN_FILE, + CredentialsProviderFactory.getCredentialsProviderTypeByConf(options)); + } + + @Test + public void getCredentialsProviderTypeByConfWhenConfNotDefined() { + Options options = new Options(); + assertEquals( + CredentialsProviderType.BEAR_TOKEN, + CredentialsProviderFactory.getCredentialsProviderTypeByConf(options)); + } + + @Test + public void getCredentialsProviderTypeByConfWhenDefineProviderType() { + Options options = new Options(); + options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + assertEquals( + CredentialsProviderType.BEAR_TOKEN_FILE, + CredentialsProviderFactory.getCredentialsProviderTypeByConf(options)); + } } From 7498e7e5b2aedf75e94a46b1e9306670a165ba25 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 9 Dec 2024 14:42:05 +0800 Subject: [PATCH 21/26] add comment for TOKEN_EXPIRATION_TIME --- .../main/java/org/apache/paimon/rest/RESTCatalogOptions.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index 85fac413bba6..71dcc0268d71 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -54,7 +54,10 @@ public class RESTCatalogOptions { ConfigOptions.key("token.expiration-time") .durationType() .defaultValue(Duration.ofHours(1)) - .withDescription("REST Catalog auth token expires in."); + .withDescription( + "REST Catalog auth token expires in.The token generates system refresh frequency is t1," + + " the token expires time is t2, we need to guarantee that t2 > t1," + + " the token validity time is [t2 - t1, t2], and the expires time defined here needs to be less than (t2 - t1)"); public static final ConfigOption TOKEN_REFRESH_ENABLED = ConfigOptions.key("token-refresh-enabled") .booleanType() From 314f965d14894b3b0cb8a74ddbd6d150cc068532 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 9 Dec 2024 14:52:56 +0800 Subject: [PATCH 22/26] delete TOKEN_REFRESH_ENABLED judge whether refresh by credentials provider --- .../org/apache/paimon/rest/RESTCatalog.java | 8 +------ .../paimon/rest/RESTCatalogOptions.java | 11 +++------- ...arTokenFileCredentialsProviderFactory.java | 8 +------ .../auth/CredentialsProviderFactoryTest.java | 21 ------------------- 4 files changed, 5 insertions(+), 43 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index ab1799a56367..e18946b3374b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -54,7 +54,6 @@ public class RESTCatalog implements Catalog { // a lazy thread pool for token refresh private final AuthSession catalogAuth; private volatile ScheduledExecutorService refreshExecutor = null; - private boolean keepTokenRefreshed; private static final ObjectMapper objectMapper = RESTObjectMapper.create(); @@ -80,8 +79,7 @@ public RESTCatalog(Options options) { CredentialsProvider credentialsProvider = CredentialsProviderFactory.createCredentialsProvider( options, RESTCatalog.class.getClassLoader()); - this.keepTokenRefreshed = options.get(RESTCatalogOptions.TOKEN_REFRESH_ENABLED); - if (keepTokenRefreshed) { + if (credentialsProvider.keepRefreshed()) { this.catalogAuth = AuthSession.fromRefreshCredentialsProvider( tokenRefreshExecutor(), this.baseHeader, credentialsProvider); @@ -215,10 +213,6 @@ private Map headers() { } private ScheduledExecutorService tokenRefreshExecutor() { - if (!keepTokenRefreshed) { - return null; - } - if (refreshExecutor == null) { synchronized (this) { if (refreshExecutor == null) { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index 71dcc0268d71..1e62d2178ee6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -55,14 +55,9 @@ public class RESTCatalogOptions { .durationType() .defaultValue(Duration.ofHours(1)) .withDescription( - "REST Catalog auth token expires in.The token generates system refresh frequency is t1," + - " the token expires time is t2, we need to guarantee that t2 > t1," + - " the token validity time is [t2 - t1, t2], and the expires time defined here needs to be less than (t2 - t1)"); - public static final ConfigOption TOKEN_REFRESH_ENABLED = - ConfigOptions.key("token-refresh-enabled") - .booleanType() - .defaultValue(false) - .withDescription("REST Catalog auth token refresh enable."); + "REST Catalog auth token expires in.The token generates system refresh frequency is t1," + + " the token expires time is t2, we need to guarantee that t2 > t1," + + " the token validity time is [t2 - t1, t2], and the expires time defined here needs to be less than (t2 - t1)"); public static final ConfigOption TOKEN_PROVIDER_PATH = ConfigOptions.key("token.provider.path") .stringType() diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java index b7a99639289e..a0fa6b405d62 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java @@ -19,7 +19,6 @@ package org.apache.paimon.rest.auth; import org.apache.paimon.options.Options; -import org.apache.paimon.rest.RESTCatalogOptions; import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_EXPIRATION_TIME; import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER_PATH; @@ -38,12 +37,7 @@ public CredentialsProvider create(Options options) { throw new IllegalArgumentException(TOKEN_PROVIDER_PATH.key() + " is required"); } String tokenFilePath = options.get(TOKEN_PROVIDER_PATH); - boolean keepTokenRefreshed = options.get(RESTCatalogOptions.TOKEN_REFRESH_ENABLED); - if (keepTokenRefreshed) { - if (!options.getOptional(TOKEN_EXPIRATION_TIME).isPresent()) { - throw new IllegalArgumentException( - TOKEN_EXPIRATION_TIME.key() + " is required when token refresh enabled"); - } + if (options.getOptional(TOKEN_EXPIRATION_TIME).isPresent()) { long tokenExpireInMills = options.get(TOKEN_EXPIRATION_TIME).toMillis(); return new BearTokenFileCredentialsProvider(tokenFilePath, tokenExpireInMills); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java index c676064e1cf6..e62a65a79aed 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java @@ -69,7 +69,6 @@ public void testCreateBearTokenFileCredentialsProviderSuccess() throws Exception String token = UUID.randomUUID().toString(); FileUtils.writeStringToFile(tokenFile, token); options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); - options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); BearTokenFileCredentialsProvider credentialsProvider = (BearTokenFileCredentialsProvider) CredentialsProviderFactory.createCredentialsProvider( @@ -95,9 +94,7 @@ public void testCreateRefreshBearTokenFileCredentialsProviderSuccess() throws Ex File tokenFile = folder.newFile(fileName); String token = UUID.randomUUID().toString(); FileUtils.writeStringToFile(tokenFile, token); - options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); - options.set(RESTCatalogOptions.TOKEN_REFRESH_ENABLED, true); options.set(RESTCatalogOptions.TOKEN_EXPIRATION_TIME, Duration.ofSeconds(10L)); BearTokenFileCredentialsProvider credentialsProvider = (BearTokenFileCredentialsProvider) @@ -106,24 +103,6 @@ public void testCreateRefreshBearTokenFileCredentialsProviderSuccess() throws Ex assertEquals(token, credentialsProvider.token()); } - @Test - public void testCreateRefreshBearTokenFileCredentialsProviderFail() throws Exception { - Options options = new Options(); - String fileName = "token"; - File tokenFile = folder.newFile(fileName); - String token = UUID.randomUUID().toString(); - FileUtils.writeStringToFile(tokenFile, token); - options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); - options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); - options.set(RESTCatalogOptions.TOKEN_REFRESH_ENABLED, true); - options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); - assertThrows( - IllegalArgumentException.class, - () -> - CredentialsProviderFactory.createCredentialsProvider( - options, this.getClass().getClassLoader())); - } - @Test public void getCredentialsProviderTypeByConfWhenDefineTokenPath() { Options options = new Options(); From 4a08b0b863e9bb2968b6aa9ccdb33f07bccb0d6a Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 9 Dec 2024 15:35:43 +0800 Subject: [PATCH 23/26] update fromRefreshCredentialsProvider in AuthSession and format some class --- .../apache/paimon/rest/auth/AuthSession.java | 43 +++++++------------ .../BaseBearTokenCredentialsProvider.java | 1 + .../BearTokenCredentialsProviderFactory.java | 1 + .../paimon/rest/auth/CredentialsProvider.java | 1 + 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index 62405113f509..a3e34ac532df 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -19,7 +19,6 @@ package org.apache.paimon.rest.auth; import org.apache.paimon.rest.RESTUtil; -import org.apache.paimon.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,22 +51,12 @@ public static AuthSession fromRefreshCredentialsProvider( long startTimeMillis = System.currentTimeMillis(); Optional expiresAtMillisOpt = credentialsProvider.expiresAtMillis(); + // when init session if credentials expire time is in the past, refresh it and update + // expiresAtMillis if (expiresAtMillisOpt.isPresent() && expiresAtMillisOpt.get() <= startTimeMillis) { - Pair refreshResult = session.refresh(); - - // if expiration is non-null, then token refresh was successful - boolean isSuccessful = refreshResult.getKey(); - if (isSuccessful) { - if (session.credentialsProvider.expiresAtMillis().isPresent()) { - // use the new expiration time from the refreshed token - expiresAtMillisOpt = session.credentialsProvider.expiresAtMillis(); - } else { - // otherwise use the expiration time from the token response - expiresAtMillisOpt = Optional.of(startTimeMillis + refreshResult.getValue()); - } - } else { - // token refresh failed, don't reattempt with the original expiration - expiresAtMillisOpt = Optional.empty(); + boolean refreshSuccessful = session.refresh(); + if (refreshSuccessful) { + expiresAtMillisOpt = session.credentialsProvider.expiresAtMillis(); } } @@ -98,7 +87,7 @@ private static void scheduleTokenRefresh( if (retryTimes < TOKEN_REFRESH_NUM_RETRIES) { long expiresInMillis = expiresAtMillis - System.currentTimeMillis(); // how much ahead of time to start the request to allow it to complete - long refreshWindowMillis = Math.min(expiresInMillis / 10, MAX_REFRESH_WINDOW_MILLIS); + long refreshWindowMillis = Math.min(expiresInMillis, MAX_REFRESH_WINDOW_MILLIS); // how much time to wait before expiration long waitIntervalMillis = expiresInMillis - refreshWindowMillis; // how much time to actually wait @@ -107,13 +96,13 @@ private static void scheduleTokenRefresh( executor.schedule( () -> { long refreshStartTime = System.currentTimeMillis(); - Pair refreshResult = session.refresh(); - boolean isSuccessful = refreshResult.getKey(); + boolean isSuccessful = session.refresh(); if (isSuccessful) { scheduleTokenRefresh( executor, session, - refreshStartTime + refreshResult.getValue(), + refreshStartTime + + session.credentialsProvider.expiresInMills().get(), 0); } else { scheduleTokenRefresh( @@ -127,19 +116,19 @@ private static void scheduleTokenRefresh( } } - public Pair refresh() { + public Boolean refresh() { if (this.credentialsProvider.supportRefresh() && this.credentialsProvider.keepRefreshed() && this.credentialsProvider.expiresInMills().isPresent()) { boolean isSuccessful = this.credentialsProvider.refresh(); - if (!isSuccessful) { - return Pair.of(false, 0L); + if (isSuccessful) { + Map currentHeaders = this.headers; + this.headers = + RESTUtil.merge(currentHeaders, this.credentialsProvider.authHeader()); } - Map currentHeaders = this.headers; - this.headers = RESTUtil.merge(currentHeaders, this.credentialsProvider.authHeader()); - return Pair.of(true, credentialsProvider.expiresInMills().get()); + return isSuccessful; } - return Pair.of(false, 0L); + return false; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java index a7f1a2b459f1..d3df87826164 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java @@ -24,6 +24,7 @@ /** Base bear token credentials provider. */ public abstract class BaseBearTokenCredentialsProvider implements CredentialsProvider { + private static final String AUTHORIZATION_HEADER = "Authorization"; private static final String BEARER_PREFIX = "Bearer "; diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java index cf3864025107..e63ac5606b01 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java @@ -24,6 +24,7 @@ /** factory for create {@link BearTokenCredentialsProvider}. */ public class BearTokenCredentialsProviderFactory implements CredentialsProviderFactory { + @Override public String identifier() { return CredentialsProviderType.BEAR_TOKEN.name(); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java index b927ff980baa..7fe8008e5947 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java @@ -23,6 +23,7 @@ /** Credentials provider. */ public interface CredentialsProvider { + Map authHeader(); boolean refresh(); From 5e09533df210f965b3b8ce50b50cd05278fc82f1 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 9 Dec 2024 16:02:51 +0800 Subject: [PATCH 24/26] update comment in AuthSession and delete validate in BearTokenCredentialsProvider --- .../java/org/apache/paimon/rest/auth/AuthSession.java | 2 +- .../paimon/rest/auth/BearTokenCredentialsProvider.java | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index a3e34ac532df..29b4b7272c01 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -86,7 +86,7 @@ private static void scheduleTokenRefresh( int retryTimes) { if (retryTimes < TOKEN_REFRESH_NUM_RETRIES) { long expiresInMillis = expiresAtMillis - System.currentTimeMillis(); - // how much ahead of time to start the request to allow it to complete + // how much ahead of time to start the refresh to allow it to complete long refreshWindowMillis = Math.min(expiresInMillis, MAX_REFRESH_WINDOW_MILLIS); // how much time to wait before expiration long waitIntervalMillis = expiresInMillis - refreshWindowMillis; diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java index dfe9725fbf39..89228fe10b28 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java @@ -18,19 +18,13 @@ package org.apache.paimon.rest.auth; -import org.apache.paimon.utils.StringUtils; - /** credentials provider for bear token. */ public class BearTokenCredentialsProvider extends BaseBearTokenCredentialsProvider { private final String token; public BearTokenCredentialsProvider(String token) { - if (StringUtils.isNullOrWhitespaceOnly(token)) { - throw new IllegalArgumentException("token is null"); - } else { - this.token = token; - } + this.token = token; } @Override From e348da19af38eaae8295c8f2db4a9fc4611e9474 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 9 Dec 2024 16:31:53 +0800 Subject: [PATCH 25/26] add UT for retry when refresh fail --- .../apache/paimon/rest/auth/AuthSession.java | 7 +++-- .../paimon/rest/auth/AuthSessionTest.java | 27 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java index 29b4b7272c01..74efb8508a06 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -18,6 +18,7 @@ package org.apache.paimon.rest.auth; +import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.rest.RESTUtil; import org.slf4j.Logger; @@ -30,8 +31,9 @@ /** Auth session. */ public class AuthSession { + + static final int TOKEN_REFRESH_NUM_RETRIES = 5; private static final Logger log = LoggerFactory.getLogger(AuthSession.class); - private static final int TOKEN_REFRESH_NUM_RETRIES = 5; private static final long MAX_REFRESH_WINDOW_MILLIS = 300_000; // 5 minutes private static final long MIN_REFRESH_WAIT_MILLIS = 10; private final CredentialsProvider credentialsProvider; @@ -74,7 +76,8 @@ public Map getHeaders() { return headers; } - private static void scheduleTokenRefresh( + @VisibleForTesting + static void scheduleTokenRefresh( ScheduledExecutorService executor, AuthSession session, long expiresAtMillis) { scheduleTokenRefresh(executor, session, expiresAtMillis, 0); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java index a614f83283d5..81b3ea57b703 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java @@ -25,15 +25,20 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ScheduledExecutorService; +import static org.apache.paimon.rest.auth.AuthSession.TOKEN_REFRESH_NUM_RETRIES; import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Test for {@link AuthSession}. */ public class AuthSessionTest { @@ -94,6 +99,28 @@ public void testRefreshCredentialsProviderIsSoonExpire() assertEquals(header.get("Authorization"), "Bearer " + token); } + @Test + public void testRetryWhenRefreshFail() throws Exception { + Map initialHeaders = new HashMap<>(); + CredentialsProvider credentialsProvider = + Mockito.mock(BearTokenFileCredentialsProvider.class); + long expiresAtMillis = System.currentTimeMillis() - 1000L; + when(credentialsProvider.expiresAtMillis()).thenReturn(Optional.of(expiresAtMillis)); + when(credentialsProvider.expiresInMills()).thenReturn(Optional.of(50L)); + when(credentialsProvider.supportRefresh()).thenReturn(true); + when(credentialsProvider.keepRefreshed()).thenReturn(true); + when(credentialsProvider.refresh()).thenReturn(false); + AuthSession session = + AuthSession.fromRefreshCredentialsProvider( + null, initialHeaders, credentialsProvider); + AuthSession.scheduleTokenRefresh( + ThreadPoolUtils.createScheduledThreadPool(1, "refresh-token"), + session, + expiresAtMillis); + Thread.sleep(10_000L); + verify(credentialsProvider, Mockito.times(TOKEN_REFRESH_NUM_RETRIES + 1)).refresh(); + } + private Pair generateTokenAndWriteToFile(String fileName) throws IOException { File tokenFile = folder.newFile(fileName); String token = UUID.randomUUID().toString(); From 944a45241df2d73b02935637aa2526e6ea5a4057 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 9 Dec 2024 16:52:33 +0800 Subject: [PATCH 26/26] update desc for TOKEN_EXPIRATION_TIME and TOKEN_PROVIDER_PATH in RESTCatalogOptions --- .../java/org/apache/paimon/rest/RESTCatalogOptions.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index 1e62d2178ee6..8f7bea91dcd3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -55,12 +55,13 @@ public class RESTCatalogOptions { .durationType() .defaultValue(Duration.ofHours(1)) .withDescription( - "REST Catalog auth token expires in.The token generates system refresh frequency is t1," + "REST Catalog auth token expires time.The token generates system refresh frequency is t1," + " the token expires time is t2, we need to guarantee that t2 > t1," - + " the token validity time is [t2 - t1, t2], and the expires time defined here needs to be less than (t2 - t1)"); + + " the token validity time is [t2 - t1, t2]," + + " and the expires time defined here needs to be less than (t2 - t1)"); public static final ConfigOption TOKEN_PROVIDER_PATH = ConfigOptions.key("token.provider.path") .stringType() .noDefaultValue() - .withDescription("REST Catalog auth token file path."); + .withDescription("REST Catalog auth token provider path."); }