Skip to content

GCP: Add properties for OAuth2 and update library #8073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danielcweeks merged 2 commits into from
Jul 21, 2023
Merged

GCP: Add properties for OAuth2 and update library #8073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
310607a
GCP: Add properties for OAtuh2 and update library
bryanck Jul 15, 2023
a12dfad
Merge branch 'master' into gcp-oauth-props
danielcweeks Jul 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion core/src/main/java/org/apache/iceberg/io/ResolvingFileIO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,13 @@ public class ResolvingFileIO implements FileIO, HadoopConfigurable {
private static final Logger LOG = LoggerFactory.getLogger(ResolvingFileIO.class);
private static final String FALLBACK_IMPL = "org.apache.iceberg.hadoop.HadoopFileIO";
private static final String S3_FILE_IO_IMPL = "org.apache.iceberg.aws.s3.S3FileIO";
private static final String GCS_FILE_IO_IMPL = "org.apache.iceberg.gcp.gcs.GCSFileIO";
private static final Map<String, String> SCHEME_TO_FILE_IO =
ImmutableMap.of(
"s3", S3_FILE_IO_IMPL,
"s3a", S3_FILE_IO_IMPL,
"s3n", S3_FILE_IO_IMPL);
"s3n", S3_FILE_IO_IMPL,
"gs", GCS_FILE_IO_IMPL);

private final Map<String, FileIO> ioInstances = Maps.newHashMap();
private final AtomicBoolean isClosed = new AtomicBoolean(false);
Expand Down
21 changes: 21 additions & 0 deletions gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.iceberg.gcp;

import java.io.Serializable;
import java.util.Date;
import java.util.Map;
import java.util.Optional;

Expand All @@ -36,6 +37,9 @@ public class GCPProperties implements Serializable {
public static final String GCS_CHANNEL_READ_CHUNK_SIZE = "gcs.channel.read.chunk-size-bytes";
public static final String GCS_CHANNEL_WRITE_CHUNK_SIZE = "gcs.channel.write.chunk-size-bytes";

public static final String GCS_OAUTH2_TOKEN = "gcs.oauth2.token";
public static final String GCS_OAUTH2_TOKEN_EXPIRES_AT = "gcs.oauth2.token-expires-at";

private String projectId;
private String clientLibToken;
private String serviceHost;
Expand All @@ -47,6 +51,9 @@ public class GCPProperties implements Serializable {
private Integer gcsChannelReadChunkSize;
private Integer gcsChannelWriteChunkSize;

private String gcsOAuth2Token;
private Date gcsOAuth2TokenExpiresAt;

public GCPProperties() {}

public GCPProperties(Map<String, String> properties) {
Expand All @@ -65,6 +72,12 @@ public GCPProperties(Map<String, String> properties) {
if (properties.containsKey(GCS_CHANNEL_WRITE_CHUNK_SIZE)) {
gcsChannelWriteChunkSize = Integer.parseInt(properties.get(GCS_CHANNEL_WRITE_CHUNK_SIZE));
}

gcsOAuth2Token = properties.get(GCS_OAUTH2_TOKEN);
if (properties.containsKey(GCS_OAUTH2_TOKEN_EXPIRES_AT)) {
gcsOAuth2TokenExpiresAt =
new Date(Long.parseLong(properties.get(GCS_OAUTH2_TOKEN_EXPIRES_AT)));
}
}

public Optional<Integer> channelReadChunkSize() {
Expand Down Expand Up @@ -98,4 +111,12 @@ public Optional<String> serviceHost() {
public Optional<String> userProject() {
return Optional.ofNullable(gcsUserProject);
}

public Optional<String> oauth2Token() {
return Optional.ofNullable(gcsOAuth2Token);
}

public Optional<Date> oauth2TokenExpiresAt() {
return Optional.ofNullable(gcsOAuth2TokenExpiresAt);
}
}
11 changes: 11 additions & 0 deletions gcp/src/main/java/org/apache/iceberg/gcp/gcs/GCSFileIO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
*/
package org.apache.iceberg.gcp.gcs;

import com.google.auth.oauth2.AccessToken;
import com.google.auth.oauth2.OAuth2Credentials;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
Expand Down Expand Up @@ -133,6 +135,15 @@ public void initialize(Map<String, String> props) {
gcpProperties.clientLibToken().ifPresent(builder::setClientLibToken);
gcpProperties.serviceHost().ifPresent(builder::setHost);

gcpProperties
.oauth2Token()
.ifPresent(
token -> {
AccessToken accessToken =
new AccessToken(token, gcpProperties.oauth2TokenExpiresAt().orElse(null));
builder.setCredentials(OAuth2Credentials.create(accessToken));
});

// Report Hadoop metrics if Hadoop is available
try {
DynConstructors.Ctor<MetricsContext> ctor =
Expand Down
2 changes: 1 addition & 1 deletion versions.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ javax.activation:activation = 1.1.1
org.glassfish.jaxb:jaxb-runtime = 2.3.3
software.amazon.awssdk:* = 2.20.18
org.projectnessie.nessie:* = 0.65.0
com.google.cloud:libraries-bom = 24.1.0
com.google.cloud:libraries-bom = 26.18.0
org.scala-lang.modules:scala-collection-compat_2.12 = 2.11.0
org.scala-lang.modules:scala-collection-compat_2.13 = 2.11.0
com.emc.ecs:object-client-bundle = 3.3.2
Expand Down