diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java index 71aecbdb5962..70ccd33253d2 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java @@ -32,31 +32,29 @@ public class CanonicalExtensionHeadersSerializer { private static final char HEADER_SEPARATOR = ':'; + private static final char HEADER_NAME_SEPARATOR = ';'; + + private final Storage.SignUrlOption.SignatureVersion signatureVersion; + + public CanonicalExtensionHeadersSerializer( + Storage.SignUrlOption.SignatureVersion signatureVersion) { + this.signatureVersion = signatureVersion; + } + + public CanonicalExtensionHeadersSerializer() { + // TODO switch this when V4 becomes default + this.signatureVersion = Storage.SignUrlOption.SignatureVersion.V2; + } public StringBuilder serialize(Map canonicalizedExtensionHeaders) { StringBuilder serializedHeaders = new StringBuilder(); if (canonicalizedExtensionHeaders == null || canonicalizedExtensionHeaders.isEmpty()) { - return serializedHeaders; } - // Make all custom header names lowercase. - Map lowercaseHeaders = new HashMap<>(); - for (String headerName : new ArrayList<>(canonicalizedExtensionHeaders.keySet())) { - - String lowercaseHeaderName = headerName.toLowerCase(); - - // If present, remove the x-goog-encryption-key and x-goog-encryption-key-sha256 headers. - if ("x-goog-encryption-key".equals(lowercaseHeaderName) - || "x-goog-encryption-key-sha256".equals(lowercaseHeaderName)) { - - continue; - } - - lowercaseHeaders.put(lowercaseHeaderName, canonicalizedExtensionHeaders.get(headerName)); - } + Map lowercaseHeaders = getLowercaseHeaders(canonicalizedExtensionHeaders); // Sort all custom headers by header name using a lexicographical sort by code point value. List sortedHeaderNames = new ArrayList<>(lowercaseHeaders.keySet()); @@ -81,4 +79,47 @@ public StringBuilder serialize(Map canonicalizedExtensionHeaders // Concatenate all custom headers return serializedHeaders; } + + public StringBuilder serializeHeaderNames(Map canonicalizedExtensionHeaders) { + StringBuilder serializedHeaders = new StringBuilder(); + + if (canonicalizedExtensionHeaders == null || canonicalizedExtensionHeaders.isEmpty()) { + return serializedHeaders; + } + Map lowercaseHeaders = getLowercaseHeaders(canonicalizedExtensionHeaders); + + List sortedHeaderNames = new ArrayList<>(lowercaseHeaders.keySet()); + Collections.sort(sortedHeaderNames); + + for (String headerName : sortedHeaderNames) { + serializedHeaders.append(headerName).append(HEADER_NAME_SEPARATOR); + } + + serializedHeaders.setLength(serializedHeaders.length() - 1); // remove trailing semicolon + + return serializedHeaders; + } + + private Map getLowercaseHeaders( + Map canonicalizedExtensionHeaders) { + // Make all custom header names lowercase. + Map lowercaseHeaders = new HashMap<>(); + for (String headerName : new ArrayList<>(canonicalizedExtensionHeaders.keySet())) { + + String lowercaseHeaderName = headerName.toLowerCase(); + + // If present and we're V2, remove the x-goog-encryption-key and x-goog-encryption-key-sha256 + // headers. (CSEK headers are allowed for V4) + if (Storage.SignUrlOption.SignatureVersion.V2.equals(signatureVersion) + && ("x-goog-encryption-key".equals(lowercaseHeaderName) + || "x-goog-encryption-key-sha256".equals(lowercaseHeaderName))) { + + continue; + } + + lowercaseHeaders.put(lowercaseHeaderName, canonicalizedExtensionHeaders.get(headerName)); + } + + return lowercaseHeaders; + } } diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java index 369776ffd9dc..7346390b3840 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java @@ -18,8 +18,16 @@ import static com.google.common.base.Preconditions.checkArgument; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.common.net.UrlEscapers; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; import java.util.Map; +import java.util.TimeZone; /** * Signature Info holds payload components of the string that requires signing. @@ -31,6 +39,8 @@ public class SignatureInfo { public static final char COMPONENT_SEPARATOR = '\n'; + public static final String GOOG4_RSA_SHA256 = "GOOG4-RSA-SHA256"; + public static final String SCOPE = "/auto/storage/goog4_request"; private final HttpMethod httpVerb; private final String contentMd5; @@ -38,23 +48,61 @@ public class SignatureInfo { private final long expiration; private final Map canonicalizedExtensionHeaders; private final URI canonicalizedResource; + private final Storage.SignUrlOption.SignatureVersion signatureVersion; + private final String accountEmail; + private final long timestamp; + + private final String yearMonthDay; + private final String exactDate; private SignatureInfo(Builder builder) { this.httpVerb = builder.httpVerb; this.contentMd5 = builder.contentMd5; this.contentType = builder.contentType; this.expiration = builder.expiration; - this.canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders; this.canonicalizedResource = builder.canonicalizedResource; + this.signatureVersion = builder.signatureVersion; + this.accountEmail = builder.accountEmail; + this.timestamp = builder.timestamp; + + if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion) + && (!builder.canonicalizedExtensionHeaders.containsKey("host"))) { + canonicalizedExtensionHeaders = + new ImmutableMap.Builder() + .putAll(builder.canonicalizedExtensionHeaders) + .put("host", "storage.googleapis.com") + .build(); + } else { + canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders; + } + + Date date = new Date(timestamp); + + SimpleDateFormat yearMonthDayFormat = new SimpleDateFormat("yyyyMMdd"); + SimpleDateFormat exactDateFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'"); + + yearMonthDayFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + exactDateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + + yearMonthDay = yearMonthDayFormat.format(date); + exactDate = exactDateFormat.format(date); } /** * Constructs payload to be signed. * - * @return paylod to sign + * @return payload to sign * @see Signed URLs */ public String constructUnsignedPayload() { + // TODO reverse order when V4 becomes default + if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)) { + return constructV4UnsignedPayload(); + } + return constructV2UnsignedPayload(); + } + + private String constructV2UnsignedPayload() { StringBuilder payload = new StringBuilder(); payload.append(httpVerb.name()).append(COMPONENT_SEPARATOR); @@ -67,12 +115,12 @@ public String constructUnsignedPayload() { payload.append(contentType); } payload.append(COMPONENT_SEPARATOR); - payload.append(expiration).append(COMPONENT_SEPARATOR); if (canonicalizedExtensionHeaders != null) { payload.append( - new CanonicalExtensionHeadersSerializer().serialize(canonicalizedExtensionHeaders)); + new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V2) + .serialize(canonicalizedExtensionHeaders)); } payload.append(canonicalizedResource); @@ -80,6 +128,59 @@ public String constructUnsignedPayload() { return payload.toString(); } + private String constructV4UnsignedPayload() { + StringBuilder payload = new StringBuilder(); + + payload.append(GOOG4_RSA_SHA256).append(COMPONENT_SEPARATOR); + payload.append(exactDate).append(COMPONENT_SEPARATOR); + payload.append(yearMonthDay).append(SCOPE).append(COMPONENT_SEPARATOR); + payload.append(constructV4CanonicalRequestHash()); + + return payload.toString(); + } + + private String constructV4CanonicalRequestHash() { + StringBuilder canonicalRequest = new StringBuilder(); + + CanonicalExtensionHeadersSerializer serializer = + new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V4); + + canonicalRequest.append(httpVerb.name()).append(COMPONENT_SEPARATOR); + canonicalRequest.append(canonicalizedResource).append(COMPONENT_SEPARATOR); + canonicalRequest.append(constructV4QueryString()).append(COMPONENT_SEPARATOR); + canonicalRequest + .append(serializer.serialize(canonicalizedExtensionHeaders)) + .append(COMPONENT_SEPARATOR); + canonicalRequest + .append(serializer.serializeHeaderNames(canonicalizedExtensionHeaders)) + .append(COMPONENT_SEPARATOR); + canonicalRequest.append("UNSIGNED-PAYLOAD"); + + return Hashing.sha256() + .hashString(canonicalRequest.toString(), StandardCharsets.UTF_8) + .toString(); + } + + public String constructV4QueryString() { + StringBuilder signedHeaders = + new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V4) + .serializeHeaderNames(canonicalizedExtensionHeaders); + + StringBuilder queryString = new StringBuilder(); + queryString.append("X-Goog-Algorithm=").append(GOOG4_RSA_SHA256).append("&"); + queryString.append( + "X-Goog-Credential=" + + UrlEscapers.urlFormParameterEscaper() + .escape(accountEmail + "/" + yearMonthDay + SCOPE) + + "&"); + queryString.append("X-Goog-Date=" + exactDate + "&"); + queryString.append("X-Goog-Expires=" + expiration + "&"); + queryString.append( + "X-Goog-SignedHeaders=" + + UrlEscapers.urlFormParameterEscaper().escape(signedHeaders.toString())); + return queryString.toString(); + } + public HttpMethod getHttpVerb() { return httpVerb; } @@ -104,6 +205,18 @@ public URI getCanonicalizedResource() { return canonicalizedResource; } + public Storage.SignUrlOption.SignatureVersion getSignatureVersion() { + return signatureVersion; + } + + public long getTimestamp() { + return timestamp; + } + + public String getAccountEmail() { + return accountEmail; + } + public static final class Builder { private final HttpMethod httpVerb; @@ -112,6 +225,9 @@ public static final class Builder { private final long expiration; private Map canonicalizedExtensionHeaders; private final URI canonicalizedResource; + private Storage.SignUrlOption.SignatureVersion signatureVersion; + private String accountEmail; + private long timestamp; /** * Constructs builder. @@ -134,6 +250,9 @@ public Builder(SignatureInfo signatureInfo) { this.expiration = signatureInfo.expiration; this.canonicalizedExtensionHeaders = signatureInfo.canonicalizedExtensionHeaders; this.canonicalizedResource = signatureInfo.canonicalizedResource; + this.signatureVersion = signatureInfo.signatureVersion; + this.accountEmail = signatureInfo.accountEmail; + this.timestamp = signatureInfo.timestamp; } public Builder setContentMd5(String contentMd5) { @@ -155,12 +274,41 @@ public Builder setCanonicalizedExtensionHeaders( return this; } + public Builder setSignatureVersion(Storage.SignUrlOption.SignatureVersion signatureVersion) { + this.signatureVersion = signatureVersion; + + return this; + } + + public Builder setAccountEmail(String accountEmail) { + this.accountEmail = accountEmail; + + return this; + } + + public Builder setTimestamp(long timestamp) { + this.timestamp = timestamp; + + return this; + } + /** Creates an {@code SignatureInfo} object from this builder. */ public SignatureInfo build() { checkArgument(httpVerb != null, "Required HTTP method"); checkArgument(canonicalizedResource != null, "Required canonicalized resource"); checkArgument(expiration >= 0, "Expiration must be greater than or equal to zero"); + if (Storage.SignUrlOption.SignatureVersion.V4.equals(signatureVersion)) { + checkArgument(accountEmail != null, "Account email required to use V4 signing"); + checkArgument(timestamp > 0, "Timestamp required to use V4 signing"); + checkArgument( + expiration <= 604800, "Expiration can't be longer than 7 days to use V4 signing"); + } + + if (canonicalizedExtensionHeaders == null) { + canonicalizedExtensionHeaders = new HashMap<>(); + } + return new SignatureInfo(this); } } diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 9d2d0d5c33f2..720664e40a57 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -886,9 +886,15 @@ enum Option { MD5, EXT_HEADERS, SERVICE_ACCOUNT_CRED, + SIGNATURE_VERSION, HOST_NAME } + enum SignatureVersion { + V2, + V4 + } + private SignUrlOption(Option option, Object value) { this.option = option; this.value = value; @@ -937,6 +943,23 @@ public static SignUrlOption withExtHeaders(Map extHeaders) { return new SignUrlOption(Option.EXT_HEADERS, extHeaders); } + /** + * Use if signature version should be V2. This is the default if neither this or {@code + * withV4Signature()} is called. + */ + public static SignUrlOption withV2Signature() { + return new SignUrlOption(Option.SIGNATURE_VERSION, SignatureVersion.V2); + } + + /** + * Use if signature version should be V4. Note that V4 Signed URLs can't have an expiration + * longer than 7 days. V2 will be the default if neither this or {@code withV2Signature()} is + * called. + */ + public static SignUrlOption withV4Signature() { + return new SignUrlOption(Option.SIGNATURE_VERSION, SignatureVersion.V4); + } + /** * Provides a service account signer to sign the URL. If not provided an attempt will be made to * get it from the environment. @@ -2101,6 +2124,16 @@ Blob create( * TimeUnit.DAYS); * } * + *

Example of creating a signed URL passing the {@link SignUrlOption#withV4Signature()} option, + * which enables V4 signing. + * + *

{@code
+   * String bucketName = "my_unique_bucket";
+   * String blobName = "my_blob_name";
+   * URL signedUrl = storage.signUrl(BlobInfo.newBuilder(bucketName, blobName).build(),
+   *     7, TimeUnit.DAYS, Storage.SignUrlOption.withV4Signature());
+   * }
+ * *

Example of creating a signed URL passing the {@link * SignUrlOption#signWith(ServiceAccountSigner)} option, that will be used for signing the URL. * diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index ac4da4c32eea..6611613f45a3 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -52,6 +52,7 @@ import com.google.cloud.storage.spi.v1.StorageRpc.RewriteResponse; import com.google.common.base.Function; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -609,6 +610,11 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio for (SignUrlOption option : options) { optionMap.put(option.getOption(), option.getValue()); } + + boolean isV4 = + SignUrlOption.SignatureVersion.V4.equals( + optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); + ServiceAccountSigner credentials = (ServiceAccountSigner) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED); if (credentials == null) { @@ -619,15 +625,19 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio } long expiration = - TimeUnit.SECONDS.convert( - getOptions().getClock().millisTime() + unit.toMillis(duration), TimeUnit.MILLISECONDS); + isV4 + ? TimeUnit.SECONDS.convert(unit.toMillis(duration), TimeUnit.MILLISECONDS) + : TimeUnit.SECONDS.convert( + getOptions().getClock().millisTime() + unit.toMillis(duration), + TimeUnit.MILLISECONDS); StringBuilder stPath = new StringBuilder(); if (!blobInfo.getBucket().startsWith(PATH_DELIMITER)) { stPath.append(PATH_DELIMITER); } stPath.append(blobInfo.getBucket()); - if (!blobInfo.getBucket().endsWith(PATH_DELIMITER)) { + if (!blobInfo.getBucket().endsWith(PATH_DELIMITER) + && !Strings.isNullOrEmpty(blobInfo.getName())) { stPath.append(PATH_DELIMITER); } if (blobInfo.getName().startsWith(PATH_DELIMITER)) { @@ -635,14 +645,15 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio } String escapedName = UrlEscapers.urlFragmentEscaper().escape(blobInfo.getName()); - stPath.append(escapedName.replace("?", "%3F")); + stPath.append(escapedName.replace("?", "%3F").replace(";", "%3B")); URI path = URI.create(stPath.toString()); try { - SignatureInfo signatureInfo = buildSignatureInfo(optionMap, blobInfo, expiration, path); - byte[] signatureBytes = - credentials.sign(signatureInfo.constructUnsignedPayload().getBytes(UTF_8)); + SignatureInfo signatureInfo = + buildSignatureInfo(optionMap, blobInfo, expiration, path, credentials.getAccount()); + String unsignedPayload = signatureInfo.constructUnsignedPayload(); + byte[] signatureBytes = credentials.sign(unsignedPayload.getBytes(UTF_8)); StringBuilder stBuilder = new StringBuilder(); if (optionMap.get(SignUrlOption.Option.HOST_NAME) == null) { stBuilder.append(STORAGE_XML_HOST_NAME).append(path); @@ -650,11 +661,20 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio stBuilder.append(optionMap.get(SignUrlOption.Option.HOST_NAME)).append(path); } - String signature = - URLEncoder.encode(BaseEncoding.base64().encode(signatureBytes), UTF_8.name()); - stBuilder.append("?GoogleAccessId=").append(credentials.getAccount()); - stBuilder.append("&Expires=").append(expiration); - stBuilder.append("&Signature=").append(signature); + if (isV4) { + BaseEncoding encoding = BaseEncoding.base16().lowerCase(); + String signature = URLEncoder.encode(encoding.encode(signatureBytes), UTF_8.name()); + stBuilder.append("?"); + stBuilder.append(signatureInfo.constructV4QueryString()); + stBuilder.append("&X-Goog-Signature=").append(signature); + } else { + BaseEncoding encoding = BaseEncoding.base64(); + String signature = URLEncoder.encode(encoding.encode(signatureBytes), UTF_8.name()); + stBuilder.append("?"); + stBuilder.append("GoogleAccessId=").append(credentials.getAccount()); + stBuilder.append("&Expires=").append(expiration); + stBuilder.append("&Signature=").append(signature); + } return new URL(stBuilder.toString()); @@ -670,10 +690,15 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio * @param blobInfo the blob info * @param expiration the expiration in seconds * @param path the resource URI + * @param accountEmail the account email * @return signature info */ private SignatureInfo buildSignatureInfo( - Map optionMap, BlobInfo blobInfo, long expiration, URI path) { + Map optionMap, + BlobInfo blobInfo, + long expiration, + URI path, + String accountEmail) { HttpMethod httpVerb = optionMap.containsKey(SignUrlOption.Option.HTTP_METHOD) @@ -693,6 +718,13 @@ private SignatureInfo buildSignatureInfo( signatureInfoBuilder.setContentType(blobInfo.getContentType()); } + signatureInfoBuilder.setSignatureVersion( + (SignUrlOption.SignatureVersion) optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); + + signatureInfoBuilder.setAccountEmail(accountEmail); + + signatureInfoBuilder.setTimestamp(getOptions().getClock().millisTime()); + @SuppressWarnings("unchecked") Map extHeaders = (Map) diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializerTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializerTest.java index 78374e7c5c8f..5df61aa22e35 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializerTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializerTest.java @@ -26,17 +26,21 @@ public class CanonicalExtensionHeadersSerializerTest { - private CanonicalExtensionHeadersSerializer serializer; + private CanonicalExtensionHeadersSerializer v2Serializer; + private CanonicalExtensionHeadersSerializer v4Serializer; @Before public void setUp() { - serializer = new CanonicalExtensionHeadersSerializer(); + v2Serializer = + new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V2); + v4Serializer = + new CanonicalExtensionHeadersSerializer(Storage.SignUrlOption.SignatureVersion.V4); } @Test public void givenNoHeadersWhenSerializeThenProduceNothing() { - StringBuilder sb = serializer.serialize(Collections.emptyMap()); + StringBuilder sb = v2Serializer.serialize(Collections.emptyMap()); assertEquals(sb.toString(), ""); } @@ -44,7 +48,7 @@ public void givenNoHeadersWhenSerializeThenProduceNothing() { @Test public void givenNullHeadersWhenSerializeThenProduceNothing() { - StringBuilder sb = serializer.serialize(null); + StringBuilder sb = v2Serializer.serialize(null); assertEquals(sb.toString(), ""); } @@ -56,7 +60,7 @@ public void givenEncryptionHeadersWhenSerializeThenAreRemvoed() { encryptionHeaders.put("x-goog-encryption-key", ""); encryptionHeaders.put("x-goog-encryption-key-sha256", ""); - StringBuilder sb = serializer.serialize(encryptionHeaders); + StringBuilder sb = v2Serializer.serialize(encryptionHeaders); assertEquals(sb.toString(), ""); } @@ -70,8 +74,24 @@ public void givenHeadersWhenSerializeThenSuccess() { encryptionHeaders.put("x-goog-encryption-key-sha256", ""); encryptionHeaders.put("X-goog-meta-OWNER", " myself and others \n"); - StringBuilder sb = serializer.serialize(encryptionHeaders); + StringBuilder sb = v2Serializer.serialize(encryptionHeaders); assertEquals(sb.toString(), "x-goog-acl:public-read\nx-goog-meta-owner:myself and others\n"); } + + @Test + public void testV4Serialization() { + Map encryptionHeaders = new HashMap<>(); + encryptionHeaders.put("x-goog-encryption-key", "key"); + encryptionHeaders.put("x-GOOg-acl", " \n public-read "); + encryptionHeaders.put("x-goog-encryption-key-sha256", "sha"); + encryptionHeaders.put("X-goog-meta-OWNER", " myself and others \n"); + + StringBuilder sb = v4Serializer.serialize(encryptionHeaders); + + assertEquals( + "x-goog-acl:public-read\nx-goog-encryption-key:key\nx-goog-encryption-key-sha256:sha" + + "\nx-goog-meta-owner:myself and others\n", + sb.toString()); + } } diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/SignatureInfoTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/SignatureInfoTest.java index 7ad9ed8c421f..7b146a713435 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/SignatureInfoTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/SignatureInfoTest.java @@ -17,6 +17,7 @@ package com.google.cloud.storage; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.google.cloud.storage.SignatureInfo.Builder; import java.net.URI; @@ -67,4 +68,34 @@ public void constructUnsignedPayloadWithExtensionHeaders() { assertEquals(rawPayload, unsignedPayload); } + + @Test + public void constructV4UnsignedPayload() { + Builder builder = new SignatureInfo.Builder(HttpMethod.PUT, 10L, URI.create(RESOURCE)); + + builder.setSignatureVersion(Storage.SignUrlOption.SignatureVersion.V4); + builder.setAccountEmail("me@google.com"); + builder.setTimestamp(1000000000000L); + + String unsignedPayload = builder.build().constructUnsignedPayload(); + + assertTrue( + unsignedPayload.startsWith( + "GOOG4-RSA-SHA256\n" + "20010909T014640Z\n" + "20010909/auto/storage/goog4_request\n")); + } + + @Test + public void constructV4QueryString() { + Builder builder = new SignatureInfo.Builder(HttpMethod.PUT, 10L, URI.create(RESOURCE)); + + builder.setSignatureVersion(Storage.SignUrlOption.SignatureVersion.V4); + builder.setAccountEmail("me@google.com"); + builder.setTimestamp(1000000000000L); + + String queryString = builder.build().constructV4QueryString(); + assertEquals( + "X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=me%40google.com%2F20010909%2F" + + "auto%2Fstorage%2Fgoog4_request&X-Goog-Date=20010909T014640Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host", + queryString); + } } diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 0d77547e7605..9a9db310e99d 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -1977,7 +1977,7 @@ public void testSignUrlForBlobWithSpecialChars() URL url = storage.signUrl(BlobInfo.newBuilder(BUCKET_NAME1, blobName).build(), 14, TimeUnit.DAYS); String escapedBlobName = - UrlEscapers.urlFragmentEscaper().escape(blobName).replace("?", "%3F"); + UrlEscapers.urlFragmentEscaper().escape(blobName).replace("?", "%3F").replace(";", "%3B"); String stringUrl = url.toString(); String expectedUrl = new StringBuilder("https://storage.googleapis.com/") @@ -2036,7 +2036,7 @@ public void testSignUrlForBlobWithSpecialCharsAndHostName() TimeUnit.DAYS, Storage.SignUrlOption.withHostName("https://example.com")); String escapedBlobName = - UrlEscapers.urlFragmentEscaper().escape(blobName).replace("?", "%3F"); + UrlEscapers.urlFragmentEscaper().escape(blobName).replace("?", "%3F").replace(";", "%3B"); String stringUrl = url.toString(); String expectedUrl = new StringBuilder("https://example.com/") diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/V4SigningTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/V4SigningTest.java new file mode 100644 index 000000000000..25f14527983c --- /dev/null +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/V4SigningTest.java @@ -0,0 +1,130 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static org.junit.Assert.assertEquals; + +import com.google.api.core.ApiClock; +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.cloud.storage.testing.RemoteStorageHelper; +import com.google.common.io.CharStreams; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.io.InputStreamReader; +import java.text.SimpleDateFormat; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.Test; + +public class V4SigningTest { + + private static class FakeClock implements ApiClock { + private final AtomicLong currentNanoTime; + + public FakeClock(long initialNanoTime) { + this.currentNanoTime = new AtomicLong(initialNanoTime); + } + + public long nanoTime() { + return this.currentNanoTime.get(); + } + + public long millisTime() { + return TimeUnit.MILLISECONDS.convert(this.nanoTime(), TimeUnit.NANOSECONDS); + } + } + + public class TestCase { + String description; + String bucket; + String object; + String method; + String expiration; + String timestamp; + String expectedUrl; + JsonObject headers; + + public String toString() { + return description; + } + } + + @Test + public void testV4UrlSigning() throws Exception { + Storage storage = + RemoteStorageHelper.create() + .getOptions() + .toBuilder() + .setCredentials( + ServiceAccountCredentials.fromStream( + getClass().getResourceAsStream("/UrlSignerV4TestAccount.json"))) + .build() + .getService(); + Gson gson = new GsonBuilder().create(); + + String testCaseJson = + CharStreams.toString( + new InputStreamReader(getClass().getResourceAsStream("/UrlSignerV4TestData.json"))); + + JsonArray testCases = gson.fromJson(testCaseJson, JsonArray.class); + + SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd'T'hhmmss'Z'"); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + + for (JsonElement testCaseElement : testCases) { + TestCase testCase = gson.fromJson(testCaseElement, TestCase.class); + + storage = + storage + .getOptions() + .toBuilder() + .setClock( + new FakeClock( + TimeUnit.NANOSECONDS.convert( + format.parse(testCase.timestamp).getTime(), TimeUnit.MILLISECONDS))) + .build() + .getService(); + + BlobInfo blob = BlobInfo.newBuilder(testCase.bucket, testCase.object).build(); + + Map headers = new HashMap<>(); + if (testCase.headers != null) { + for (Map.Entry entry : testCase.headers.entrySet()) { + headers.put(entry.getKey(), entry.getValue().getAsString()); + } + } + + assertEquals( + testCase.expectedUrl, + storage + .signUrl( + blob, + Long.valueOf(testCase.expiration), + TimeUnit.SECONDS, + Storage.SignUrlOption.httpMethod(HttpMethod.valueOf(testCase.method)), + Storage.SignUrlOption.withExtHeaders(headers), + Storage.SignUrlOption.withV4Signature()) + .toString()); + } + } +} diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 0e9e33f177bf..3f647a68c390 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -162,6 +162,7 @@ public class ITStorageTest { public static void beforeClass() throws IOException { remoteStorageHelper = RemoteStorageHelper.create(); storage = remoteStorageHelper.getOptions().getService(); + storage.create( BucketInfo.newBuilder(BUCKET) .setLocation("us") @@ -1843,6 +1844,25 @@ public void testPostSignedUrl() throws IOException { assertEquals(blob.getName(), remoteBlob.getName()); } + @Test + public void testV4SignedUrl() throws IOException { + if (storage.getOptions().getCredentials() != null) { + assumeTrue(storage.getOptions().getCredentials() instanceof ServiceAccountSigner); + } + + String blobName = "test-get-signed-url-blob/with/slashes/and?special=!#$&'()*+,:;=?@[]"; + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); + Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); + assertNotNull(remoteBlob); + URL url = storage.signUrl(blob, 1, TimeUnit.HOURS, Storage.SignUrlOption.withV4Signature()); + URLConnection connection = url.openConnection(); + byte[] readBytes = new byte[BLOB_BYTE_CONTENT.length]; + try (InputStream responseStream = connection.getInputStream()) { + assertEquals(BLOB_BYTE_CONTENT.length, responseStream.read(readBytes)); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } + } + @Test public void testGetBlobs() { String sourceBlobName1 = "test-get-blobs-1"; diff --git a/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json new file mode 100644 index 000000000000..fe2701badc3d --- /dev/null +++ b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestAccount.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "dummy-project-id", + "private_key_id": "ffffffffffffffffffffffffffffffffffffffff", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsPzMirIottfQ2\nryjQmPWocSEeGo7f7Q4/tMQXHlXFzo93AGgU2t+clEj9L5loNhLVq+vk+qmnyDz5\nQ04y8jVWyMYzzGNNrGRW/yaYqnqlKZCy1O3bmnNjV7EDbC/jE1ZLBY0U3HaSHfn6\nS9ND8MXdgD0/ulRTWwq6vU8/w6i5tYsU7n2LLlQTl1fQ7/emO9nYcCFJezHZVa0H\nmeWsdHwWsok0skwQYQNIzP3JF9BpR5gJT2gNge6KopDesJeLoLzaX7cUnDn+CAnn\nLuLDwwSsIVKyVxhBFsFXPplgpaQRwmGzwEbf/Xpt9qo26w2UMgn30jsOaKlSeAX8\ncS6ViF+tAgMBAAECggEACKRuJCP8leEOhQziUx8Nmls8wmYqO4WJJLyk5xUMUC22\nSI4CauN1e0V8aQmxnIc0CDkFT7qc9xBmsMoF+yvobbeKrFApvlyzNyM7tEa/exh8\nDGD/IzjbZ8VfWhDcUTwn5QE9DCoon9m1sG+MBNlokB3OVOt8LieAAREdEBG43kJu\nyQTOkY9BGR2AY1FnAl2VZ/jhNDyrme3tp1sW1BJrawzR7Ujo8DzlVcS2geKA9at7\n55ua5GbHz3hfzFgjVXDfnkWzId6aHypUyqHrSn1SqGEbyXTaleKTc6Pgv0PgkJjG\nhZazWWdSuf1T5Xbs0OhAK9qraoAzT6cXXvMEvvPt6QKBgQDXcZKqJAOnGEU4b9+v\nOdoh+nssdrIOBNMu1m8mYbUVYS1aakc1iDGIIWNM3qAwbG+yNEIi2xi80a2RMw2T\n9RyCNB7yqCXXVKLBiwg9FbKMai6Vpk2bWIrzahM9on7AhCax/X2AeOp+UyYhFEy6\nUFG4aHb8THscL7b515ukSuKb5QKBgQDMq+9PuaB0eHsrmL6q4vHNi3MLgijGg/zu\nAXaPygSYAwYW8KglcuLZPvWrL6OG0+CrfmaWTLsyIZO4Uhdj7MLvX6yK7IMnagvk\nL3xjgxSklEHJAwi5wFeJ8ai/1MIuCn8p2re3CbwISKpvf7Sgs/W4196P4vKvTiAz\njcTiSYFIKQKBgCjMpkS4O0TakMlGTmsFnqyOneLmu4NyIHgfPb9cA4n/9DHKLKAT\noaWxBPgatOVWs7RgtyGYsk+XubHkpC6f3X0+15mGhFwJ+CSE6tN+l2iF9zp52vqP\nQwkjzm7+pdhZbmaIpcq9m1K+9lqPWJRz/3XXuqi+5xWIZ7NaxGvRjqaNAoGAdK2b\nutZ2y48XoI3uPFsuP+A8kJX+CtWZrlE1NtmS7tnicdd19AtfmTuUL6fz0FwfW4Su\nlQZfPT/5B339CaEiq/Xd1kDor+J7rvUHM2+5p+1A54gMRGCLRv92FQ4EON0RC1o9\nm2I4SHysdO3XmjmdXmfp4BsgAKJIJzutvtbqlakCgYB+Cb10z37NJJ+WgjDt+yT2\nyUNH17EAYgWXryfRgTyi2POHuJitd64Xzuy6oBVs3wVveYFM6PIKXlj8/DahYX5I\nR2WIzoCNLL3bEZ+nC6Jofpb4kspoAeRporj29SgesK6QBYWHWX2H645RkRGYGpDo\n51gjy9m/hSNqBbH2zmh04A==\n-----END PRIVATE KEY-----\n", + "client_email": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com", + "client_id": "000000000000000000000", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestData.json b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestData.json new file mode 100644 index 000000000000..ae5a9cc6b0e5 --- /dev/null +++ b/google-cloud-clients/google-cloud-storage/src/test/resources/UrlSignerV4TestData.json @@ -0,0 +1,122 @@ +[ + { + "description": "Simple GET", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74" + }, + + { + "description": "Simple PUT", + "bucket": "test-bucket", + "object": "test-object", + "method": "PUT", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=8adff1d4285739e31aa68e73767a46bc5511fde377497dbe08481bf5ceb34e29cc9a59921748d8ec3dd4085b7e9b7772a952afedfcdaecb3ae8352275b8b7c867f204e3db85076220a3127a8a9589302fc1181eae13b9b7fe41109ec8cdc93c1e8bac2d7a0cc32a109ca02d06957211326563ab3d3e678a0ba296e298b5fc5e14593c99d444c94724cc4be97015dbff1dca377b508fa0cb7169195de98d0e4ac96c42b918d28c8d92d33e1bd125ce0fb3cd7ad2c45dae65c22628378f6584971b8bf3945b26f2611eb651e9b6a8648970c1ecf386bb71327b082e7296c4e1ee2fc0bdd8983da80af375c817fb1ad491d0bc22c0f51dba0d66e2cffbc90803e47" + }, + + { + "description": "POST for resumable uploads", + "bucket": "test-bucket", + "object": "test-object", + "method": "POST", + "expiration": 10, + "headers": { + "x-goog-resumable": "start" + }, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-resumable&X-Goog-Signature=4a6d39b23343cedf4c30782aed4b384001828c79ffa3a080a481ea01a640dea0a0ceb58d67a12cef3b243c3f036bb3799c6ee88e8db3eaf7d0bdd4b70a228d0736e07eaa1ee076aff5c6ce09dff1f1f03a0d8ead0d2893408dd3604fdabff553aa6d7af2da67cdba6790006a70240f96717b98f1a6ccb24f00940749599be7ef72aaa5358db63ddd54b2de9e2d6d6a586eac4fe25f36d86fc6ab150418e9c6fa01b732cded226c6d62fc95b72473a4cc55a8257482583fe66d9ab6ede909eb41516a8690946c3e87b0f2052eb0e97e012a14b2f721c42e6e19b8a1cd5658ea36264f10b9b1ada66b8ed5bf7ed7d1708377ac6e5fe608ae361fb594d2e5b24c54" + }, + + { + "description": "Vary expiration and timestamp", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 20, + "timestamp": "20190301T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190301%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190301T090000Z&X-Goog-Expires=20&X-Goog-SignedHeaders=host&X-Goog-Signature=9669ed5b10664dc594c758296580662912cf4bcc5a4ba0b6bf055bcbf6f34eed7bdad664f534962174a924741a0c273a4f67bc1847cef20192a6beab44223bd9d4fbbd749c407b79997598c30f82ddc269ff47ec09fa3afe74e00616d438df0d96a7d8ad0adacfad1dc3286f864d924fe919fb0dce45d3d975c5afe8e13af2db9cc37ba77835f92f7669b61e94c6d562196c1274529e76cfff1564cc2cad7d5387dc8e12f7a5dfd925685fe92c30b43709eee29fa2f66067472cee5423d1a3a4182fe8cea75c9329d181dc6acad7c393cd04f8bf5bc0515127d8ebd65d80c08e19ad03316053ea60033fd1b1fd85a69c576415da3bf0a3718d9ea6d03e0d66f0" + }, + + { + "description": "Vary bucket and object", + "bucket": "test-bucket2", + "object": "test-object2", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket2/test-object2?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=36e3d58dfd3ec1d2dd2f24b5ee372a71e811ffaa2162a2b871d26728d0354270bc116face87127532969c4a3967ed05b7309af741e19c7202f3167aa8c2ac420b61417d6451442bb91d7c822cd17be8783f01e05372769c88913561d27e6660dd8259f0081a71f831be6c50283626cbf04494ac10c394b29bb3bce74ab91548f58a37118a452693cf0483d77561fc9cac8f1765d2c724994cca46a83517a10157ee0347a233a2aaeae6e6ab5e204ff8fc5f54f90a3efdb8301d9fff5475d58cd05b181affd657f48203f4fb133c3a3d355b8eefbd10d5a0a5fd70d06e9515460ad74e22334b2cba4b29cae4f6f285cdb92d8f3126d7a1479ca3bdb69c207d860" + }, + + { + "description": "Simple headers", + "bucket": "test-bucket", + "object": "test-object", + "headers": { + "foo": "foo-value", + "BAR": "BAR-value" + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost&X-Goog-Signature=68ecd3b008328ed30d91e2fe37444ed7b9b03f28ed4424555b5161980531ef87db1c3a5bc0265aad5640af30f96014c94fb2dba7479c41bfe1c020eb90c0c6d387d4dd09d4a5df8b60ea50eb6b01cdd786a1e37020f5f95eb8f9b6cd3f65a1f8a8a65c9fcb61ea662959efd9cd73b683f8d8804ef4d6d9b2852419b013368842731359d7f9e6d1139032ceca75d5e67cee5fd0192ea2125e5f2955d38d3d50cf116f3a52e6a62de77f6207f5b95aaa1d7d0f8a46de89ea72e7ea30f21286318d7eba0142232b0deb3a1dc9e1e812a981c66b5ffda3c6b01a8a9d113155792309fd53a3acfd054ca7776e8eec28c26480cd1e3c812f67f91d14217f39a606669d" + }, + + { + "description": "Headers should be trimmed", + "bucket": "test-bucket", + "object": "test-object", + "headers": { + "leading": " xyz", + "trailing": "abc ", + "collapsed": "abc def" + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=collapsed%3Bhost%3Bleading%3Btrailing&X-Goog-Signature=1839511d6238d9ac2bbcbba8b23515b3757db35dfa7b8f9bc4b8b4aa270224df747c812526f1a3bcf294d67ed84cd14e074c36bc090e0a542782934a7c925af4a5ea68123e97533704ce8b08ccdf5fe6b412f89c9fc4de243e29abdb098382c5672188ee3f6fef7131413e252c78e7a35658825ad842a50609e9cc463731e17284ff7a14824c989f87cef22fb99dfec20cfeed69d8b3a08f00b43b8284eecd535e50e982b05cd74c5750cd5f986cfc21a2a05f7f3ab7fc31bd684ed1b823b64d29281e923fc6580c49005552ca19c253de087d9d2df881144e44eda40965cfdb4889bf3a35553c9809f4ed20b8355be481b92b9618952b6a04f3017b36053e15" + }, + + { + "description": "Header value with multiple inline values", + "bucket": "test-bucket", + "object": "test-object", + "headers": { + "multiple": " xyz , abc, def , xyz " + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bmultiple&X-Goog-Signature=5cc113735625341f59c7203f0c2c9febc95ba6af6b9c38814f8e523214712087dc0996e4960d273ae1889f248ac1e58d4d19cb3a69ad7670e9a8ca1b434e878f59339dc7006cf32dfd715337e9f593e0504371839174962a08294586e0c78160a7aa303397888c8350637c6af3b32ac310886cc4590bfda9ca561ee58fb5b8ec56bc606d2ada6e7df31f4276e9dcb96bcaea39dc2cd096f3fad774f9c4b30e317ad43736c05f76831437f44e8726c1e90d3f6c9827dc273f211f32fc85658dfc5d357eb606743a6b00a29e519eef1bebaf9db3e8f4b1f5f9afb648ad06e60bc42fa8b57025056697c874c9ea76f5a73201c9717ea43e54713ff3502ff3fc626b" + }, + + { + "description": "Customer-supplied encryption key", + "bucket": "test-bucket", + "object": "test-object", + "headers": + { + "X-Goog-Encryption-Key": "key", + "X-Goog-Encryption-Key-Sha256": "key-hash", + "X-Goog-Encryption-Algorithm": "AES256" + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-encryption-algorithm%3Bx-goog-encryption-key%3Bx-goog-encryption-key-sha256&X-Goog-Signature=278a1c5a3bad248637054a047014760353942433955871031ed08f515b54588654ad033e91f046ab202b68673030e117d1b786c325e870238b035ba75b3feed560a17aff9bab6bddebd4a31a52cb68b214e27d3b0bd886502c6b36b164306fe88b5a07c6063592afe746b2a5d205dbe90dd5386b94f0a78f75d9f53ee884e18f476e8fc2eb1dd910ce0b4ae1f5d7b09876ef9bf983f539c028429e14bad3c75dbd4ed1ae37856f6d6f8a1805eaf8b52a0d6fc993902e4c1ee8de477661f7b67c3663000474cb00e178189789b2a3ed6bd21b4ade684fca8108ac4dd106acb17f5954d045775f7aa5a98ebda5d3075e11a8ea49c64c6ad1481e463e8c9f11f704" + }, + + { + "description": "List Objects", + "bucket": "test-bucket", + "object": "", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=6dbe94f8e52b2b8a9a476b1c857efa474e09944e2b52b925800316e094a7169d8dbe0df9c0ac08dabb22ac7e827470ceccd65f5a3eadba2a4fb9beebfe37f0d9bb1e552b851fa31a25045bdf019e507f5feb44f061551ef1aeb18dcec0e38ba2e2f77d560a46eaace9c56ed9aa642281301a9d848b0eb30749e34bc7f73a3d596240533466ff9b5f289cd0d4c845c7d96b82a35a5abd0c3aff83e4440ee6873e796087f43545544dc8c01afe1d79c726696b6f555371e491980e7ec145cca0803cf562c38f3fa1d724242f5dea25aac91d74ec9ddd739ff65523627763eaef25cd1f95ad985aaf0079b7c74eb5bcb2870a9b137a7b2c8e41fbe838c95872f75b" + } +] \ No newline at end of file diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java index 015c2d785f89..48b8ce6aebc8 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/storage/snippets/StorageSnippets.java @@ -39,6 +39,7 @@ import com.google.cloud.storage.Bucket; import com.google.cloud.storage.BucketInfo; import com.google.cloud.storage.CopyWriter; +import com.google.cloud.storage.HttpMethod; import com.google.cloud.storage.Storage; import com.google.cloud.storage.Storage.BlobGetOption; import com.google.cloud.storage.Storage.BlobListOption; @@ -1474,4 +1475,72 @@ public Bucket getBucketPolicyOnly(String bucketName) throws StorageException { // [END storage_get_bucket_policy_only] return bucket; } + + /** Example of how to generate a GET V4 Signed URL */ + public URL generateV4GetObjectSignedUrl(String bucketName, String objectName) + throws StorageException { + // [START storage_generate_signed_url_v4] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The name of an object, e.g. "my-object" + // String objectName = "my-object"; + + // Define resource + BlobInfo blobinfo = BlobInfo.newBuilder(BlobId.of(bucketName, objectName)).build(); + + // Generate Signed URL + URL url = + storage.signUrl(blobinfo, 15, TimeUnit.MINUTES, Storage.SignUrlOption.withV4Signature()); + + System.out.println("Generated GET signed URL:"); + System.out.println(url); + System.out.println("You can use this URL with any user agent, for example:"); + System.out.println("curl '" + url + "'"); + // [END storage_generate_signed_url_v4] + return url; + } + + /** Example of how to generate a PUT V4 Signed URL */ + public URL generateV4GPutbjectSignedUrl(String bucketName, String objectName) + throws StorageException { + // [START storage_generate_upload_signed_url_v4] + // Instantiate a Google Cloud Storage client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name of a bucket, e.g. "my-bucket" + // String bucketName = "my-bucket"; + + // The name of a new object to upload, e.g. "my-object" + // String objectName = "my-object"; + + // Define Resource + BlobInfo blobinfo = BlobInfo.newBuilder(BlobId.of(bucketName, objectName)).build(); + + // Generate Signed URL + Map extensionHeaders = new HashMap<>(); + extensionHeaders.put("Content-Type", "application/octet-stream"); + + URL url = + storage.signUrl( + blobinfo, + 15, + TimeUnit.MINUTES, + Storage.SignUrlOption.httpMethod(HttpMethod.PUT), + Storage.SignUrlOption.withExtHeaders(extensionHeaders), + Storage.SignUrlOption.withV4Signature()); + + System.out.println("Generated PUT signed URL:"); + System.out.println(url); + System.out.println("You can use this URL with any user agent, for example:"); + System.out.println( + "curl -X PUT -H 'Content-Type: application/octet-stream'--upload-file my-file '" + + url + + "'"); + // [END storage_generate_upload_signed_url_v4] + return url; + } } diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java index 3d44c94ad34a..8fadb384def6 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/storage/snippets/ITStorageSnippets.java @@ -44,6 +44,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintStream; import java.net.URL; import java.net.URLConnection; @@ -58,6 +59,7 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import javax.net.ssl.HttpsURLConnection; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; @@ -581,4 +583,29 @@ public void testBucketPolicyOnly() { bucket = storageSnippets.disableBucketPolicyOnly(tempBucket); assertFalse(bucket.getIamConfiguration().isBucketPolicyOnlyEnabled()); } + + @Test + public void testV4SignedURLs() throws IOException { + String tempBucket = RemoteStorageHelper.generateBucketName(); + Bucket bucket = storageSnippets.createBucket(tempBucket); + assertNotNull(bucket); + String tempObject = "test-upload-signed-url-object"; + URL uploadUrl = storageSnippets.generateV4GPutbjectSignedUrl(tempBucket, tempObject); + HttpsURLConnection connection = (HttpsURLConnection) uploadUrl.openConnection(); + connection.setRequestMethod("PUT"); + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", "application/octet-stream"); + byte[] write = new byte[BLOB_BYTE_CONTENT.length]; + try (OutputStream out = connection.getOutputStream()) { + out.write(BLOB_BYTE_CONTENT); + assertEquals(connection.getResponseCode(), 200); + } + URL downloadUrl = storageSnippets.generateV4GetObjectSignedUrl(tempBucket, tempObject); + connection = (HttpsURLConnection) downloadUrl.openConnection(); + byte[] readBytes = new byte[BLOB_BYTE_CONTENT.length]; + try (InputStream responseStream = connection.getInputStream()) { + assertEquals(BLOB_BYTE_CONTENT.length, responseStream.read(readBytes)); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } + } }