From 6af9ce25060470bb45ab2cf6d9cf96ba47821370 Mon Sep 17 00:00:00 2001 From: dmitry-fa Date: Fri, 17 Apr 2020 16:45:56 +0300 Subject: [PATCH 1/9] feat: add storage.upload(path) --- .../clirr-ignored-differences.xml | 14 ++ .../com/google/cloud/storage/Storage.java | 109 ++++++++++ .../com/google/cloud/storage/StorageImpl.java | 58 ++++++ .../google/cloud/storage/StorageImplTest.java | 187 +++++++++++++++++- .../cloud/storage/it/ITStorageTest.java | 41 ++++ 5 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 google-cloud-storage/clirr-ignored-differences.xml diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml new file mode 100644 index 0000000000..fea926f574 --- /dev/null +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -0,0 +1,14 @@ + + + + + 7012 + com/google/cloud/storage/Storage + void upload(*.BlobInfo, java.nio.file.Path, *.Storage$BlobWriteOption[]) + + + 7012 + com/google/cloud/storage/Storage + void upload(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[]) + + diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index a9bb589a2b..7272bb693f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -37,9 +37,11 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.io.BaseEncoding; +import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.net.URL; +import java.nio.file.Path; import java.security.Key; import java.util.Arrays; import java.util.Collections; @@ -1811,6 +1813,113 @@ Blob create( @Deprecated Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options); + /** + * Uploads {@code path} to the blob using {@link #writer}. By default any MD5 and CRC32C values in + * the given {@code blobInfo} are ignored unless requested via the {@link + * BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. Folder upload is + * not supported. + * + *

Example of uploading a file: + * + * 

{@code
+   * String bucketName = "my-unique-bucket";
+   * String fileName = "readme.txt";
+   * BlobId blobId = BlobId.of(bucketName, fileName);
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
+   * storage.upload(blobInfo, Paths.get(fileName));
+   * }
+ * + * @param blobInfo blob to create + * @param path file to upload + * @param options blob write options + * @throws IOException on I/O error + * @throws StorageException on failure + * @see #upload(BlobInfo, Path, int, BlobWriteOption...) + */ + void upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException; + + /** + * Uploads {@code path} to the blob using {@link #writer} and {@code bufferSize}. By default any + * MD5 and CRC32C values in the given {@code blobInfo} are ignored unless requested via the {@link + * BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. Folder upload is + * not supported. + * + *

{@link #upload(BlobInfo, Path, BlobWriteOption...)} invokes this method with a buffer size + * of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the upload + * performance but require more memory. This can cause an OutOfMemoryError or add significant + * garbage collection overhead. Smaller buffer sizes reduce memory consumption, that is noticeable + * when uploading many objects in parallel. Buffer sizes less than 256 KiB are treated as 256 KiB. + * + *

Example of uploading a humongous file: + * + * 

{@code
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("video/webm").build();
+   *
+   * int largeBufferSize = 150 * 1024 * 1024;
+   * Path file = Paths.get("humongous.file");
+   * storage.upload(blobInfo, file, largeBufferSize);
+   * }
+ * + * @param blobInfo blob to create + * @param path file to upload + * @param bufferSize size of the buffer I/O operations + * @param options blob write options + * @throws IOException on I/O error + * @throws StorageException on failure + */ + void upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) + throws IOException; + + /** + * Reads bytes from an input stream and uploads those bytes to the blob using {@link #writer}. By + * default any MD5 and CRC32C values in the given {@code blobInfo} are ignored unless requested + * via the {@link BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. + * + *

Example of uploading data with CRC32C checksum: + * + * 

{@code
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * byte[] content = "Hello, world".getBytes(StandardCharsets.UTF_8);
+   * Hasher hasher = Hashing.crc32c().newHasher().putBytes(content);
+   * String crc32c = BaseEncoding.base64().encode(Ints.toByteArray(hasher.hash().asInt()));
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(crc32c).build();
+   * storage.upload(blobInfo, new ByteArrayInputStream(content), Storage.BlobWriteOption.crc32cMatch());
+   * }
+ * + * @param blobInfo blob to create + * @param content input stream to read from + * @param options blob write options + * @throws IOException on I/O error + * @throws StorageException on failure + * @see #upload(BlobInfo, InputStream, int, BlobWriteOption...) + */ + void upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) + throws IOException; + + /** + * Reads bytes from an input stream and uploads those bytes to the blob using {@link #writer} and + * {@code bufferSize}. By default any MD5 and CRC32C values in the given {@code blobInfo} are + * ignored unless requested via the {@link BlobWriteOption#md5Match()} and {@link + * BlobWriteOption#crc32cMatch()} options. + * + *

{@link #upload(BlobInfo, InputStream, BlobWriteOption...)} )} invokes this method with a + * buffer size of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the + * upload performance but require more memory. This can cause an OutOfMemoryError or add + * significant garbage collection overhead. Smaller buffer sizes reduce memory consumption, that + * is noticeable when uploading many objects in parallel. Buffer sizes less than 256 KiB are + * treated as 256 KiB. + * + * @param blobInfo blob to create + * @param content input stream to read from + * @param bufferSize size of the buffer I/O operations + * @param options blob write options + * @throws IOException on I/O error + * @throws StorageException on failure + */ + void upload(BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) + throws IOException; + /** * Returns the requested bucket or {@code null} if not found. * diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 0f38e38272..54e181d9e0 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -48,6 +48,7 @@ import com.google.cloud.ReadChannel; import com.google.cloud.RetryHelper.RetryHelperException; import com.google.cloud.Tuple; +import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Entity; import com.google.cloud.storage.HmacKey.HmacKeyMetadata; import com.google.cloud.storage.spi.v1.StorageRpc; @@ -66,12 +67,18 @@ import com.google.common.io.BaseEncoding; import com.google.common.primitives.Ints; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; @@ -92,6 +99,9 @@ final class StorageImpl extends BaseService implements Storage { private static final String STORAGE_XML_URI_HOST_NAME = "storage.googleapis.com"; + private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024; + private static final int MIN_BUFFER_SIZE = 256 * 1024; + private static final Function, Boolean> DELETE_FUNCTION = new Function, Boolean>() { @Override @@ -204,6 +214,54 @@ public StorageObject call() { } } + @Override + public void upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException { + upload(blobInfo, path, DEFAULT_BUFFER_SIZE, options); + } + + @Override + public void upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) + throws IOException { + if (Files.isDirectory(path)) { + throw new StorageException(0, path + " is a directory"); + } + try (InputStream input = Files.newInputStream(path)) { + upload(blobInfo, input, bufferSize, options); + } + } + + @Override + public void upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) + throws IOException { + upload(blobInfo, content, DEFAULT_BUFFER_SIZE, options); + } + + @Override + public void upload( + BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) + throws IOException { + try (WriteChannel writer = writer(blobInfo, options)) { + upload(Channels.newChannel(content), writer, bufferSize); + } + } + + /* + * Uploads the given content to the storage using specified write channel and the given buffer + * size. This method does not close any channels. + */ + private static void upload(ReadableByteChannel reader, WriteChannel writer, int bufferSize) + throws IOException { + bufferSize = Math.max(bufferSize, MIN_BUFFER_SIZE); + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + writer.setChunkSize(bufferSize); + + while (reader.read(buffer) >= 0) { + buffer.flip(); + writer.write(buffer); + buffer.clear(); + } + } + @Override public Bucket get(String bucket, BucketGetOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb(); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index e1dd06e4f8..473ce06f1d 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -59,11 +59,16 @@ import com.google.common.io.BaseEncoding; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.InvalidKeyException; import java.security.Key; import java.security.KeyFactory; @@ -113,7 +118,8 @@ public class StorageImplTest { "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key"; private static final Long RETENTION_PERIOD = 10L; private static final String USER_PROJECT = "test-project"; - + private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024; + private static final int MIN_BUFFER_SIZE = 256 * 1024; // BucketInfo objects private static final BucketInfo BUCKET_INFO1 = BucketInfo.newBuilder(BUCKET_NAME1).setMetageneration(42L).build(); @@ -3072,4 +3078,183 @@ public void testWriterWithSignedURL() throws MalformedURLException { assertNotNull(writer); assertTrue(writer.isOpen()); } + + @Test + public void testUploadNonExistentFile() { + EasyMock.replay(storageRpcMock); + initializeService(); + String fileName = "non_existing_file.txt"; + try { + storage.upload(BLOB_INFO1, Paths.get(fileName)); + Assert.fail(); + } catch (IOException e) { + assertEquals(NoSuchFileException.class, e.getClass()); + assertEquals(fileName, e.getMessage()); + } + } + + @Test + public void testUploadDirectory() throws IOException { + EasyMock.replay(storageRpcMock); + initializeService(); + Path dir = Files.createTempDirectory("unit_"); + try { + storage.upload(BLOB_INFO1, dir); + Assert.fail(); + } catch (StorageException e) { + assertEquals(dir + " is a directory", e.getMessage()); + } + } + + private BlobInfo initializeUpload(byte[] bytes) { + return initializeUpload(bytes, DEFAULT_BUFFER_SIZE, EMPTY_RPC_OPTIONS); + } + + private BlobInfo initializeUpload(byte[] bytes, int bufferSize) { + return initializeUpload(bytes, bufferSize, EMPTY_RPC_OPTIONS); + } + + private BlobInfo initializeUpload( + byte[] bytes, int bufferSize, Map rpcOptions) { + String uploadId = "upload-id"; + byte[] buffer = new byte[bufferSize]; + System.arraycopy(bytes, 0, buffer, 0, bytes.length); + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + EasyMock.expect(storageRpcMock.open(info.toPb(), rpcOptions)).andReturn(uploadId); + storageRpcMock.write( + EasyMock.eq(uploadId), + EasyMock.aryEq(buffer), + EasyMock.eq(0), + EasyMock.eq(0L), + EasyMock.eq(bytes.length), + EasyMock.eq(true)); + EasyMock.replay(storageRpcMock); + initializeService(); + return info; + } + + @Test + public void testUploadFile() throws Exception { + byte[] dataToSend = {1, 2, 3, 4}; + Path tempFile = Files.createTempFile("testUpload", ".tmp"); + Files.write(tempFile, dataToSend); + + BlobInfo info = initializeUpload(dataToSend); + storage.upload(info, tempFile); + } + + @Test + public void testUploadStream() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + + BlobInfo info = initializeUpload(dataToSend); + storage.upload(info, stream); + } + + @Test + public void testUploadWithOptions() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + + BlobInfo info = initializeUpload(dataToSend, DEFAULT_BUFFER_SIZE, KMS_KEY_NAME_OPTIONS); + storage.upload(info, stream, BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + } + + @Test + public void testUploadWithBufferSize() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int bufferSize = MIN_BUFFER_SIZE * 2; + + BlobInfo info = initializeUpload(dataToSend, bufferSize); + storage.upload(info, stream, bufferSize); + } + + @Test + public void testUploadWithBufferSizeAndOptions() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int bufferSize = MIN_BUFFER_SIZE * 2; + + BlobInfo info = initializeUpload(dataToSend, bufferSize, KMS_KEY_NAME_OPTIONS); + storage.upload(info, stream, bufferSize, BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + } + + @Test + public void testUploadWithSmallBufferSize() throws Exception { + byte[] dataToSend = new byte[100_000]; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int smallBufferSize = 100; + + BlobInfo info = initializeUpload(dataToSend, MIN_BUFFER_SIZE); + storage.upload(info, stream, smallBufferSize); + } + + @Test + public void testUploadWithException() throws Exception { + initializeService(); + String uploadId = "id-exception"; + byte[] bytes = new byte[10]; + byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; + System.arraycopy(bytes, 0, buffer, 0, bytes.length); + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + EasyMock.expect(storageRpcMock.open(info.toPb(), EMPTY_RPC_OPTIONS)).andReturn(uploadId); + Exception runtimeException = new RuntimeException("message"); + storageRpcMock.write( + EasyMock.eq(uploadId), + EasyMock.aryEq(buffer), + EasyMock.eq(0), + EasyMock.eq(0L), + EasyMock.eq(bytes.length), + EasyMock.eq(true)); + EasyMock.expectLastCall().andThrow(runtimeException); + EasyMock.replay(storageRpcMock); + + InputStream input = new ByteArrayInputStream(bytes); + try { + storage.upload(info, input); + Assert.fail(); + } catch (StorageException e) { + assertSame(runtimeException, e.getCause()); + } + } + + @Test + public void testUploadMultipleParts() throws Exception { + initializeService(); + String uploadId = "id-multiple-parts"; + int extraBytes = 10; + int totalSize = MIN_BUFFER_SIZE + extraBytes; + byte[] dataToSend = new byte[totalSize]; + dataToSend[0] = 42; + dataToSend[MIN_BUFFER_SIZE + 1] = 43; + + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + EasyMock.expect(storageRpcMock.open(info.toPb(), EMPTY_RPC_OPTIONS)).andReturn(uploadId); + byte[] buffer1 = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(dataToSend, 0, buffer1, 0, MIN_BUFFER_SIZE); + storageRpcMock.write( + EasyMock.eq(uploadId), + EasyMock.aryEq(buffer1), + EasyMock.eq(0), + EasyMock.eq(0L), + EasyMock.eq(MIN_BUFFER_SIZE), + EasyMock.eq(false)); + + byte[] buffer2 = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(dataToSend, MIN_BUFFER_SIZE, buffer2, 0, extraBytes); + storageRpcMock.write( + EasyMock.eq(uploadId), + EasyMock.aryEq(buffer2), + EasyMock.eq(0), + EasyMock.eq((long) MIN_BUFFER_SIZE), + EasyMock.eq(extraBytes), + EasyMock.eq(true)); + + EasyMock.replay(storageRpcMock); + + InputStream input = new ByteArrayInputStream(dataToSend); + storage.upload(info, input, MIN_BUFFER_SIZE); + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 52a6acd7e4..47f404f2c4 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -103,6 +103,8 @@ import java.net.URL; import java.net.URLConnection; import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.Key; import java.util.ArrayList; import java.util.Arrays; @@ -3208,4 +3210,43 @@ public void testBucketLogging() throws ExecutionException, InterruptedException RemoteStorageHelper.forceDelete(storage, loggingBucket, 5, TimeUnit.SECONDS); } } + + @Test + public void testUploadFromDownloadTo() throws Exception { + String blobName = "test-uploadFrom-downloadTo-blob"; + BlobId blobId = BlobId.of(BUCKET, blobName); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); + + Path tempFileFrom = Files.createTempFile("ITStorageTest_", ".tmp"); + Files.write(tempFileFrom, BLOB_BYTE_CONTENT); + storage.upload(blobInfo, tempFileFrom); + + Path tempFileTo = Files.createTempFile("ITStorageTest_", ".tmp"); + storage.get(blobId).downloadTo(tempFileTo); + byte[] readBytes = Files.readAllBytes(tempFileTo); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } + + @Test + public void testUploadWithEncryption() throws Exception { + String blobName = "test-upload-withEncryption"; + BlobId blobId = BlobId.of(BUCKET, blobName); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); + + ByteArrayInputStream content = new ByteArrayInputStream(BLOB_BYTE_CONTENT); + storage.upload(blobInfo, content, Storage.BlobWriteOption.encryptionKey(KEY)); + + Blob blob = storage.get(blobId); + try { + blob.getContent(); + fail("StorageException was expected"); + } catch (StorageException e) { + String expectedMessage = + "The target object is encrypted by a customer-supplied encryption key."; + assertTrue(e.getMessage().contains(expectedMessage)); + assertEquals(400, e.getCode()); + } + byte[] readBytes = blob.getContent(Blob.BlobSourceOption.decryptionKey(KEY)); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } } From 085739f3fc426c57108b3605538b901fd5526dc9 Mon Sep 17 00:00:00 2001 From: dmitry-fa Date: Tue, 21 Apr 2020 17:14:52 +0300 Subject: [PATCH 2/9] feat: use mockito framework --- google-cloud-storage/pom.xml | 5 + .../com/google/cloud/storage/StorageImpl.java | 4 +- .../cloud/storage/StorageImplMockitoTest.java | 583 ++++++++++++++++++ .../google/cloud/storage/StorageImplTest.java | 184 ------ pom.xml | 7 + 5 files changed, 597 insertions(+), 186 deletions(-) create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 94cbf8a789..6f5d790e2e 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -156,6 +156,11 @@ easymock test + + org.mockito + mockito-core + test + org.objenesis objenesis diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 54e181d9e0..544660c3b5 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -241,7 +241,7 @@ public void upload( BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) throws IOException { try (WriteChannel writer = writer(blobInfo, options)) { - upload(Channels.newChannel(content), writer, bufferSize); + uploadHelper(Channels.newChannel(content), writer, bufferSize); } } @@ -249,7 +249,7 @@ public void upload( * Uploads the given content to the storage using specified write channel and the given buffer * size. This method does not close any channels. */ - private static void upload(ReadableByteChannel reader, WriteChannel writer, int bufferSize) + private static void uploadHelper(ReadableByteChannel reader, WriteChannel writer, int bufferSize) throws IOException { bufferSize = Math.max(bufferSize, MIN_BUFFER_SIZE); ByteBuffer buffer = ByteBuffer.allocate(bufferSize); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java new file mode 100644 index 0000000000..eb1867e327 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java @@ -0,0 +1,583 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.core.ApiClock; +import com.google.cloud.Identity; +import com.google.cloud.Policy; +import com.google.cloud.ServiceOptions; +import com.google.cloud.storage.spi.StorageRpcFactory; +import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.BaseEncoding; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.Key; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.spec.EncodedKeySpec; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.util.Map; +import javax.crypto.spec.SecretKeySpec; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class StorageImplMockitoTest { + + private static final String BUCKET_NAME1 = "b1"; + private static final String BUCKET_NAME2 = "b2"; + private static final String BUCKET_NAME3 = "b3"; + private static final String BLOB_NAME1 = "n1"; + private static final String BLOB_NAME2 = "n2"; + private static final String BLOB_NAME3 = "n3"; + private static final byte[] BLOB_CONTENT = {0xD, 0xE, 0xA, 0xD}; + private static final byte[] BLOB_SUB_CONTENT = {0xE, 0xA}; + private static final String CONTENT_MD5 = "O1R4G1HJSDUISJjoIYmVhQ=="; + private static final String CONTENT_CRC32C = "9N3EPQ=="; + private static final String SUB_CONTENT_MD5 = "5e7c7CdasUiOn3BO560jPg=="; + private static final String SUB_CONTENT_CRC32C = "bljNYA=="; + private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024; + private static final String BASE64_KEY = "JVzfVl8NLD9FjedFuStegjRfES5ll5zc59CIXw572OA="; + private static final Key KEY = + new SecretKeySpec(BaseEncoding.base64().decode(BASE64_KEY), "AES256"); + private static final String KMS_KEY_NAME = + "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key"; + private static final Long RETENTION_PERIOD = 10L; + private static final String USER_PROJECT = "test-project"; + private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024; + private static final int MIN_BUFFER_SIZE = 256 * 1024; + // BucketInfo objects + private static final BucketInfo BUCKET_INFO1 = + BucketInfo.newBuilder(BUCKET_NAME1).setMetageneration(42L).build(); + private static final BucketInfo BUCKET_INFO2 = BucketInfo.newBuilder(BUCKET_NAME2).build(); + private static final BucketInfo BUCKET_INFO3 = + BucketInfo.newBuilder(BUCKET_NAME3) + .setRetentionPeriod(RETENTION_PERIOD) + .setRetentionPolicyIsLocked(true) + .setMetageneration(42L) + .build(); + + // BlobInfo objects + private static final BlobInfo BLOB_INFO1 = + BlobInfo.newBuilder(BUCKET_NAME1, BLOB_NAME1, 24L) + .setMetageneration(42L) + .setContentType("application/json") + .setMd5("md5string") + .build(); + private static final BlobInfo BLOB_INFO2 = BlobInfo.newBuilder(BUCKET_NAME1, BLOB_NAME2).build(); + private static final BlobInfo BLOB_INFO3 = BlobInfo.newBuilder(BUCKET_NAME1, BLOB_NAME3).build(); + + // Empty StorageRpc options + private static final Map EMPTY_RPC_OPTIONS = ImmutableMap.of(); + + // Bucket target options + private static final Storage.BucketTargetOption BUCKET_TARGET_METAGENERATION = + Storage.BucketTargetOption.metagenerationMatch(); + private static final Storage.BucketTargetOption BUCKET_TARGET_PREDEFINED_ACL = + Storage.BucketTargetOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE); + private static final Storage.BucketTargetOption BUCKET_TARGET_USER_PROJECT = + Storage.BucketTargetOption.userProject(USER_PROJECT); + private static final Map BUCKET_TARGET_OPTIONS = + ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BUCKET_INFO1.getMetageneration(), + StorageRpc.Option.PREDEFINED_ACL, BUCKET_TARGET_PREDEFINED_ACL.getValue()); + private static final Map BUCKET_TARGET_OPTIONS_LOCK_RETENTION_POLICY = + ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, + BUCKET_INFO3.getMetageneration(), + StorageRpc.Option.USER_PROJECT, + USER_PROJECT); + + // Blob target options (create, update, compose) + private static final Storage.BlobTargetOption BLOB_TARGET_GENERATION = + Storage.BlobTargetOption.generationMatch(); + private static final Storage.BlobTargetOption BLOB_TARGET_METAGENERATION = + Storage.BlobTargetOption.metagenerationMatch(); + private static final Storage.BlobTargetOption BLOB_TARGET_DISABLE_GZIP_CONTENT = + Storage.BlobTargetOption.disableGzipContent(); + private static final Storage.BlobTargetOption BLOB_TARGET_NOT_EXIST = + Storage.BlobTargetOption.doesNotExist(); + private static final Storage.BlobTargetOption BLOB_TARGET_PREDEFINED_ACL = + Storage.BlobTargetOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE); + private static final Map BLOB_TARGET_OPTIONS_CREATE = + ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_INFO1.getMetageneration(), + StorageRpc.Option.IF_GENERATION_MATCH, 0L, + StorageRpc.Option.PREDEFINED_ACL, BUCKET_TARGET_PREDEFINED_ACL.getValue()); + private static final Map BLOB_TARGET_OPTIONS_CREATE_DISABLE_GZIP_CONTENT = + ImmutableMap.of(StorageRpc.Option.IF_DISABLE_GZIP_CONTENT, true); + private static final Map BLOB_TARGET_OPTIONS_UPDATE = + ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_INFO1.getMetageneration(), + StorageRpc.Option.PREDEFINED_ACL, BUCKET_TARGET_PREDEFINED_ACL.getValue()); + private static final Map BLOB_TARGET_OPTIONS_COMPOSE = + ImmutableMap.of( + StorageRpc.Option.IF_GENERATION_MATCH, BLOB_INFO1.getGeneration(), + StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_INFO1.getMetageneration()); + + // Blob write options (create, writer) + private static final Storage.BlobWriteOption BLOB_WRITE_METAGENERATION = + Storage.BlobWriteOption.metagenerationMatch(); + private static final Storage.BlobWriteOption BLOB_WRITE_NOT_EXIST = + Storage.BlobWriteOption.doesNotExist(); + private static final Storage.BlobWriteOption BLOB_WRITE_PREDEFINED_ACL = + Storage.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE); + private static final Storage.BlobWriteOption BLOB_WRITE_MD5_HASH = + Storage.BlobWriteOption.md5Match(); + private static final Storage.BlobWriteOption BLOB_WRITE_CRC2C = + Storage.BlobWriteOption.crc32cMatch(); + + // Bucket get/source options + private static final Storage.BucketSourceOption BUCKET_SOURCE_METAGENERATION = + Storage.BucketSourceOption.metagenerationMatch(BUCKET_INFO1.getMetageneration()); + private static final Map BUCKET_SOURCE_OPTIONS = + ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BUCKET_SOURCE_METAGENERATION.getValue()); + private static final Storage.BucketGetOption BUCKET_GET_METAGENERATION = + Storage.BucketGetOption.metagenerationMatch(BUCKET_INFO1.getMetageneration()); + private static final Storage.BucketGetOption BUCKET_GET_FIELDS = + Storage.BucketGetOption.fields(Storage.BucketField.LOCATION, Storage.BucketField.ACL); + private static final Storage.BucketGetOption BUCKET_GET_EMPTY_FIELDS = + Storage.BucketGetOption.fields(); + private static final Map BUCKET_GET_OPTIONS = + ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BUCKET_SOURCE_METAGENERATION.getValue()); + + // Blob get/source options + private static final Storage.BlobGetOption BLOB_GET_METAGENERATION = + Storage.BlobGetOption.metagenerationMatch(BLOB_INFO1.getMetageneration()); + private static final Storage.BlobGetOption BLOB_GET_GENERATION = + Storage.BlobGetOption.generationMatch(BLOB_INFO1.getGeneration()); + private static final Storage.BlobGetOption BLOB_GET_GENERATION_FROM_BLOB_ID = + Storage.BlobGetOption.generationMatch(); + private static final Storage.BlobGetOption BLOB_GET_FIELDS = + Storage.BlobGetOption.fields(Storage.BlobField.CONTENT_TYPE, Storage.BlobField.CRC32C); + private static final Storage.BlobGetOption BLOB_GET_EMPTY_FIELDS = Storage.BlobGetOption.fields(); + private static final Map BLOB_GET_OPTIONS = + ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_GET_METAGENERATION.getValue(), + StorageRpc.Option.IF_GENERATION_MATCH, BLOB_GET_GENERATION.getValue()); + private static final Storage.BlobSourceOption BLOB_SOURCE_METAGENERATION = + Storage.BlobSourceOption.metagenerationMatch(BLOB_INFO1.getMetageneration()); + private static final Storage.BlobSourceOption BLOB_SOURCE_GENERATION = + Storage.BlobSourceOption.generationMatch(BLOB_INFO1.getGeneration()); + private static final Storage.BlobSourceOption BLOB_SOURCE_GENERATION_FROM_BLOB_ID = + Storage.BlobSourceOption.generationMatch(); + private static final Map BLOB_SOURCE_OPTIONS = + ImmutableMap.of( + StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_SOURCE_METAGENERATION.getValue(), + StorageRpc.Option.IF_GENERATION_MATCH, BLOB_SOURCE_GENERATION.getValue()); + private static final Map BLOB_SOURCE_OPTIONS_COPY = + ImmutableMap.of( + StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH, BLOB_SOURCE_METAGENERATION.getValue(), + StorageRpc.Option.IF_SOURCE_GENERATION_MATCH, BLOB_SOURCE_GENERATION.getValue()); + + // Bucket list options + private static final Storage.BucketListOption BUCKET_LIST_PAGE_SIZE = + Storage.BucketListOption.pageSize(42L); + private static final Storage.BucketListOption BUCKET_LIST_PREFIX = + Storage.BucketListOption.prefix("prefix"); + private static final Storage.BucketListOption BUCKET_LIST_FIELDS = + Storage.BucketListOption.fields(Storage.BucketField.LOCATION, Storage.BucketField.ACL); + private static final Storage.BucketListOption BUCKET_LIST_EMPTY_FIELDS = + Storage.BucketListOption.fields(); + private static final Map BUCKET_LIST_OPTIONS = + ImmutableMap.of( + StorageRpc.Option.MAX_RESULTS, BUCKET_LIST_PAGE_SIZE.getValue(), + StorageRpc.Option.PREFIX, BUCKET_LIST_PREFIX.getValue()); + + // Blob list options + private static final Storage.BlobListOption BLOB_LIST_PAGE_SIZE = + Storage.BlobListOption.pageSize(42L); + private static final Storage.BlobListOption BLOB_LIST_PREFIX = + Storage.BlobListOption.prefix("prefix"); + private static final Storage.BlobListOption BLOB_LIST_FIELDS = + Storage.BlobListOption.fields(Storage.BlobField.CONTENT_TYPE, Storage.BlobField.MD5HASH); + private static final Storage.BlobListOption BLOB_LIST_VERSIONS = + Storage.BlobListOption.versions(false); + private static final Storage.BlobListOption BLOB_LIST_EMPTY_FIELDS = + Storage.BlobListOption.fields(); + private static final Map BLOB_LIST_OPTIONS = + ImmutableMap.of( + StorageRpc.Option.MAX_RESULTS, BLOB_LIST_PAGE_SIZE.getValue(), + StorageRpc.Option.PREFIX, BLOB_LIST_PREFIX.getValue(), + StorageRpc.Option.VERSIONS, BLOB_LIST_VERSIONS.getValue()); + + // ACLs + private static final Acl ACL = Acl.of(Acl.User.ofAllAuthenticatedUsers(), Acl.Role.OWNER); + private static final Acl OTHER_ACL = + Acl.of(new Acl.Project(Acl.Project.ProjectRole.OWNERS, "p"), Acl.Role.READER); + + // Customer supplied encryption key options + private static final Map ENCRYPTION_KEY_OPTIONS = + ImmutableMap.of(StorageRpc.Option.CUSTOMER_SUPPLIED_KEY, BASE64_KEY); + + // Customer managed encryption key options + private static final Map KMS_KEY_NAME_OPTIONS = + ImmutableMap.of(StorageRpc.Option.KMS_KEY_NAME, KMS_KEY_NAME); + // IAM policies + private static final String POLICY_ETAG1 = "CAE="; + private static final String POLICY_ETAG2 = "CAI="; + private static final Policy LIB_POLICY1 = + Policy.newBuilder() + .addIdentity(StorageRoles.objectViewer(), Identity.allUsers()) + .addIdentity( + StorageRoles.objectAdmin(), + Identity.user("test1@gmail.com"), + Identity.user("test2@gmail.com")) + .setEtag(POLICY_ETAG1) + .setVersion(1) + .build(); + + private static final ServiceAccount SERVICE_ACCOUNT = ServiceAccount.of("test@google.com"); + + private static final com.google.api.services.storage.model.Policy API_POLICY1 = + new com.google.api.services.storage.model.Policy() + .setBindings( + ImmutableList.of( + new com.google.api.services.storage.model.Policy.Bindings() + .setMembers(ImmutableList.of("allUsers")) + .setRole("roles/storage.objectViewer"), + new com.google.api.services.storage.model.Policy.Bindings() + .setMembers(ImmutableList.of("user:test1@gmail.com", "user:test2@gmail.com")) + .setRole("roles/storage.objectAdmin"))) + .setEtag(POLICY_ETAG1) + .setVersion(1); + + private static final String PRIVATE_KEY_STRING = + "MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoG" + + "BAL2xolH1zrISQ8+GzOV29BNjjzq4/HIP8Psd1+cZb81vDklSF+95wB250MSE0BDc81pvIMwj5OmIfLg1NY6uB" + + "1xavOPpVdx1z664AGc/BEJ1zInXGXaQ6s+SxGenVq40Yws57gikQGMZjttpf1Qbz4DjkxsbRoeaRHn06n9pH1e" + + "jAgMBAAECgYEAkWcm0AJF5LMhbWKbjkxm/LG06UNApkHX6vTOOOODkonM/qDBnhvKCj8Tan+PaU2j7679Cd19q" + + "xCm4SBQJET7eBhqLD9L2j9y0h2YUQnLbISaqUS1/EXcr2C1Lf9VCEn1y/GYuDYqs85rGoQ4ZYfM9ClROSq86fH" + + "+cbIIssqJqukCQQD18LjfJz/ichFeli5/l1jaFid2XoCH3T6TVuuysszVx68fh60gSIxEF/0X2xB+wuPxTP4IQ" + + "+t8tD/ktd232oWXAkEAxXPych2QBHePk9/lek4tOkKBgfnDzex7S/pI0G1vpB3VmzBbCsokn9lpOv7JV8071GD" + + "lW/7R6jlLfpQy3hN31QJAE10osSk99m5Uv8XDU3hvHnywDrnSFOBulNs7I47AYfSe7TSZhPkxUgsxejddTR27J" + + "LyTI8N1PxRSE4feNSOXcQJAMMKJRJT4U6IS2rmXubREhvaVdLtxFxEnAYQ1JwNfZm/XqBMw6GEy2iaeTetNXVl" + + "ZRQEIoscyn1y2v/No/F5iYQJBAKBOGASoQcBjGTOg/H/SfcE8QVNsKEpthRrs6CkpT80aZ/AV+ksfoIf2zw2M3" + + "mAHfrO+TBLdz4sicuFQvlN9SEc="; + + private static final String PUBLIC_KEY_STRING = + "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC9saJR9c6y" + + "EkPPhszldvQTY486uPxyD/D7HdfnGW/Nbw5JUhfvecAdudDEhNAQ3PNabyDMI+TpiHy4NTWOrgdcWrzj6VXcdc" + + "+uuABnPwRCdcyJ1xl2kOrPksRnp1auNGMLOe4IpEBjGY7baX9UG8+A45MbG0aHmkR59Op/aR9XowIDAQAB"; + + private static final String SIGNED_URL = + "http://www.test.com/test-bucket/test1.txt?GoogleAccessId=testClient-test@test.com&Expires=1553839761&Signature=MJUBXAZ7"; + + private static final ApiClock TIME_SOURCE = + new ApiClock() { + @Override + public long nanoTime() { + return 42_000_000_000L; + } + + @Override + public long millisTime() { + return 42_000L; + } + }; + + // List of chars under test were taken from + // https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters + private static final Map RFC3986_URI_ENCODING_MAP = + ImmutableMap.builder() + .put('!', "%21") + .put('#', "%23") + .put('$', "%24") + .put('&', "%26") + .put('\'', "%27") + .put('(', "%28") + .put(')', "%29") + .put('*', "%2A") + .put('+', "%2B") + .put(',', "%2C") + // NOTE: Whether the forward slash character should be encoded depends on the URI segment + // being encoded. The path segment should not encode forward slashes, but others (e.g. + // query parameter keys and values) should encode them. Tests verifying encoding behavior + // in path segments should make a copy of this map and replace the mapping for '/' to "/". + .put('/', "%2F") + .put(':', "%3A") + .put(';', "%3B") + .put('=', "%3D") + .put('?', "%3F") + .put('@', "%40") + .put('[', "%5B") + .put(']', "%5D") + // In addition to [a-zA-Z0-9], these chars should not be URI-encoded: + .put('-', "-") + .put('_', "_") + .put('.', ".") + .put('~', "~") + .build(); + + private static final String ACCOUNT = "account"; + private static PrivateKey privateKey; + private static PublicKey publicKey; + + private StorageOptions options; + private StorageRpcFactory rpcFactoryMock; + private StorageRpc storageRpcMock; + private Storage storage; + + private Blob expectedBlob1, expectedBlob2, expectedBlob3; + private Bucket expectedBucket1, expectedBucket2, expectedBucket3; + + @BeforeClass + public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException { + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + EncodedKeySpec privateKeySpec = + new PKCS8EncodedKeySpec(BaseEncoding.base64().decode(PRIVATE_KEY_STRING)); + privateKey = keyFactory.generatePrivate(privateKeySpec); + EncodedKeySpec publicKeySpec = + new X509EncodedKeySpec(BaseEncoding.base64().decode(PUBLIC_KEY_STRING)); + publicKey = keyFactory.generatePublic(publicKeySpec); + } + + @Before + public void setUp() { + rpcFactoryMock = mock(StorageRpcFactory.class); + storageRpcMock = mock(StorageRpc.class); + when(rpcFactoryMock.create(any(StorageOptions.class))).thenReturn(storageRpcMock); + options = + StorageOptions.newBuilder() + .setProjectId("projectId") + .setClock(TIME_SOURCE) + .setServiceRpcFactory(rpcFactoryMock) + .setRetrySettings(ServiceOptions.getNoRetrySettings()) + .build(); + } + + private void initializeService() { + storage = options.getService(); + initializeServiceDependentObjects(); + } + + private void initializeServiceDependentObjects() { + expectedBlob1 = new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO1)); + expectedBlob2 = new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO2)); + expectedBlob3 = new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO3)); + expectedBucket1 = new Bucket(storage, new BucketInfo.BuilderImpl(BUCKET_INFO1)); + expectedBucket2 = new Bucket(storage, new BucketInfo.BuilderImpl(BUCKET_INFO2)); + expectedBucket3 = new Bucket(storage, new BucketInfo.BuilderImpl(BUCKET_INFO3)); + } + + @Test + public void testUploadNonExistentFile() { + initializeService(); + String fileName = "non_existing_file.txt"; + try { + storage.upload(BLOB_INFO1, Paths.get(fileName)); + fail(); + } catch (IOException e) { + assertEquals(NoSuchFileException.class, e.getClass()); + assertEquals(fileName, e.getMessage()); + } + } + + @Test + public void testUploadDirectory() throws IOException { + initializeService(); + Path dir = Files.createTempDirectory("unit_"); + try { + storage.upload(BLOB_INFO1, dir); + fail(); + } catch (StorageException e) { + assertEquals(dir + " is a directory", e.getMessage()); + } + } + + private static class UploadParameters { + final String uploadId; + final byte[] buffer; + final int length; + final boolean isLast; + final BlobInfo blobInfo; + + private UploadParameters( + String uploadId, byte[] buffer, int length, boolean isLast, BlobInfo blobInfo) { + this.uploadId = uploadId; + this.buffer = buffer; + this.length = length; + this.isLast = isLast; + this.blobInfo = blobInfo; + } + } + + private UploadParameters initializeUpload(byte[] bytes) { + return initializeUpload(bytes, DEFAULT_BUFFER_SIZE, EMPTY_RPC_OPTIONS); + } + + private UploadParameters initializeUpload(byte[] bytes, int bufferSize) { + return initializeUpload(bytes, bufferSize, EMPTY_RPC_OPTIONS); + } + + private UploadParameters initializeUpload( + byte[] bytes, int bufferSize, Map rpcOptions) { + String uploadId = "upload-id"; + byte[] buffer = new byte[bufferSize]; + System.arraycopy(bytes, 0, buffer, 0, bytes.length); + BlobInfo blobInfo = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + when(storageRpcMock.open(blobInfo.toPb(), rpcOptions)).thenReturn(uploadId); + initializeService(); + return new UploadParameters(uploadId, buffer, bytes.length, true, blobInfo); + } + + private void verifyUpload(UploadParameters parameters) { + verify(storageRpcMock) + .write(parameters.uploadId, parameters.buffer, 0, 0L, parameters.length, parameters.isLast); + } + + @Test + public void testUploadFile() throws Exception { + byte[] dataToSend = {1, 2, 3, 4}; + Path tempFile = Files.createTempFile("testUpload", ".tmp"); + Files.write(tempFile, dataToSend); + + UploadParameters uploadParameters = initializeUpload(dataToSend); + storage.upload(uploadParameters.blobInfo, tempFile); + verifyUpload(uploadParameters); + } + + @Test + public void testUploadStream() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + + UploadParameters uploadParameters = initializeUpload(dataToSend); + storage.upload(uploadParameters.blobInfo, stream); + verifyUpload(uploadParameters); + } + + @Test + public void testUploadWithOptions() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + + UploadParameters uploadParameters = + initializeUpload(dataToSend, DEFAULT_BUFFER_SIZE, KMS_KEY_NAME_OPTIONS); + storage.upload( + uploadParameters.blobInfo, stream, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + verifyUpload(uploadParameters); + } + + @Test + public void testUploadWithBufferSize() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int bufferSize = MIN_BUFFER_SIZE * 2; + + UploadParameters uploadParameters = initializeUpload(dataToSend, bufferSize); + storage.upload(uploadParameters.blobInfo, stream, bufferSize); + verifyUpload(uploadParameters); + } + + @Test + public void testUploadWithBufferSizeAndOptions() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int bufferSize = MIN_BUFFER_SIZE * 2; + + UploadParameters uploadParameters = + initializeUpload(dataToSend, bufferSize, KMS_KEY_NAME_OPTIONS); + storage.upload( + uploadParameters.blobInfo, + stream, + bufferSize, + Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + verifyUpload(uploadParameters); + } + + @Test + public void testUploadWithSmallBufferSize() throws Exception { + byte[] dataToSend = new byte[100_000]; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int smallBufferSize = 100; + + UploadParameters uploadParameters = initializeUpload(dataToSend, MIN_BUFFER_SIZE); + storage.upload(uploadParameters.blobInfo, stream, smallBufferSize); + verifyUpload(uploadParameters); + } + + @Test + public void testUploadWithException() throws Exception { + initializeService(); + String uploadId = "id-exception"; + byte[] bytes = new byte[10]; + byte[] buffer = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(bytes, 0, buffer, 0, bytes.length); + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + when(storageRpcMock.open(info.toPb(), EMPTY_RPC_OPTIONS)).thenReturn(uploadId); + Exception runtimeException = new RuntimeException("message"); + doThrow(runtimeException) + .when(storageRpcMock) + .write(uploadId, buffer, 0, 0L, bytes.length, true); + + InputStream input = new ByteArrayInputStream(bytes); + try { + storage.upload(info, input, MIN_BUFFER_SIZE); + fail(); + } catch (StorageException e) { + assertSame(runtimeException, e.getCause()); + } + } + + @Test + public void testUploadMultipleParts() throws Exception { + initializeService(); + String uploadId = "id-multiple-parts"; + int extraBytes = 10; + int totalSize = MIN_BUFFER_SIZE + extraBytes; + byte[] dataToSend = new byte[totalSize]; + dataToSend[0] = 42; + dataToSend[MIN_BUFFER_SIZE + 1] = 43; + + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + when(storageRpcMock.open(info.toPb(), EMPTY_RPC_OPTIONS)).thenReturn(uploadId); + + InputStream input = new ByteArrayInputStream(dataToSend); + storage.upload(info, input, MIN_BUFFER_SIZE); + + byte[] buffer1 = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(dataToSend, 0, buffer1, 0, MIN_BUFFER_SIZE); + verify(storageRpcMock).write(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false); + + byte[] buffer2 = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(dataToSend, MIN_BUFFER_SIZE, buffer2, 0, extraBytes); + verify(storageRpcMock).write(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true); + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 473ce06f1d..68e3225842 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -59,16 +59,11 @@ import com.google.common.io.BaseEncoding; import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.security.InvalidKeyException; import java.security.Key; import java.security.KeyFactory; @@ -3078,183 +3073,4 @@ public void testWriterWithSignedURL() throws MalformedURLException { assertNotNull(writer); assertTrue(writer.isOpen()); } - - @Test - public void testUploadNonExistentFile() { - EasyMock.replay(storageRpcMock); - initializeService(); - String fileName = "non_existing_file.txt"; - try { - storage.upload(BLOB_INFO1, Paths.get(fileName)); - Assert.fail(); - } catch (IOException e) { - assertEquals(NoSuchFileException.class, e.getClass()); - assertEquals(fileName, e.getMessage()); - } - } - - @Test - public void testUploadDirectory() throws IOException { - EasyMock.replay(storageRpcMock); - initializeService(); - Path dir = Files.createTempDirectory("unit_"); - try { - storage.upload(BLOB_INFO1, dir); - Assert.fail(); - } catch (StorageException e) { - assertEquals(dir + " is a directory", e.getMessage()); - } - } - - private BlobInfo initializeUpload(byte[] bytes) { - return initializeUpload(bytes, DEFAULT_BUFFER_SIZE, EMPTY_RPC_OPTIONS); - } - - private BlobInfo initializeUpload(byte[] bytes, int bufferSize) { - return initializeUpload(bytes, bufferSize, EMPTY_RPC_OPTIONS); - } - - private BlobInfo initializeUpload( - byte[] bytes, int bufferSize, Map rpcOptions) { - String uploadId = "upload-id"; - byte[] buffer = new byte[bufferSize]; - System.arraycopy(bytes, 0, buffer, 0, bytes.length); - BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); - EasyMock.expect(storageRpcMock.open(info.toPb(), rpcOptions)).andReturn(uploadId); - storageRpcMock.write( - EasyMock.eq(uploadId), - EasyMock.aryEq(buffer), - EasyMock.eq(0), - EasyMock.eq(0L), - EasyMock.eq(bytes.length), - EasyMock.eq(true)); - EasyMock.replay(storageRpcMock); - initializeService(); - return info; - } - - @Test - public void testUploadFile() throws Exception { - byte[] dataToSend = {1, 2, 3, 4}; - Path tempFile = Files.createTempFile("testUpload", ".tmp"); - Files.write(tempFile, dataToSend); - - BlobInfo info = initializeUpload(dataToSend); - storage.upload(info, tempFile); - } - - @Test - public void testUploadStream() throws Exception { - byte[] dataToSend = {1, 2, 3, 4, 5}; - ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); - - BlobInfo info = initializeUpload(dataToSend); - storage.upload(info, stream); - } - - @Test - public void testUploadWithOptions() throws Exception { - byte[] dataToSend = {1, 2, 3, 4, 5, 6}; - ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); - - BlobInfo info = initializeUpload(dataToSend, DEFAULT_BUFFER_SIZE, KMS_KEY_NAME_OPTIONS); - storage.upload(info, stream, BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); - } - - @Test - public void testUploadWithBufferSize() throws Exception { - byte[] dataToSend = {1, 2, 3, 4, 5, 6}; - ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); - int bufferSize = MIN_BUFFER_SIZE * 2; - - BlobInfo info = initializeUpload(dataToSend, bufferSize); - storage.upload(info, stream, bufferSize); - } - - @Test - public void testUploadWithBufferSizeAndOptions() throws Exception { - byte[] dataToSend = {1, 2, 3, 4, 5, 6}; - ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); - int bufferSize = MIN_BUFFER_SIZE * 2; - - BlobInfo info = initializeUpload(dataToSend, bufferSize, KMS_KEY_NAME_OPTIONS); - storage.upload(info, stream, bufferSize, BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); - } - - @Test - public void testUploadWithSmallBufferSize() throws Exception { - byte[] dataToSend = new byte[100_000]; - ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); - int smallBufferSize = 100; - - BlobInfo info = initializeUpload(dataToSend, MIN_BUFFER_SIZE); - storage.upload(info, stream, smallBufferSize); - } - - @Test - public void testUploadWithException() throws Exception { - initializeService(); - String uploadId = "id-exception"; - byte[] bytes = new byte[10]; - byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; - System.arraycopy(bytes, 0, buffer, 0, bytes.length); - BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); - EasyMock.expect(storageRpcMock.open(info.toPb(), EMPTY_RPC_OPTIONS)).andReturn(uploadId); - Exception runtimeException = new RuntimeException("message"); - storageRpcMock.write( - EasyMock.eq(uploadId), - EasyMock.aryEq(buffer), - EasyMock.eq(0), - EasyMock.eq(0L), - EasyMock.eq(bytes.length), - EasyMock.eq(true)); - EasyMock.expectLastCall().andThrow(runtimeException); - EasyMock.replay(storageRpcMock); - - InputStream input = new ByteArrayInputStream(bytes); - try { - storage.upload(info, input); - Assert.fail(); - } catch (StorageException e) { - assertSame(runtimeException, e.getCause()); - } - } - - @Test - public void testUploadMultipleParts() throws Exception { - initializeService(); - String uploadId = "id-multiple-parts"; - int extraBytes = 10; - int totalSize = MIN_BUFFER_SIZE + extraBytes; - byte[] dataToSend = new byte[totalSize]; - dataToSend[0] = 42; - dataToSend[MIN_BUFFER_SIZE + 1] = 43; - - BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); - EasyMock.expect(storageRpcMock.open(info.toPb(), EMPTY_RPC_OPTIONS)).andReturn(uploadId); - byte[] buffer1 = new byte[MIN_BUFFER_SIZE]; - System.arraycopy(dataToSend, 0, buffer1, 0, MIN_BUFFER_SIZE); - storageRpcMock.write( - EasyMock.eq(uploadId), - EasyMock.aryEq(buffer1), - EasyMock.eq(0), - EasyMock.eq(0L), - EasyMock.eq(MIN_BUFFER_SIZE), - EasyMock.eq(false)); - - byte[] buffer2 = new byte[MIN_BUFFER_SIZE]; - System.arraycopy(dataToSend, MIN_BUFFER_SIZE, buffer2, 0, extraBytes); - storageRpcMock.write( - EasyMock.eq(uploadId), - EasyMock.aryEq(buffer2), - EasyMock.eq(0), - EasyMock.eq((long) MIN_BUFFER_SIZE), - EasyMock.eq(extraBytes), - EasyMock.eq(true)); - - EasyMock.replay(storageRpcMock); - - InputStream input = new ByteArrayInputStream(dataToSend); - storage.upload(info, input, MIN_BUFFER_SIZE); - } } diff --git a/pom.xml b/pom.xml index 8ad94856fb..5f829b5ddb 100644 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,7 @@ 1.93.4 1.9.0 4.13 + 1.10.19 1.4.3 1.3.2 1.18 @@ -190,6 +191,12 @@ 3.6 test + + org.mockito + mockito-core + ${mockito.version} + test + org.objenesis objenesis From 83ad7022e89f0db93561700477eee54804b2bdfe Mon Sep 17 00:00:00 2001 From: dmitry-fa Date: Mon, 27 Apr 2020 16:15:51 +0300 Subject: [PATCH 3/9] feat: update upload to return blob --- .../clirr-ignored-differences.xml | 16 +- .../cloud/storage/BlobWriteChannel.java | 15 +- .../com/google/cloud/storage/Storage.java | 20 ++- .../com/google/cloud/storage/StorageImpl.java | 19 ++- .../cloud/storage/spi/v1/HttpStorageRpc.java | 12 +- .../cloud/storage/spi/v1/StorageRpc.java | 5 +- .../cloud/storage/BlobWriteChannelTest.java | 139 +++++++++++++----- .../cloud/storage/StorageImplMockitoTest.java | 105 ++++++------- .../cloud/storage/it/ITStorageTest.java | 8 +- 9 files changed, 221 insertions(+), 118 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index fea926f574..0f5c53287e 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -4,11 +4,23 @@ 7012 com/google/cloud/storage/Storage - void upload(*.BlobInfo, java.nio.file.Path, *.Storage$BlobWriteOption[]) + *.Blob upload(*.BlobInfo, java.nio.file.Path, *.Storage$BlobWriteOption[]) 7012 com/google/cloud/storage/Storage - void upload(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[]) + *.Blob upload(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[]) + + + 7006 + com/google/cloud/storage/spi/v1/StorageRpc + void write(*.String, byte[], int, long, int, boolean) + com.google.api.services.storage.model.StorageObject + + + 7006 + com/google/cloud/storage/spi/v1/HttpStorageRpc + void write(*.String, byte[], int, long, int, boolean) + com.google.api.services.storage.model.StorageObject diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java index ec5376697c..93314dabf0 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java @@ -19,6 +19,7 @@ import static com.google.cloud.RetryHelper.runWithRetries; import static java.util.concurrent.Executors.callable; +import com.google.api.services.storage.model.StorageObject; import com.google.cloud.BaseWriteChannel; import com.google.cloud.RestorableState; import com.google.cloud.RetryHelper; @@ -47,6 +48,13 @@ class BlobWriteChannel extends BaseWriteChannel { super(options, null, uploadId); } + // Contains metadata of the updated object or null if upload is not completed. + private StorageObject objectProto; + + StorageObject getObjectProto() { + return objectProto; + } + @Override protected void flushBuffer(final int length, final boolean last) { try { @@ -55,9 +63,10 @@ protected void flushBuffer(final int length, final boolean last) { new Runnable() { @Override public void run() { - getOptions() - .getStorageRpcV1() - .write(getUploadId(), getBuffer(), 0, getPosition(), length, last); + objectProto = + getOptions() + .getStorageRpcV1() + .write(getUploadId(), getBuffer(), 0, getPosition(), length, last); } }), getOptions().getRetrySettings(), diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 7272bb693f..a1094eac7b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -1720,7 +1720,7 @@ public static Builder newBuilder() { * Blob blob = storage.create(blobInfo); * } * - * @return a [@code Blob} with complete information + * @return a {@code Blob} with complete information * @throws StorageException upon failure */ Blob create(BlobInfo blobInfo, BlobTargetOption... options); @@ -1741,7 +1741,7 @@ public static Builder newBuilder() { * Blob blob = storage.create(blobInfo, "Hello, World!".getBytes(UTF_8)); * } * - * @return a [@code Blob} with complete information + * @return a {@code Blob} with complete information * @throws StorageException upon failure * @see Hashes and ETags */ @@ -1764,7 +1764,7 @@ public static Builder newBuilder() { * Blob blob = storage.create(blobInfo, "Hello, World!".getBytes(UTF_8), 7, 5); * } * - * @return a [@code Blob} with complete information + * @return a {@code Blob} with complete information * @throws StorageException upon failure * @see Hashes and ETags */ @@ -1807,7 +1807,7 @@ Blob create( * Blob blob = storage.create(blobInfo, content, BlobWriteOption.encryptionKey(encryptionKey)); * } * - * @return a [@code Blob} with complete information + * @return a {@code Blob} with complete information * @throws StorageException upon failure */ @Deprecated @@ -1832,11 +1832,12 @@ Blob create( * @param blobInfo blob to create * @param path file to upload * @param options blob write options + * @return a {@code Blob} with complete information * @throws IOException on I/O error * @throws StorageException on failure * @see #upload(BlobInfo, Path, int, BlobWriteOption...) */ - void upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException; + Blob upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException; /** * Uploads {@code path} to the blob using {@link #writer} and {@code bufferSize}. By default any @@ -1865,10 +1866,11 @@ Blob create( * @param path file to upload * @param bufferSize size of the buffer I/O operations * @param options blob write options + * @return a {@code Blob} with complete information * @throws IOException on I/O error * @throws StorageException on failure */ - void upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) + Blob upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) throws IOException; /** @@ -1890,11 +1892,12 @@ void upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... opt * @param blobInfo blob to create * @param content input stream to read from * @param options blob write options + * @return a {@code Blob} with complete information * @throws IOException on I/O error * @throws StorageException on failure * @see #upload(BlobInfo, InputStream, int, BlobWriteOption...) */ - void upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) + Blob upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) throws IOException; /** @@ -1914,10 +1917,11 @@ void upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) * @param content input stream to read from * @param bufferSize size of the buffer I/O operations * @param options blob write options + * @return a {@code Blob} with complete information * @throws IOException on I/O error * @throws StorageException on failure */ - void upload(BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) + Blob upload(BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) throws IOException; /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 544660c3b5..fc99d9e176 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -215,34 +215,39 @@ public StorageObject call() { } @Override - public void upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException { - upload(blobInfo, path, DEFAULT_BUFFER_SIZE, options); + public Blob upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException { + return upload(blobInfo, path, DEFAULT_BUFFER_SIZE, options); } @Override - public void upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) + public Blob upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) throws IOException { if (Files.isDirectory(path)) { throw new StorageException(0, path + " is a directory"); } try (InputStream input = Files.newInputStream(path)) { - upload(blobInfo, input, bufferSize, options); + return upload(blobInfo, input, bufferSize, options); } } @Override - public void upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) + public Blob upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) throws IOException { - upload(blobInfo, content, DEFAULT_BUFFER_SIZE, options); + return upload(blobInfo, content, DEFAULT_BUFFER_SIZE, options); } @Override - public void upload( + public Blob upload( BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) throws IOException { + + BlobWriteChannel blobWriteChannel; try (WriteChannel writer = writer(blobInfo, options)) { + blobWriteChannel = (BlobWriteChannel) writer; uploadHelper(Channels.newChannel(content), writer, bufferSize); } + StorageObject objectProto = blobWriteChannel.getObjectProto(); + return Blob.fromPb(this, objectProto); } /* diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index c677cbdb6a..6bc5d5e172 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -713,7 +713,7 @@ public Tuple read( } @Override - public void write( + public StorageObject write( String uploadId, byte[] toWrite, int toWriteOffset, @@ -722,9 +722,10 @@ public void write( boolean last) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_WRITE); Scope scope = tracer.withSpan(span); + StorageObject updatedBlob = null; try { if (length == 0 && !last) { - return; + return updatedBlob; } GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = @@ -745,6 +746,9 @@ public void write( range.append('*'); } httpRequest.getHeaders().setContentRange(range.toString()); + if (last) { + httpRequest.setParser(storage.getObjectParser()); + } int code; String message; IOException exception = null; @@ -753,6 +757,9 @@ public void write( response = httpRequest.execute(); code = response.getStatusCode(); message = response.getStatusMessage(); + if (last && (code == 200 || code == 201)) { + updatedBlob = response.parseAs(StorageObject.class); + } } catch (HttpResponseException ex) { exception = ex; code = ex.getStatusCode(); @@ -778,6 +785,7 @@ public void write( scope.close(); span.end(); } + return updatedBlob; } @Override diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 36c7a5ff1b..97d15dc82c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -313,11 +313,12 @@ StorageObject compose( String open(String signedURL); /** - * Writes the provided bytes to a storage object at the provided location. + * Writes the provided bytes to a storage object at the provided location. If {@code last=true} + * returns metadata of the updated object, otherwise returns null. * * @throws StorageException upon failure */ - void write( + StorageObject write( String uploadId, byte[] toWrite, int toWriteOffset, diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java index fdbb932b0c..cec9e52b72 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java @@ -27,10 +27,14 @@ import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.api.services.storage.model.StorageObject; import com.google.cloud.RestorableState; import com.google.cloud.WriteChannel; import com.google.cloud.storage.spi.StorageRpcFactory; @@ -57,6 +61,7 @@ public class BlobWriteChannelTest { private static final String BLOB_NAME = "n"; private static final String UPLOAD_ID = "uploadid"; private static final BlobInfo BLOB_INFO = BlobInfo.newBuilder(BUCKET_NAME, BLOB_NAME).build(); + private static final StorageObject UPDATED_BLOB = new StorageObject(); private static final Map EMPTY_RPC_OPTIONS = ImmutableMap.of(); private static final int MIN_CHUNK_SIZE = 256 * 1024; private static final int DEFAULT_CHUNK_SIZE = 60 * MIN_CHUNK_SIZE; // 15MiB @@ -94,6 +99,7 @@ public void testCreate() { replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); + assertNull(writer.getObjectProto()); } @Test @@ -104,6 +110,7 @@ public void testCreateRetryableError() { replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); + assertNull(writer.getObjectProto()); } @Test @@ -131,28 +138,44 @@ public void testWriteWithoutFlush() throws IOException { public void testWriteWithFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(CUSTOM_CHUNK_SIZE), eq(false)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(CUSTOM_CHUNK_SIZE), + eq(false))) + .andReturn(null); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); writer.setChunkSize(CUSTOM_CHUNK_SIZE); ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE); assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer)); assertArrayEquals(buffer.array(), capturedBuffer.getValue()); + assertNull(writer.getObjectProto()); } @Test public void testWritesAndFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(DEFAULT_CHUNK_SIZE), eq(false)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(DEFAULT_CHUNK_SIZE), + eq(false))) + .andReturn(null); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE]; for (int i = 0; i < buffers.length; i++) { buffers[i] = randomBuffer(MIN_CHUNK_SIZE); assertEquals(MIN_CHUNK_SIZE, writer.write(buffers[i])); + assertNull(writer.getObjectProto()); } for (int i = 0; i < buffers.length; i++) { assertArrayEquals( @@ -166,13 +189,17 @@ public void testWritesAndFlush() throws IOException { public void testCloseWithoutFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); writer.close(); assertArrayEquals(new byte[0], capturedBuffer.getValue()); - assertTrue(!writer.isOpen()); + assertFalse(writer.isOpen()); + assertSame(UPDATED_BLOB, writer.getObjectProto()); } @Test @@ -180,8 +207,15 @@ public void testCloseWithFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(MIN_CHUNK_SIZE), eq(true)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(MIN_CHUNK_SIZE), + eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); @@ -189,14 +223,18 @@ public void testCloseWithFlush() throws IOException { writer.close(); assertEquals(DEFAULT_CHUNK_SIZE, capturedBuffer.getValue().length); assertArrayEquals(buffer.array(), Arrays.copyOf(capturedBuffer.getValue(), MIN_CHUNK_SIZE)); - assertTrue(!writer.isOpen()); + assertFalse(writer.isOpen()); + assertSame(UPDATED_BLOB, writer.getObjectProto()); } @Test public void testWriteClosed() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); writer.close(); @@ -206,6 +244,7 @@ public void testWriteClosed() throws IOException { } catch (IOException ex) { // expected } + assertSame(UPDATED_BLOB, writer.getObjectProto()); } @Test @@ -213,13 +252,15 @@ public void testSaveAndRestore() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); - storageRpcMock.write( - eq(UPLOAD_ID), - capture(capturedBuffer), - eq(0), - captureLong(capturedPosition), - eq(DEFAULT_CHUNK_SIZE), - eq(false)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + captureLong(capturedPosition), + eq(DEFAULT_CHUNK_SIZE), + eq(false))) + .andReturn(null); expectLastCall().times(2); replay(storageRpcMock); ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE); @@ -239,7 +280,10 @@ public void testSaveAndRestore() throws IOException { public void testSaveAndRestoreClosed() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); writer.close(); @@ -283,8 +327,15 @@ public void testWriteWithSignedURLAndWithoutFlush() throws IOException { public void testWriteWithSignedURLAndWithFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(CUSTOM_CHUNK_SIZE), eq(false)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(CUSTOM_CHUNK_SIZE), + eq(false))) + .andReturn(null); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); writer.setChunkSize(CUSTOM_CHUNK_SIZE); @@ -297,8 +348,15 @@ public void testWriteWithSignedURLAndWithFlush() throws IOException { public void testWriteWithSignedURLAndFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(DEFAULT_CHUNK_SIZE), eq(false)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(DEFAULT_CHUNK_SIZE), + eq(false))) + .andReturn(null); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE]; @@ -318,7 +376,10 @@ public void testWriteWithSignedURLAndFlush() throws IOException { public void testCloseWithSignedURLWithoutFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); assertTrue(writer.isOpen()); @@ -332,8 +393,15 @@ public void testCloseWithSignedURLWithFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(MIN_CHUNK_SIZE), eq(true)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(MIN_CHUNK_SIZE), + eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); assertTrue(writer.isOpen()); @@ -348,7 +416,10 @@ public void testCloseWithSignedURLWithFlush() throws IOException { public void testWriteWithSignedURLClosed() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); writer.close(); @@ -365,13 +436,15 @@ public void testSaveAndRestoreWithSignedURL() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); - storageRpcMock.write( - eq(UPLOAD_ID), - capture(capturedBuffer), - eq(0), - captureLong(capturedPosition), - eq(DEFAULT_CHUNK_SIZE), - eq(false)); + expect( + storageRpcMock.write( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + captureLong(capturedPosition), + eq(DEFAULT_CHUNK_SIZE), + eq(false))) + .andReturn(null); expectLastCall().times(2); replay(storageRpcMock); ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java index eb1867e327..8eea17d4e3 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java @@ -22,10 +22,10 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.api.core.ApiClock; +import com.google.api.services.storage.model.StorageObject; import com.google.cloud.Identity; import com.google.cloud.Policy; import com.google.cloud.ServiceOptions; @@ -37,6 +37,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -354,7 +355,7 @@ public long millisTime() { private StorageRpc storageRpcMock; private Storage storage; - private Blob expectedBlob1, expectedBlob2, expectedBlob3; + private Blob expectedBlob1, expectedBlob2, expectedBlob3, expectedUpdated; private Bucket expectedBucket1, expectedBucket2, expectedBucket3; @BeforeClass @@ -394,6 +395,7 @@ private void initializeServiceDependentObjects() { expectedBucket1 = new Bucket(storage, new BucketInfo.BuilderImpl(BUCKET_INFO1)); expectedBucket2 = new Bucket(storage, new BucketInfo.BuilderImpl(BUCKET_INFO2)); expectedBucket3 = new Bucket(storage, new BucketInfo.BuilderImpl(BUCKET_INFO3)); + expectedUpdated = null; } @Test @@ -421,45 +423,30 @@ public void testUploadDirectory() throws IOException { } } - private static class UploadParameters { - final String uploadId; - final byte[] buffer; - final int length; - final boolean isLast; - final BlobInfo blobInfo; - - private UploadParameters( - String uploadId, byte[] buffer, int length, boolean isLast, BlobInfo blobInfo) { - this.uploadId = uploadId; - this.buffer = buffer; - this.length = length; - this.isLast = isLast; - this.blobInfo = blobInfo; - } - } - - private UploadParameters initializeUpload(byte[] bytes) { + private BlobInfo initializeUpload(byte[] bytes) { return initializeUpload(bytes, DEFAULT_BUFFER_SIZE, EMPTY_RPC_OPTIONS); } - private UploadParameters initializeUpload(byte[] bytes, int bufferSize) { + private BlobInfo initializeUpload(byte[] bytes, int bufferSize) { return initializeUpload(bytes, bufferSize, EMPTY_RPC_OPTIONS); } - private UploadParameters initializeUpload( + private BlobInfo initializeUpload( byte[] bytes, int bufferSize, Map rpcOptions) { String uploadId = "upload-id"; byte[] buffer = new byte[bufferSize]; System.arraycopy(bytes, 0, buffer, 0, bytes.length); BlobInfo blobInfo = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + StorageObject storageObject = new StorageObject(); + storageObject.setBucket(BLOB_INFO1.getBucket()); + storageObject.setName(BLOB_INFO1.getName()); + storageObject.setSize(BigInteger.valueOf(bytes.length)); when(storageRpcMock.open(blobInfo.toPb(), rpcOptions)).thenReturn(uploadId); + when(storageRpcMock.write(uploadId, buffer, 0, 0L, bytes.length, true)) + .thenReturn(storageObject); initializeService(); - return new UploadParameters(uploadId, buffer, bytes.length, true, blobInfo); - } - - private void verifyUpload(UploadParameters parameters) { - verify(storageRpcMock) - .write(parameters.uploadId, parameters.buffer, 0, 0L, parameters.length, parameters.isLast); + expectedUpdated = Blob.fromPb(storage, storageObject); + return blobInfo; } @Test @@ -468,9 +455,9 @@ public void testUploadFile() throws Exception { Path tempFile = Files.createTempFile("testUpload", ".tmp"); Files.write(tempFile, dataToSend); - UploadParameters uploadParameters = initializeUpload(dataToSend); - storage.upload(uploadParameters.blobInfo, tempFile); - verifyUpload(uploadParameters); + BlobInfo blobInfo = initializeUpload(dataToSend); + Blob blob = storage.upload(blobInfo, tempFile); + assertEquals(expectedUpdated, blob); } @Test @@ -478,9 +465,9 @@ public void testUploadStream() throws Exception { byte[] dataToSend = {1, 2, 3, 4, 5}; ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); - UploadParameters uploadParameters = initializeUpload(dataToSend); - storage.upload(uploadParameters.blobInfo, stream); - verifyUpload(uploadParameters); + BlobInfo blobInfo = initializeUpload(dataToSend); + Blob blob = storage.upload(blobInfo, stream); + assertEquals(expectedUpdated, blob); } @Test @@ -488,11 +475,9 @@ public void testUploadWithOptions() throws Exception { byte[] dataToSend = {1, 2, 3, 4, 5, 6}; ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); - UploadParameters uploadParameters = - initializeUpload(dataToSend, DEFAULT_BUFFER_SIZE, KMS_KEY_NAME_OPTIONS); - storage.upload( - uploadParameters.blobInfo, stream, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); - verifyUpload(uploadParameters); + BlobInfo blobInfo = initializeUpload(dataToSend, DEFAULT_BUFFER_SIZE, KMS_KEY_NAME_OPTIONS); + Blob blob = storage.upload(blobInfo, stream, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + assertEquals(expectedUpdated, blob); } @Test @@ -501,9 +486,9 @@ public void testUploadWithBufferSize() throws Exception { ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); int bufferSize = MIN_BUFFER_SIZE * 2; - UploadParameters uploadParameters = initializeUpload(dataToSend, bufferSize); - storage.upload(uploadParameters.blobInfo, stream, bufferSize); - verifyUpload(uploadParameters); + BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize); + Blob blob = storage.upload(blobInfo, stream, bufferSize); + assertEquals(expectedUpdated, blob); } @Test @@ -512,14 +497,11 @@ public void testUploadWithBufferSizeAndOptions() throws Exception { ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); int bufferSize = MIN_BUFFER_SIZE * 2; - UploadParameters uploadParameters = - initializeUpload(dataToSend, bufferSize, KMS_KEY_NAME_OPTIONS); - storage.upload( - uploadParameters.blobInfo, - stream, - bufferSize, - Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); - verifyUpload(uploadParameters); + BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize, KMS_KEY_NAME_OPTIONS); + Blob blob = + storage.upload( + blobInfo, stream, bufferSize, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + assertEquals(expectedUpdated, blob); } @Test @@ -528,9 +510,9 @@ public void testUploadWithSmallBufferSize() throws Exception { ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); int smallBufferSize = 100; - UploadParameters uploadParameters = initializeUpload(dataToSend, MIN_BUFFER_SIZE); - storage.upload(uploadParameters.blobInfo, stream, smallBufferSize); - verifyUpload(uploadParameters); + BlobInfo blobInfo = initializeUpload(dataToSend, MIN_BUFFER_SIZE); + Blob blob = storage.upload(blobInfo, stream, smallBufferSize); + assertEquals(expectedUpdated, blob); } @Test @@ -566,18 +548,25 @@ public void testUploadMultipleParts() throws Exception { dataToSend[0] = 42; dataToSend[MIN_BUFFER_SIZE + 1] = 43; + StorageObject storageObject = new StorageObject(); + storageObject.setBucket(BLOB_INFO1.getBucket()); + storageObject.setName(BLOB_INFO1.getName()); + storageObject.setSize(BigInteger.valueOf(totalSize)); + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); when(storageRpcMock.open(info.toPb(), EMPTY_RPC_OPTIONS)).thenReturn(uploadId); - InputStream input = new ByteArrayInputStream(dataToSend); - storage.upload(info, input, MIN_BUFFER_SIZE); - byte[] buffer1 = new byte[MIN_BUFFER_SIZE]; System.arraycopy(dataToSend, 0, buffer1, 0, MIN_BUFFER_SIZE); - verify(storageRpcMock).write(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false); + when(storageRpcMock.write(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false)).thenReturn(null); byte[] buffer2 = new byte[MIN_BUFFER_SIZE]; System.arraycopy(dataToSend, MIN_BUFFER_SIZE, buffer2, 0, extraBytes); - verify(storageRpcMock).write(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true); + when(storageRpcMock.write(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true)) + .thenReturn(storageObject); + + InputStream input = new ByteArrayInputStream(dataToSend); + Blob blob = storage.upload(info, input, MIN_BUFFER_SIZE); + assertEquals(Blob.fromPb(storage, storageObject), blob); } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 47f404f2c4..983940270e 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -3219,7 +3219,10 @@ public void testUploadFromDownloadTo() throws Exception { Path tempFileFrom = Files.createTempFile("ITStorageTest_", ".tmp"); Files.write(tempFileFrom, BLOB_BYTE_CONTENT); - storage.upload(blobInfo, tempFileFrom); + Blob blob = storage.upload(blobInfo, tempFileFrom); + assertEquals(BUCKET, blob.getBucket()); + assertEquals(blobName, blob.getName()); + assertEquals(BLOB_BYTE_CONTENT.length, (long) blob.getSize()); Path tempFileTo = Files.createTempFile("ITStorageTest_", ".tmp"); storage.get(blobId).downloadTo(tempFileTo); @@ -3234,9 +3237,8 @@ public void testUploadWithEncryption() throws Exception { BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); ByteArrayInputStream content = new ByteArrayInputStream(BLOB_BYTE_CONTENT); - storage.upload(blobInfo, content, Storage.BlobWriteOption.encryptionKey(KEY)); + Blob blob = storage.upload(blobInfo, content, Storage.BlobWriteOption.encryptionKey(KEY)); - Blob blob = storage.get(blobId); try { blob.getContent(); fail("StorageException was expected"); From 94d29483957cc4d7e815e80feb77119f4215fbf7 Mon Sep 17 00:00:00 2001 From: dmitry-fa Date: Mon, 27 Apr 2020 20:09:45 +0300 Subject: [PATCH 4/9] feat: do not parse response from writers for signed urls --- .../com/google/cloud/storage/spi/v1/HttpStorageRpc.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 6bc5d5e172..bd461d2c7a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -757,7 +757,11 @@ public StorageObject write( response = httpRequest.execute(); code = response.getStatusCode(); message = response.getStatusMessage(); - if (last && (code == 200 || code == 201)) { + String contentType = response.getContentType(); + if (last + && (code == 200 || code == 201) + && contentType != null + && contentType.startsWith("application/json")) { updatedBlob = response.parseAs(StorageObject.class); } } catch (HttpResponseException ex) { From 67a701e1970eea0f4f23d5088eeff0415b19ce19 Mon Sep 17 00:00:00 2001 From: dmitry-fa Date: Wed, 29 Apr 2020 22:25:50 +0300 Subject: [PATCH 5/9] fix reviewer's comments --- .../com/google/cloud/storage/BlobWriteChannel.java | 8 ++++---- .../java/com/google/cloud/storage/Storage.java | 8 ++++---- .../java/com/google/cloud/storage/StorageImpl.java | 2 +- .../google/cloud/storage/BlobWriteChannelTest.java | 14 +++++++------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java index 93314dabf0..ee5b7b6a56 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java @@ -49,10 +49,10 @@ class BlobWriteChannel extends BaseWriteChannel { } // Contains metadata of the updated object or null if upload is not completed. - private StorageObject objectProto; + private StorageObject storageObject; - StorageObject getObjectProto() { - return objectProto; + StorageObject getStorageObject() { + return storageObject; } @Override @@ -63,7 +63,7 @@ protected void flushBuffer(final int length, final boolean last) { new Runnable() { @Override public void run() { - objectProto = + storageObject = getOptions() .getStorageRpcV1() .write(getUploadId(), getBuffer(), 0, getPosition(), length, last); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 709ce90146..6d2424d8ee 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -1834,7 +1834,7 @@ Blob create( * @param options blob write options * @return a {@code Blob} with complete information * @throws IOException on I/O error - * @throws StorageException on failure + * @throws StorageException on server side error * @see #upload(BlobInfo, Path, int, BlobWriteOption...) */ Blob upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException; @@ -1868,7 +1868,7 @@ Blob create( * @param options blob write options * @return a {@code Blob} with complete information * @throws IOException on I/O error - * @throws StorageException on failure + * @throws StorageException on server side error */ Blob upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) throws IOException; @@ -1894,7 +1894,7 @@ Blob upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... opt * @param options blob write options * @return a {@code Blob} with complete information * @throws IOException on I/O error - * @throws StorageException on failure + * @throws StorageException on server side error * @see #upload(BlobInfo, InputStream, int, BlobWriteOption...) */ Blob upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) @@ -1919,7 +1919,7 @@ Blob upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) * @param options blob write options * @return a {@code Blob} with complete information * @throws IOException on I/O error - * @throws StorageException on failure + * @throws StorageException on server side error */ Blob upload(BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) throws IOException; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index fc99d9e176..bba1a16a7f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -246,7 +246,7 @@ public Blob upload( blobWriteChannel = (BlobWriteChannel) writer; uploadHelper(Channels.newChannel(content), writer, bufferSize); } - StorageObject objectProto = blobWriteChannel.getObjectProto(); + StorageObject objectProto = blobWriteChannel.getStorageObject(); return Blob.fromPb(this, objectProto); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java index cec9e52b72..9cdf453f1e 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java @@ -99,7 +99,7 @@ public void testCreate() { replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); - assertNull(writer.getObjectProto()); + assertNull(writer.getStorageObject()); } @Test @@ -110,7 +110,7 @@ public void testCreateRetryableError() { replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); - assertNull(writer.getObjectProto()); + assertNull(writer.getStorageObject()); } @Test @@ -153,7 +153,7 @@ public void testWriteWithFlush() throws IOException { ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE); assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer)); assertArrayEquals(buffer.array(), capturedBuffer.getValue()); - assertNull(writer.getObjectProto()); + assertNull(writer.getStorageObject()); } @Test @@ -175,7 +175,7 @@ public void testWritesAndFlush() throws IOException { for (int i = 0; i < buffers.length; i++) { buffers[i] = randomBuffer(MIN_CHUNK_SIZE); assertEquals(MIN_CHUNK_SIZE, writer.write(buffers[i])); - assertNull(writer.getObjectProto()); + assertNull(writer.getStorageObject()); } for (int i = 0; i < buffers.length; i++) { assertArrayEquals( @@ -199,7 +199,7 @@ public void testCloseWithoutFlush() throws IOException { writer.close(); assertArrayEquals(new byte[0], capturedBuffer.getValue()); assertFalse(writer.isOpen()); - assertSame(UPDATED_BLOB, writer.getObjectProto()); + assertSame(UPDATED_BLOB, writer.getStorageObject()); } @Test @@ -224,7 +224,7 @@ public void testCloseWithFlush() throws IOException { assertEquals(DEFAULT_CHUNK_SIZE, capturedBuffer.getValue().length); assertArrayEquals(buffer.array(), Arrays.copyOf(capturedBuffer.getValue(), MIN_CHUNK_SIZE)); assertFalse(writer.isOpen()); - assertSame(UPDATED_BLOB, writer.getObjectProto()); + assertSame(UPDATED_BLOB, writer.getStorageObject()); } @Test @@ -244,7 +244,7 @@ public void testWriteClosed() throws IOException { } catch (IOException ex) { // expected } - assertSame(UPDATED_BLOB, writer.getObjectProto()); + assertSame(UPDATED_BLOB, writer.getStorageObject()); } @Test From d62e615c6fa2bff7c6f42a5c7738317828f39cc3 Mon Sep 17 00:00:00 2001 From: dmitry-fa Date: Thu, 18 Jun 2020 13:36:17 +0300 Subject: [PATCH 6/9] Update Storage.upload() functionality after merge --- .../clirr-ignored-differences.xml | 28 ++- .../com/google/cloud/storage/StorageImpl.java | 63 ++++++ .../storage/testing/StorageRpcTestBase.java | 2 +- .../cloud/storage/StorageImplMockitoTest.java | 182 ++++++++++++++++++ .../cloud/storage/it/ITStorageTest.java | 42 ++++ 5 files changed, 306 insertions(+), 11 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index ace7a6ef0e..79081a5c2d 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -2,28 +2,36 @@ - com/google/cloud/storage/Storage - com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.PostPolicyV4$PostFieldsV4, com.google.cloud.storage.PostPolicyV4$PostConditionsV4, com.google.cloud.storage.Storage$PostPolicyV4Option[]) 7012 + com/google/cloud/storage/Storage + *.Blob upload(*.BlobInfo, java.nio.file.Path, *.Storage$BlobWriteOption[]) - com/google/cloud/storage/Storage - com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.PostPolicyV4$PostFieldsV4, com.google.cloud.storage.Storage$PostPolicyV4Option[]) 7012 + com/google/cloud/storage/Storage + *.Blob upload(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[]) - com/google/cloud/storage/Storage - com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.PostPolicyV4$PostConditionsV4, com.google.cloud.storage.Storage$PostPolicyV4Option[]) - 7012 + 7006 + com/google/cloud/storage/spi/v1/StorageRpc + void write(*.String, byte[], int, long, int, boolean) + com.google.api.services.storage.model.StorageObject - com/google/cloud/storage/Storage - com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.Storage$PostPolicyV4Option[]) - 7012 + 7006 + com/google/cloud/storage/spi/v1/HttpStorageRpc + void write(*.String, byte[], int, long, int, boolean) + com.google.api.services.storage.model.StorageObject com/google/cloud/storage/BucketInfo$Builder com.google.cloud.storage.BucketInfo$Builder deleteLifecycleRules() 7013 + + 7006 + com/google/cloud/storage/testing/StorageRpcTestBase + void write(*.String, byte[], int, long, int, boolean) + com.google.api.services.storage.model.StorageObject + diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 0e24521eb6..90e20b3272 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -48,6 +48,7 @@ import com.google.cloud.ReadChannel; import com.google.cloud.RetryHelper.RetryHelperException; import com.google.cloud.Tuple; +import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Entity; import com.google.cloud.storage.HmacKey.HmacKeyMetadata; import com.google.cloud.storage.PostPolicyV4.ConditionV4Type; @@ -70,12 +71,18 @@ import com.google.common.io.BaseEncoding; import com.google.common.primitives.Ints; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; @@ -99,6 +106,9 @@ final class StorageImpl extends BaseService implements Storage { private static final String STORAGE_XML_URI_HOST_NAME = "storage.googleapis.com"; + private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024; + private static final int MIN_BUFFER_SIZE = 256 * 1024; + private static final Function, Boolean> DELETE_FUNCTION = new Function, Boolean>() { @Override @@ -211,6 +221,59 @@ public StorageObject call() { } } + @Override + public Blob upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException { + return upload(blobInfo, path, DEFAULT_BUFFER_SIZE, options); + } + + @Override + public Blob upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) + throws IOException { + if (Files.isDirectory(path)) { + throw new StorageException(0, path + " is a directory"); + } + try (InputStream input = Files.newInputStream(path)) { + return upload(blobInfo, input, bufferSize, options); + } + } + + @Override + public Blob upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) + throws IOException { + return upload(blobInfo, content, DEFAULT_BUFFER_SIZE, options); + } + + @Override + public Blob upload( + BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) + throws IOException { + + BlobWriteChannel blobWriteChannel; + try (WriteChannel writer = writer(blobInfo, options)) { + blobWriteChannel = (BlobWriteChannel) writer; + uploadHelper(Channels.newChannel(content), writer, bufferSize); + } + StorageObject objectProto = blobWriteChannel.getStorageObject(); + return Blob.fromPb(this, objectProto); + } + + /* + * Uploads the given content to the storage using specified write channel and the given buffer + * size. This method does not close any channels. + */ + private static void uploadHelper(ReadableByteChannel reader, WriteChannel writer, int bufferSize) + throws IOException { + bufferSize = Math.max(bufferSize, MIN_BUFFER_SIZE); + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + writer.setChunkSize(bufferSize); + + while (reader.read(buffer) >= 0) { + buffer.flip(); + writer.write(buffer); + buffer.clear(); + } + } + @Override public Bucket get(String bucket, BucketGetOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb(); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java index 6bd2d487ea..809b24b27f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java @@ -129,7 +129,7 @@ public String open(String signedURL) { } @Override - public void write( + public StorageObject write( String uploadId, byte[] toWrite, int toWriteOffset, diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java index e8525c5c6d..5dfcd6a048 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java @@ -41,7 +41,11 @@ import com.google.common.io.BaseEncoding; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.Key; import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; @@ -961,6 +965,184 @@ public void testCreateBlobFromStreamRetryableException() throws IOException { } } + @Test + public void testUploadDirectory() throws IOException { + initializeService(); + Path dir = Files.createTempDirectory("unit_"); + try { + storage.upload(BLOB_INFO1, dir); + fail(); + } catch (StorageException e) { + assertEquals(dir + " is a directory", e.getMessage()); + } + } + + private BlobInfo initializeUpload(byte[] bytes) { + return initializeUpload(bytes, DEFAULT_BUFFER_SIZE, EMPTY_RPC_OPTIONS); + } + + private BlobInfo initializeUpload(byte[] bytes, int bufferSize) { + return initializeUpload(bytes, bufferSize, EMPTY_RPC_OPTIONS); + } + + private BlobInfo initializeUpload( + byte[] bytes, int bufferSize, Map rpcOptions) { + String uploadId = "upload-id"; + byte[] buffer = new byte[bufferSize]; + System.arraycopy(bytes, 0, buffer, 0, bytes.length); + BlobInfo blobInfo = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + StorageObject storageObject = new StorageObject(); + storageObject.setBucket(BLOB_INFO1.getBucket()); + storageObject.setName(BLOB_INFO1.getName()); + storageObject.setSize(BigInteger.valueOf(bytes.length)); + doReturn(uploadId) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .open(blobInfo.toPb(), rpcOptions); + + doReturn(storageObject) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .write(uploadId, buffer, 0, 0L, bytes.length, true); + + initializeService(); + expectedUpdated = Blob.fromPb(storage, storageObject); + return blobInfo; + } + + @Test + public void testUploadFile() throws Exception { + byte[] dataToSend = {1, 2, 3, 4}; + Path tempFile = Files.createTempFile("testUpload", ".tmp"); + Files.write(tempFile, dataToSend); + + BlobInfo blobInfo = initializeUpload(dataToSend); + Blob blob = storage.upload(blobInfo, tempFile); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testUploadStream() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + + BlobInfo blobInfo = initializeUpload(dataToSend); + Blob blob = storage.upload(blobInfo, stream); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testUploadWithOptions() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + + BlobInfo blobInfo = initializeUpload(dataToSend, DEFAULT_BUFFER_SIZE, KMS_KEY_NAME_OPTIONS); + Blob blob = storage.upload(blobInfo, stream, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testUploadWithBufferSize() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int bufferSize = MIN_BUFFER_SIZE * 2; + + BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize); + Blob blob = storage.upload(blobInfo, stream, bufferSize); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testUploadWithBufferSizeAndOptions() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int bufferSize = MIN_BUFFER_SIZE * 2; + + BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize, KMS_KEY_NAME_OPTIONS); + Blob blob = + storage.upload( + blobInfo, stream, bufferSize, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testUploadWithSmallBufferSize() throws Exception { + byte[] dataToSend = new byte[100_000]; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int smallBufferSize = 100; + + BlobInfo blobInfo = initializeUpload(dataToSend, MIN_BUFFER_SIZE); + Blob blob = storage.upload(blobInfo, stream, smallBufferSize); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testUploadWithException() throws Exception { + initializeService(); + String uploadId = "id-exception"; + byte[] bytes = new byte[10]; + byte[] buffer = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(bytes, 0, buffer, 0, bytes.length); + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + doReturn(uploadId) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .open(info.toPb(), EMPTY_RPC_OPTIONS); + + Exception runtimeException = new RuntimeException("message"); + doThrow(runtimeException) + .when(storageRpcMock) + .write(uploadId, buffer, 0, 0L, bytes.length, true); + + InputStream input = new ByteArrayInputStream(bytes); + try { + storage.upload(info, input, MIN_BUFFER_SIZE); + fail(); + } catch (StorageException e) { + assertSame(runtimeException, e.getCause()); + } + } + + @Test + public void testUploadMultipleParts() throws Exception { + initializeService(); + String uploadId = "id-multiple-parts"; + int extraBytes = 10; + int totalSize = MIN_BUFFER_SIZE + extraBytes; + byte[] dataToSend = new byte[totalSize]; + dataToSend[0] = 42; + dataToSend[MIN_BUFFER_SIZE + 1] = 43; + + StorageObject storageObject = new StorageObject(); + storageObject.setBucket(BLOB_INFO1.getBucket()); + storageObject.setName(BLOB_INFO1.getName()); + storageObject.setSize(BigInteger.valueOf(totalSize)); + + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + doReturn(uploadId) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .open(info.toPb(), EMPTY_RPC_OPTIONS); + + byte[] buffer1 = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(dataToSend, 0, buffer1, 0, MIN_BUFFER_SIZE); + doReturn(null) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .write(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false); + + byte[] buffer2 = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(dataToSend, MIN_BUFFER_SIZE, buffer2, 0, extraBytes); + doReturn(storageObject) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .write(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true); + + InputStream input = new ByteArrayInputStream(dataToSend); + Blob blob = storage.upload(info, input, MIN_BUFFER_SIZE); + assertEquals(Blob.fromPb(storage, storageObject), blob); + } + private void verifyChannelRead(ReadChannel channel, byte[] bytes) throws IOException { assertNotNull(channel); assertTrue(channel.isOpen()); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 7966c9281c..2f753946fd 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -109,6 +109,7 @@ import java.net.URLConnection; import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.Path; import java.security.Key; import java.util.ArrayList; import java.util.Arrays; @@ -3322,4 +3323,45 @@ public void testDeleteLifecycleRules() throws ExecutionException, InterruptedExc RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } } + + @Test + public void testUploadFromDownloadTo() throws Exception { + String blobName = "test-uploadFrom-downloadTo-blob"; + BlobId blobId = BlobId.of(BUCKET, blobName); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); + + Path tempFileFrom = Files.createTempFile("ITStorageTest_", ".tmp"); + Files.write(tempFileFrom, BLOB_BYTE_CONTENT); + Blob blob = storage.upload(blobInfo, tempFileFrom); + assertEquals(BUCKET, blob.getBucket()); + assertEquals(blobName, blob.getName()); + assertEquals(BLOB_BYTE_CONTENT.length, (long) blob.getSize()); + + Path tempFileTo = Files.createTempFile("ITStorageTest_", ".tmp"); + storage.get(blobId).downloadTo(tempFileTo); + byte[] readBytes = Files.readAllBytes(tempFileTo); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } + + @Test + public void testUploadWithEncryption() throws Exception { + String blobName = "test-upload-withEncryption"; + BlobId blobId = BlobId.of(BUCKET, blobName); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); + + ByteArrayInputStream content = new ByteArrayInputStream(BLOB_BYTE_CONTENT); + Blob blob = storage.upload(blobInfo, content, Storage.BlobWriteOption.encryptionKey(KEY)); + + try { + blob.getContent(); + fail("StorageException was expected"); + } catch (StorageException e) { + String expectedMessage = + "The target object is encrypted by a customer-supplied encryption key."; + assertTrue(e.getMessage().contains(expectedMessage)); + assertEquals(400, e.getCode()); + } + byte[] readBytes = blob.getContent(Blob.BlobSourceOption.decryptionKey(KEY)); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } } From d55513fdf83e99626fdb5ad3fc086660474035d5 Mon Sep 17 00:00:00 2001 From: dmitry-fa Date: Thu, 18 Jun 2020 14:33:00 +0300 Subject: [PATCH 7/9] feat: deprecate StorageRpc.write(), introduce StorageRpc.upload() --- .../clirr-ignored-differences.xml | 17 ++--------- .../cloud/storage/BlobWriteChannel.java | 2 +- .../cloud/storage/spi/v1/HttpStorageRpc.java | 13 ++++++++- .../cloud/storage/spi/v1/StorageRpc.java | 28 +++++++++++++++++-- .../storage/testing/StorageRpcTestBase.java | 13 ++++++++- .../cloud/storage/BlobWriteChannelTest.java | 26 ++++++++--------- .../cloud/storage/StorageImplMockitoTest.java | 8 +++--- 7 files changed, 70 insertions(+), 37 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index 79081a5c2d..28e520fd35 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -12,26 +12,13 @@ *.Blob upload(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[]) - 7006 + 7012 com/google/cloud/storage/spi/v1/StorageRpc - void write(*.String, byte[], int, long, int, boolean) - com.google.api.services.storage.model.StorageObject - - - 7006 - com/google/cloud/storage/spi/v1/HttpStorageRpc - void write(*.String, byte[], int, long, int, boolean) - com.google.api.services.storage.model.StorageObject + *.StorageObject upload(*.String, byte[], int, long, int, boolean) com/google/cloud/storage/BucketInfo$Builder com.google.cloud.storage.BucketInfo$Builder deleteLifecycleRules() 7013 - - 7006 - com/google/cloud/storage/testing/StorageRpcTestBase - void write(*.String, byte[], int, long, int, boolean) - com.google.api.services.storage.model.StorageObject - diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java index ee5b7b6a56..ca34524f26 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java @@ -66,7 +66,7 @@ public void run() { storageObject = getOptions() .getStorageRpcV1() - .write(getUploadId(), getBuffer(), 0, getPosition(), length, last); + .upload(getUploadId(), getBuffer(), 0, getPosition(), length, last); } }), getOptions().getRetrySettings(), diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index bf56c95c02..039f651101 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -718,7 +718,18 @@ public Tuple read( } @Override - public StorageObject write( + public void write( + String uploadId, + byte[] toWrite, + int toWriteOffset, + long destOffset, + int length, + boolean last) { + upload(uploadId, toWrite, toWriteOffset, destOffset, length, last); + } + + @Override + public StorageObject upload( String uploadId, byte[] toWrite, int toWriteOffset, diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 97d15dc82c..e06c971b69 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -313,12 +313,36 @@ StorageObject compose( String open(String signedURL); /** - * Writes the provided bytes to a storage object at the provided location. If {@code last=true} + * Writes the provided bytes to a storage object at the provided location. + * + *

This method is {@link Deprecated} because it does not return {@code StorageObject}, use + * {@link #upload(String, byte[], int, long, int, boolean)} instead. + * + * @throws StorageException upon failure + */ + @Deprecated + void write( + String uploadId, + byte[] toWrite, + int toWriteOffset, + long destOffset, + int length, + boolean last); + + /** + * Uploads the provided bytes to a storage object at the provided location. If {@code last=true} * returns metadata of the updated object, otherwise returns null. * + * @param uploadId resumable upload ID + * @param toWrite a portion of the content + * @param toWriteOffset starting position in the {@code toWrite} array + * @param destOffset starting position in the destination data + * @param length the number of bytes to be uploaded + * @param last true, if {@code toWrite} is the final content portion * @throws StorageException upon failure + * @return */ - StorageObject write( + StorageObject upload( String uploadId, byte[] toWrite, int toWriteOffset, diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java index 809b24b27f..424646d485 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java @@ -129,7 +129,18 @@ public String open(String signedURL) { } @Override - public StorageObject write( + public void write( + String uploadId, + byte[] toWrite, + int toWriteOffset, + long destOffset, + int length, + boolean last) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public StorageObject upload( String uploadId, byte[] toWrite, int toWriteOffset, diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java index 9cdf453f1e..0a0a872642 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java @@ -139,7 +139,7 @@ public void testWriteWithFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -161,7 +161,7 @@ public void testWritesAndFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -190,7 +190,7 @@ public void testCloseWithoutFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -208,7 +208,7 @@ public void testCloseWithFlush() throws IOException { Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -232,7 +232,7 @@ public void testWriteClosed() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -253,7 +253,7 @@ public void testSaveAndRestore() throws IOException { Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -281,7 +281,7 @@ public void testSaveAndRestoreClosed() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -328,7 +328,7 @@ public void testWriteWithSignedURLAndWithFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -349,7 +349,7 @@ public void testWriteWithSignedURLAndFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -377,7 +377,7 @@ public void testCloseWithSignedURLWithoutFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -394,7 +394,7 @@ public void testCloseWithSignedURLWithFlush() throws IOException { Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -417,7 +417,7 @@ public void testWriteWithSignedURLClosed() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -437,7 +437,7 @@ public void testSaveAndRestoreWithSignedURL() throws IOException { Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); expect( - storageRpcMock.write( + storageRpcMock.upload( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java index 5dfcd6a048..9a6276c1d3 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java @@ -1003,7 +1003,7 @@ private BlobInfo initializeUpload( doReturn(storageObject) .doThrow(UNEXPECTED_CALL_EXCEPTION) .when(storageRpcMock) - .write(uploadId, buffer, 0, 0L, bytes.length, true); + .upload(uploadId, buffer, 0, 0L, bytes.length, true); initializeService(); expectedUpdated = Blob.fromPb(storage, storageObject); @@ -1092,7 +1092,7 @@ public void testUploadWithException() throws Exception { Exception runtimeException = new RuntimeException("message"); doThrow(runtimeException) .when(storageRpcMock) - .write(uploadId, buffer, 0, 0L, bytes.length, true); + .upload(uploadId, buffer, 0, 0L, bytes.length, true); InputStream input = new ByteArrayInputStream(bytes); try { @@ -1129,14 +1129,14 @@ public void testUploadMultipleParts() throws Exception { doReturn(null) .doThrow(UNEXPECTED_CALL_EXCEPTION) .when(storageRpcMock) - .write(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false); + .upload(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false); byte[] buffer2 = new byte[MIN_BUFFER_SIZE]; System.arraycopy(dataToSend, MIN_BUFFER_SIZE, buffer2, 0, extraBytes); doReturn(storageObject) .doThrow(UNEXPECTED_CALL_EXCEPTION) .when(storageRpcMock) - .write(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true); + .upload(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true); InputStream input = new ByteArrayInputStream(dataToSend); Blob blob = storage.upload(info, input, MIN_BUFFER_SIZE); From 773d587bff7a217dba25485d7d2799ab213df281 Mon Sep 17 00:00:00 2001 From: dmitry-fa Date: Thu, 18 Jun 2020 18:57:22 +0300 Subject: [PATCH 8/9] feat: deprecate StorageRpc.write(), introduce StorageRpc.upload() --- .../clirr-ignored-differences.xml | 2 +- .../cloud/storage/BlobWriteChannel.java | 3 ++- .../cloud/storage/spi/v1/HttpStorageRpc.java | 4 +-- .../cloud/storage/spi/v1/StorageRpc.java | 8 ++---- .../storage/testing/StorageRpcTestBase.java | 2 +- .../cloud/storage/BlobWriteChannelTest.java | 26 +++++++++---------- .../cloud/storage/StorageImplMockitoTest.java | 8 +++--- 7 files changed, 25 insertions(+), 28 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index 28e520fd35..b653e5c34b 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -14,7 +14,7 @@ 7012 com/google/cloud/storage/spi/v1/StorageRpc - *.StorageObject upload(*.String, byte[], int, long, int, boolean) + *.StorageObject writeWithResponse(*.String, byte[], int, long, int, boolean) com/google/cloud/storage/BucketInfo$Builder diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java index ca34524f26..0c9520849b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java @@ -66,7 +66,8 @@ public void run() { storageObject = getOptions() .getStorageRpcV1() - .upload(getUploadId(), getBuffer(), 0, getPosition(), length, last); + .writeWithResponse( + getUploadId(), getBuffer(), 0, getPosition(), length, last); } }), getOptions().getRetrySettings(), diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 039f651101..519a9e9fa2 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -725,11 +725,11 @@ public void write( long destOffset, int length, boolean last) { - upload(uploadId, toWrite, toWriteOffset, destOffset, length, last); + writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last); } @Override - public StorageObject upload( + public StorageObject writeWithResponse( String uploadId, byte[] toWrite, int toWriteOffset, diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index e06c971b69..7ae9c8ec1c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -315,12 +315,8 @@ StorageObject compose( /** * Writes the provided bytes to a storage object at the provided location. * - *

This method is {@link Deprecated} because it does not return {@code StorageObject}, use - * {@link #upload(String, byte[], int, long, int, boolean)} instead. - * * @throws StorageException upon failure */ - @Deprecated void write( String uploadId, byte[] toWrite, @@ -330,7 +326,7 @@ void write( boolean last); /** - * Uploads the provided bytes to a storage object at the provided location. If {@code last=true} + * Writes the provided bytes to a storage object at the provided location. If {@code last=true} * returns metadata of the updated object, otherwise returns null. * * @param uploadId resumable upload ID @@ -342,7 +338,7 @@ void write( * @throws StorageException upon failure * @return */ - StorageObject upload( + StorageObject writeWithResponse( String uploadId, byte[] toWrite, int toWriteOffset, diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java index 424646d485..7733f13eb6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java @@ -140,7 +140,7 @@ public void write( } @Override - public StorageObject upload( + public StorageObject writeWithResponse( String uploadId, byte[] toWrite, int toWriteOffset, diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java index 0a0a872642..a18345be89 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java @@ -139,7 +139,7 @@ public void testWriteWithFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -161,7 +161,7 @@ public void testWritesAndFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -190,7 +190,7 @@ public void testCloseWithoutFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -208,7 +208,7 @@ public void testCloseWithFlush() throws IOException { Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -232,7 +232,7 @@ public void testWriteClosed() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -253,7 +253,7 @@ public void testSaveAndRestore() throws IOException { Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -281,7 +281,7 @@ public void testSaveAndRestoreClosed() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -328,7 +328,7 @@ public void testWriteWithSignedURLAndWithFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -349,7 +349,7 @@ public void testWriteWithSignedURLAndFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -377,7 +377,7 @@ public void testCloseWithSignedURLWithoutFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -394,7 +394,7 @@ public void testCloseWithSignedURLWithFlush() throws IOException { Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), @@ -417,7 +417,7 @@ public void testWriteWithSignedURLClosed() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) .andReturn(UPDATED_BLOB); replay(storageRpcMock); @@ -437,7 +437,7 @@ public void testSaveAndRestoreWithSignedURL() throws IOException { Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); expect( - storageRpcMock.upload( + storageRpcMock.writeWithResponse( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java index 9a6276c1d3..e92df55cd4 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java @@ -1003,7 +1003,7 @@ private BlobInfo initializeUpload( doReturn(storageObject) .doThrow(UNEXPECTED_CALL_EXCEPTION) .when(storageRpcMock) - .upload(uploadId, buffer, 0, 0L, bytes.length, true); + .writeWithResponse(uploadId, buffer, 0, 0L, bytes.length, true); initializeService(); expectedUpdated = Blob.fromPb(storage, storageObject); @@ -1092,7 +1092,7 @@ public void testUploadWithException() throws Exception { Exception runtimeException = new RuntimeException("message"); doThrow(runtimeException) .when(storageRpcMock) - .upload(uploadId, buffer, 0, 0L, bytes.length, true); + .writeWithResponse(uploadId, buffer, 0, 0L, bytes.length, true); InputStream input = new ByteArrayInputStream(bytes); try { @@ -1129,14 +1129,14 @@ public void testUploadMultipleParts() throws Exception { doReturn(null) .doThrow(UNEXPECTED_CALL_EXCEPTION) .when(storageRpcMock) - .upload(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false); + .writeWithResponse(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false); byte[] buffer2 = new byte[MIN_BUFFER_SIZE]; System.arraycopy(dataToSend, MIN_BUFFER_SIZE, buffer2, 0, extraBytes); doReturn(storageObject) .doThrow(UNEXPECTED_CALL_EXCEPTION) .when(storageRpcMock) - .upload(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true); + .writeWithResponse(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true); InputStream input = new ByteArrayInputStream(dataToSend); Blob blob = storage.upload(info, input, MIN_BUFFER_SIZE); From d7c7f1b5ae87b9ab4ad8d706941abb3bbee6daeb Mon Sep 17 00:00:00 2001 From: dmitry-fa Date: Fri, 19 Jun 2020 10:51:37 +0300 Subject: [PATCH 9/9] rename upload to createFrom --- .../clirr-ignored-differences.xml | 4 +- .../com/google/cloud/storage/Storage.java | 25 ++++++------ .../com/google/cloud/storage/StorageImpl.java | 15 +++---- .../cloud/storage/StorageImplMockitoTest.java | 39 ++++++++++--------- .../cloud/storage/it/ITStorageTest.java | 4 +- 5 files changed, 45 insertions(+), 42 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index b653e5c34b..bca2faaff1 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -4,12 +4,12 @@ 7012 com/google/cloud/storage/Storage - *.Blob upload(*.BlobInfo, java.nio.file.Path, *.Storage$BlobWriteOption[]) + *.Blob createFrom(*.BlobInfo, java.nio.file.Path, *.Storage$BlobWriteOption[]) 7012 com/google/cloud/storage/Storage - *.Blob upload(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[]) + *.Blob createFrom(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[]) 7012 diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index b365565575..eb15dd37a2 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -1929,7 +1929,7 @@ Blob create( * String fileName = "readme.txt"; * BlobId blobId = BlobId.of(bucketName, fileName); * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build(); - * storage.upload(blobInfo, Paths.get(fileName)); + * storage.createFrom(blobInfo, Paths.get(fileName)); * } * * @param blobInfo blob to create @@ -1938,9 +1938,9 @@ Blob create( * @return a {@code Blob} with complete information * @throws IOException on I/O error * @throws StorageException on server side error - * @see #upload(BlobInfo, Path, int, BlobWriteOption...) + * @see #createFrom(BlobInfo, Path, int, BlobWriteOption...) */ - Blob upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException; + Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException; /** * Uploads {@code path} to the blob using {@link #writer} and {@code bufferSize}. By default any @@ -1948,8 +1948,8 @@ Blob create( * BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. Folder upload is * not supported. * - *

{@link #upload(BlobInfo, Path, BlobWriteOption...)} invokes this method with a buffer size - * of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the upload + *

{@link #createFrom(BlobInfo, Path, BlobWriteOption...)} invokes this method with a buffer + * size of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the upload * performance but require more memory. This can cause an OutOfMemoryError or add significant * garbage collection overhead. Smaller buffer sizes reduce memory consumption, that is noticeable * when uploading many objects in parallel. Buffer sizes less than 256 KiB are treated as 256 KiB. @@ -1962,7 +1962,7 @@ Blob create( * * int largeBufferSize = 150 * 1024 * 1024; * Path file = Paths.get("humongous.file"); - * storage.upload(blobInfo, file, largeBufferSize); + * storage.createFrom(blobInfo, file, largeBufferSize); * } * * @param blobInfo blob to create @@ -1973,7 +1973,7 @@ Blob create( * @throws IOException on I/O error * @throws StorageException on server side error */ - Blob upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) + Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) throws IOException; /** @@ -1989,7 +1989,7 @@ Blob upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... opt * Hasher hasher = Hashing.crc32c().newHasher().putBytes(content); * String crc32c = BaseEncoding.base64().encode(Ints.toByteArray(hasher.hash().asInt())); * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(crc32c).build(); - * storage.upload(blobInfo, new ByteArrayInputStream(content), Storage.BlobWriteOption.crc32cMatch()); + * storage.createFrom(blobInfo, new ByteArrayInputStream(content), Storage.BlobWriteOption.crc32cMatch()); * } * * @param blobInfo blob to create @@ -1998,9 +1998,9 @@ Blob upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... opt * @return a {@code Blob} with complete information * @throws IOException on I/O error * @throws StorageException on server side error - * @see #upload(BlobInfo, InputStream, int, BlobWriteOption...) + * @see #createFrom(BlobInfo, InputStream, int, BlobWriteOption...) */ - Blob upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) + Blob createFrom(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) throws IOException; /** @@ -2009,7 +2009,7 @@ Blob upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) * ignored unless requested via the {@link BlobWriteOption#md5Match()} and {@link * BlobWriteOption#crc32cMatch()} options. * - *

{@link #upload(BlobInfo, InputStream, BlobWriteOption...)} )} invokes this method with a + *

{@link #createFrom(BlobInfo, InputStream, BlobWriteOption...)} )} invokes this method with a * buffer size of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the * upload performance but require more memory. This can cause an OutOfMemoryError or add * significant garbage collection overhead. Smaller buffer sizes reduce memory consumption, that @@ -2024,7 +2024,8 @@ Blob upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) * @throws IOException on I/O error * @throws StorageException on server side error */ - Blob upload(BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) + Blob createFrom( + BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) throws IOException; /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 90e20b3272..b1f2bfe3c3 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -222,29 +222,30 @@ public StorageObject call() { } @Override - public Blob upload(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException { - return upload(blobInfo, path, DEFAULT_BUFFER_SIZE, options); + public Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options) + throws IOException { + return createFrom(blobInfo, path, DEFAULT_BUFFER_SIZE, options); } @Override - public Blob upload(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) + public Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) throws IOException { if (Files.isDirectory(path)) { throw new StorageException(0, path + " is a directory"); } try (InputStream input = Files.newInputStream(path)) { - return upload(blobInfo, input, bufferSize, options); + return createFrom(blobInfo, input, bufferSize, options); } } @Override - public Blob upload(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) + public Blob createFrom(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) throws IOException { - return upload(blobInfo, content, DEFAULT_BUFFER_SIZE, options); + return createFrom(blobInfo, content, DEFAULT_BUFFER_SIZE, options); } @Override - public Blob upload( + public Blob createFrom( BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) throws IOException { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java index e92df55cd4..9eaefc66fb 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java @@ -966,11 +966,11 @@ public void testCreateBlobFromStreamRetryableException() throws IOException { } @Test - public void testUploadDirectory() throws IOException { + public void testCreateFromDirectory() throws IOException { initializeService(); Path dir = Files.createTempDirectory("unit_"); try { - storage.upload(BLOB_INFO1, dir); + storage.createFrom(BLOB_INFO1, dir); fail(); } catch (StorageException e) { assertEquals(dir + " is a directory", e.getMessage()); @@ -1011,73 +1011,74 @@ private BlobInfo initializeUpload( } @Test - public void testUploadFile() throws Exception { + public void testCreateFromFile() throws Exception { byte[] dataToSend = {1, 2, 3, 4}; - Path tempFile = Files.createTempFile("testUpload", ".tmp"); + Path tempFile = Files.createTempFile("testCreateFrom", ".tmp"); Files.write(tempFile, dataToSend); BlobInfo blobInfo = initializeUpload(dataToSend); - Blob blob = storage.upload(blobInfo, tempFile); + Blob blob = storage.createFrom(blobInfo, tempFile); assertEquals(expectedUpdated, blob); } @Test - public void testUploadStream() throws Exception { + public void testCreateFromStream() throws Exception { byte[] dataToSend = {1, 2, 3, 4, 5}; ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); BlobInfo blobInfo = initializeUpload(dataToSend); - Blob blob = storage.upload(blobInfo, stream); + Blob blob = storage.createFrom(blobInfo, stream); assertEquals(expectedUpdated, blob); } @Test - public void testUploadWithOptions() throws Exception { + public void testCreateFromWithOptions() throws Exception { byte[] dataToSend = {1, 2, 3, 4, 5, 6}; ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); BlobInfo blobInfo = initializeUpload(dataToSend, DEFAULT_BUFFER_SIZE, KMS_KEY_NAME_OPTIONS); - Blob blob = storage.upload(blobInfo, stream, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + Blob blob = + storage.createFrom(blobInfo, stream, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); assertEquals(expectedUpdated, blob); } @Test - public void testUploadWithBufferSize() throws Exception { + public void testCreateFromWithBufferSize() throws Exception { byte[] dataToSend = {1, 2, 3, 4, 5, 6}; ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); int bufferSize = MIN_BUFFER_SIZE * 2; BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize); - Blob blob = storage.upload(blobInfo, stream, bufferSize); + Blob blob = storage.createFrom(blobInfo, stream, bufferSize); assertEquals(expectedUpdated, blob); } @Test - public void testUploadWithBufferSizeAndOptions() throws Exception { + public void testCreateFromWithBufferSizeAndOptions() throws Exception { byte[] dataToSend = {1, 2, 3, 4, 5, 6}; ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); int bufferSize = MIN_BUFFER_SIZE * 2; BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize, KMS_KEY_NAME_OPTIONS); Blob blob = - storage.upload( + storage.createFrom( blobInfo, stream, bufferSize, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); assertEquals(expectedUpdated, blob); } @Test - public void testUploadWithSmallBufferSize() throws Exception { + public void testCreateFromWithSmallBufferSize() throws Exception { byte[] dataToSend = new byte[100_000]; ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); int smallBufferSize = 100; BlobInfo blobInfo = initializeUpload(dataToSend, MIN_BUFFER_SIZE); - Blob blob = storage.upload(blobInfo, stream, smallBufferSize); + Blob blob = storage.createFrom(blobInfo, stream, smallBufferSize); assertEquals(expectedUpdated, blob); } @Test - public void testUploadWithException() throws Exception { + public void testCreateFromWithException() throws Exception { initializeService(); String uploadId = "id-exception"; byte[] bytes = new byte[10]; @@ -1096,7 +1097,7 @@ public void testUploadWithException() throws Exception { InputStream input = new ByteArrayInputStream(bytes); try { - storage.upload(info, input, MIN_BUFFER_SIZE); + storage.createFrom(info, input, MIN_BUFFER_SIZE); fail(); } catch (StorageException e) { assertSame(runtimeException, e.getCause()); @@ -1104,7 +1105,7 @@ public void testUploadWithException() throws Exception { } @Test - public void testUploadMultipleParts() throws Exception { + public void testCreateFromMultipleParts() throws Exception { initializeService(); String uploadId = "id-multiple-parts"; int extraBytes = 10; @@ -1139,7 +1140,7 @@ public void testUploadMultipleParts() throws Exception { .writeWithResponse(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true); InputStream input = new ByteArrayInputStream(dataToSend); - Blob blob = storage.upload(info, input, MIN_BUFFER_SIZE); + Blob blob = storage.createFrom(info, input, MIN_BUFFER_SIZE); assertEquals(Blob.fromPb(storage, storageObject), blob); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 2f753946fd..3d3bda7a9d 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -3332,7 +3332,7 @@ public void testUploadFromDownloadTo() throws Exception { Path tempFileFrom = Files.createTempFile("ITStorageTest_", ".tmp"); Files.write(tempFileFrom, BLOB_BYTE_CONTENT); - Blob blob = storage.upload(blobInfo, tempFileFrom); + Blob blob = storage.createFrom(blobInfo, tempFileFrom); assertEquals(BUCKET, blob.getBucket()); assertEquals(blobName, blob.getName()); assertEquals(BLOB_BYTE_CONTENT.length, (long) blob.getSize()); @@ -3350,7 +3350,7 @@ public void testUploadWithEncryption() throws Exception { BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); ByteArrayInputStream content = new ByteArrayInputStream(BLOB_BYTE_CONTENT); - Blob blob = storage.upload(blobInfo, content, Storage.BlobWriteOption.encryptionKey(KEY)); + Blob blob = storage.createFrom(blobInfo, content, Storage.BlobWriteOption.encryptionKey(KEY)); try { blob.getContent();