From 7e5820a4fbb4c5ba0860275cee54c08a77068b3f Mon Sep 17 00:00:00 2001 From: andrey-qlogic Date: Fri, 30 Nov 2018 12:32:50 +0000 Subject: [PATCH 1/4] 4107: Disconnect response after request --- .../com/google/cloud/storage/spi/v1/HttpStorageRpc.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 709e01a25064..0687fcb49c40 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -715,14 +715,19 @@ public void write( int code; String message; IOException exception = null; + HttpResponse response = null; try { - HttpResponse response = httpRequest.execute(); + response = httpRequest.execute(); code = response.getStatusCode(); message = response.getStatusMessage(); } catch (HttpResponseException ex) { exception = ex; code = ex.getStatusCode(); message = ex.getStatusMessage(); + } finally { + if (response != null ) { + response.disconnect(); + } } if (!last && code != 308 || last && !(code == 200 || code == 201)) { if (exception != null) { From 792220e83f4ed935cf693a7760d986369ea46598 Mon Sep 17 00:00:00 2001 From: andrey-qlogic Date: Sat, 1 Dec 2018 14:33:31 +0000 Subject: [PATCH 2/4] 4107: Added integration test testWriteChannelWithConnectionPool --- .../cloud/storage/it/ITStorageTest.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 0bf20e58dcf2..4f7b6aa36482 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -28,9 +28,12 @@ import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.apache.ApacheHttpTransport; import com.google.api.client.util.DateTime; import com.google.api.gax.paging.Page; import com.google.auth.ServiceAccountSigner; +import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.Identity; import com.google.cloud.Policy; @@ -46,6 +49,8 @@ import com.google.cloud.kms.v1.KeyManagementServiceGrpc; import com.google.cloud.kms.v1.KeyManagementServiceGrpc.KeyManagementServiceBlockingStub; import com.google.cloud.kms.v1.KeyRingName; +import com.google.cloud.TransportOptions; +import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.kms.v1.LocationName; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Acl.Role; @@ -111,6 +116,8 @@ import java.util.logging.Logger; import java.util.zip.GZIPInputStream; import javax.crypto.spec.SecretKeySpec; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -184,6 +191,15 @@ public static void afterClass() throws ExecutionException, InterruptedException } } + private static class CustomHttpTransportFactory implements HttpTransportFactory { + @Override + public HttpTransport create() { + PoolingHttpClientConnectionManager manager = new PoolingHttpClientConnectionManager(); + manager.setMaxTotal(1); + return new ApacheHttpTransport(HttpClients.createMinimal(manager)); + } + } + private static void prepareKmsKeys() throws IOException { String projectId = remoteStorageHelper.getOptions().getProjectId(); GoogleCredentials credentials = GoogleCredentials.getApplicationDefault(); @@ -1744,6 +1760,27 @@ public void testWriteChannelExistingBlob() throws IOException { assertTrue(storage.delete(BUCKET, blobName)); } + @Test(timeout = 5000) + public void testWriteChannelWithConnectionPool() throws IOException { + TransportOptions transportOptions = HttpTransportOptions.newBuilder() + .setHttpTransportFactory(new CustomHttpTransportFactory()).build(); + Storage storageWithPool = + StorageOptions.newBuilder().setTransportOptions(transportOptions).build().getService(); + String blobName = "test-read-and-write-channels-blob"; + BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); + byte[] stringBytes; + try (WriteChannel writer = storageWithPool.writer(blob)) { + stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); + writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); + writer.write(ByteBuffer.wrap(stringBytes)); + } + try (WriteChannel writer = storageWithPool.writer(blob)) { + stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); + writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); + writer.write(ByteBuffer.wrap(stringBytes)); + } + } + @Test public void testGetSignedUrl() throws IOException { if (storage.getOptions().getCredentials() != null) { From 537af3f8a2f2ef724b204bccf6947cb0917792df Mon Sep 17 00:00:00 2001 From: andrey-qlogic Date: Mon, 3 Dec 2018 20:36:38 +0300 Subject: [PATCH 3/4] 4107: Renamed blobName. --- .../test/java/com/google/cloud/storage/it/ITStorageTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 4f7b6aa36482..c547a58b1868 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -1766,7 +1766,7 @@ public void testWriteChannelWithConnectionPool() throws IOException { .setHttpTransportFactory(new CustomHttpTransportFactory()).build(); Storage storageWithPool = StorageOptions.newBuilder().setTransportOptions(transportOptions).build().getService(); - String blobName = "test-read-and-write-channels-blob"; + String blobName = "test-custom-pool-management"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; try (WriteChannel writer = storageWithPool.writer(blob)) { From fab96fcacc67931c26c119882b459617e20c5e58 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Tue, 4 Dec 2018 10:45:12 -0800 Subject: [PATCH 4/4] Fix lint --- .../google/cloud/storage/spi/v1/HttpStorageRpc.java | 2 +- .../com/google/cloud/storage/it/ITStorageTest.java | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 0687fcb49c40..22407f902c92 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -725,7 +725,7 @@ public void write( code = ex.getStatusCode(); message = ex.getStatusMessage(); } finally { - if (response != null ) { + if (response != null) { response.disconnect(); } } diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index c547a58b1868..aba71eb7ddb9 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -39,7 +39,9 @@ import com.google.cloud.Policy; import com.google.cloud.ReadChannel; import com.google.cloud.RestorableState; +import com.google.cloud.TransportOptions; import com.google.cloud.WriteChannel; +import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.kms.v1.CreateCryptoKeyRequest; import com.google.cloud.kms.v1.CreateKeyRingRequest; import com.google.cloud.kms.v1.CryptoKey; @@ -49,8 +51,6 @@ import com.google.cloud.kms.v1.KeyManagementServiceGrpc; import com.google.cloud.kms.v1.KeyManagementServiceGrpc.KeyManagementServiceBlockingStub; import com.google.cloud.kms.v1.KeyRingName; -import com.google.cloud.TransportOptions; -import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.kms.v1.LocationName; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Acl.Role; @@ -1762,8 +1762,10 @@ public void testWriteChannelExistingBlob() throws IOException { @Test(timeout = 5000) public void testWriteChannelWithConnectionPool() throws IOException { - TransportOptions transportOptions = HttpTransportOptions.newBuilder() - .setHttpTransportFactory(new CustomHttpTransportFactory()).build(); + TransportOptions transportOptions = + HttpTransportOptions.newBuilder() + .setHttpTransportFactory(new CustomHttpTransportFactory()) + .build(); Storage storageWithPool = StorageOptions.newBuilder().setTransportOptions(transportOptions).build().getService(); String blobName = "test-custom-pool-management";