From 0613f5391a6ebebed13cc2f01ce45d221a8cc26e Mon Sep 17 00:00:00 2001 From: ozarov Date: Mon, 2 Mar 2015 08:24:21 -0800 Subject: [PATCH 01/48] storage work in progress --- .../java/com/google/gcloud/storage/Acl.java | 48 +++- .../com/google/gcloud/storage/Bucket.java | 40 +-- .../com/google/gcloud/storage/BucketImpl.java | 2 +- .../com/google/gcloud/storage/BucketInfo.java | 108 +++++++ .../java/com/google/gcloud/storage/Cors.java | 25 +- .../com/google/gcloud/storage/ObjectInfo.java | 266 ++++++++++++++++++ ...putChannel.java => ObjectReadChannel.java} | 8 +- ...utChannel.java => ObjectWriteChannel.java} | 8 +- .../google/gcloud/storage/StorageObject.java | 94 ------- .../google/gcloud/storage/StorageService.java | 35 ++- 10 files changed, 480 insertions(+), 154 deletions(-) create mode 100644 src/main/java/com/google/gcloud/storage/BucketInfo.java create mode 100644 src/main/java/com/google/gcloud/storage/ObjectInfo.java rename src/main/java/com/google/gcloud/storage/{InputChannel.java => ObjectReadChannel.java} (83%) rename src/main/java/com/google/gcloud/storage/{OutputChannel.java => ObjectWriteChannel.java} (83%) delete mode 100644 src/main/java/com/google/gcloud/storage/StorageObject.java diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index 4c218848ea9a..aed45c58ffe5 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -16,7 +16,22 @@ package com.google.gcloud.storage; -public interface Acl { +public final class Acl { + + private final String domain; + private final Entity entity; + private final String entityId; + + String email(); + + String etag(); + + String generation(); + + + ProjectTeam projectTeam(); + + String role(); class ProjectTeam { // ProjectNumber: The project number. @@ -26,18 +41,29 @@ class ProjectTeam { //Team string `json:"team,omitempty"` } - enum Entity { - USER_ID("user-userId"), - USER_EMAIL("user-emailAddress"), - GROUP_ID("group-groupId"), - GROUP_EMAIL("group-emailAddress"), - ALL_USERS("allUsers"), - ALL_AUTHENTICATED_USERS("allAuthenticatedUsers"); + public static abstract class Entity { + + private final Type type; - private final String value; + public static final Entity ALL_USERS = new Entity(Type.ALL_USERS); + + public enum Type { + USER_ID("user-userId"), + USER_EMAIL("user-emailAddress"), + GROUP_ID("group-groupId"), + GROUP_EMAIL("group-emailAddress"), + ALL_USERS("allUsers"), + ALL_AUTHENTICATED_USERS("allAuthenticatedUsers"); + + private final String value; + + Type(String value) { + this.value = value; + } + } - Entity(String value) { - this.value = value; + Entity(EntityType type) { + this.type = type; } } diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index 86942b1779e2..13e615e8e1b5 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -16,47 +16,9 @@ package com.google.gcloud.storage; -import java.util.Iterator; - public interface Bucket { - String id(); - - String name(); - - Cors cors(); - - Acl acl(); - - Acl defaultObjectAcl(); - - void updateCors(Cors cors); - - void updateAcl(Acl acl); - - void updateDefaultObjectAcl(); - - void delete(String... objectName); - - void compose(Iterable sourceObjectNames, String destObjectName); - - void copy(String sourceObjectName, StorageObject.Key destObjectKey); - - - // TODO (ozarov): consider replace with Object that has a reference to bucket and name - // that object can return its own meta-data, update its own meta-data, replace its content - // via a stream or byteBuffer, read its content (via stream or ByteBuffer),... - //void copy(String source, String bucket, String dest); - // Also consider read with an offset (and limit). - - // returns null if not exists - StorageObject get(String objectName); - - Iterator get(String... objectName); - - InputChannel getInputChannel(String ObjectName); - OutputChannel getOutputChannel(String ObjectName); + BucketInfo info(); - // TODO: add listing } diff --git a/src/main/java/com/google/gcloud/storage/BucketImpl.java b/src/main/java/com/google/gcloud/storage/BucketImpl.java index 2a7b8b20a6c2..addf35b8f8de 100644 --- a/src/main/java/com/google/gcloud/storage/BucketImpl.java +++ b/src/main/java/com/google/gcloud/storage/BucketImpl.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import com.google.gcloud.storage.StorageObject.Key; import java.util.Iterator; public class BucketImpl implements Bucket { private final StorageServiceImpl storageService; private final com.google.api.services.storage.model.Bucket model; BucketImpl(StorageServiceImpl storageService, com.google.api.services.storage.model.Bucket model) { this.storageService = storageService; this.model = model; } @Override public String id() { return null; } @Override public String name() { return null; } @Override public Cors cors() { return null; } @Override public Acl acl() { return null; } @Override public Acl defaultObjectAcl() { return null; } @Override public void updateCors(Cors cors) { } @Override public void updateAcl(Acl acl) { } @Override public void updateDefaultObjectAcl() { } @Override public void delete(String... objectName) { } @Override public void compose(Iterable sourceObjectNames, String destObjectName) { } @Override public void copy(String sourceObjectName, Key destObjectKey) { } @Override public StorageObject get(String objectName) { return null; } @Override public Iterator get(String... objectName) { return null; } @Override public InputChannel getInputChannel(String ObjectName) { return null; } @Override public OutputChannel getOutputChannel(String ObjectName) { return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import com.google.gcloud.storage.StorageObject.Key; import java.util.Iterator; public class BucketImpl implements Bucket { private final StorageServiceImpl storageService; private final com.google.api.services.storage.model.Bucket model; BucketImpl(StorageServiceImpl storageService, com.google.api.services.storage.model.Bucket model) { this.storageService = storageService; this.model = model; } @Override public String id() { return null; } @Override public String name() { return null; } @Override public Cors cors() { return null; } @Override public Acl acl() { return null; } @Override public Acl defaultObjectAcl() { return null; } @Override public void updateCors(Cors cors) { } @Override public void updateAcl(Acl acl) { } @Override public void updateDefaultObjectAcl() { } @Override public void delete(String... objectName) { } @Override public void compose(Iterable sourceObjectNames, String destObjectName) { } @Override public void copy(String sourceObjectName, Key destObjectKey) { } @Override public StorageObject get(String objectName) { return null; } @Override public Iterator get(String... objectName) { return null; } @Override public ObjectReadChannel getInputChannel(String ObjectName) { return null; } @Override public ObjectWriteChannel getOutputChannel(String ObjectName) { return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java new file mode 100644 index 000000000000..2bf3900002c3 --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -0,0 +1,108 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +public final class BucketInfo { + + private final String id; + private final String name; + private final long createTime; + private final long metageneration; + private final Cors cors; + private final Acl acl; + private final Acl defaultAcl; + + public enum StorageClass { + DURABLE_REDUCED_AVAILABILITY, + STANDARD + } + + public final static class Builder { + + private final String id; + private final String name; + private final long createTime; + private final long metageneration; + private Cors cors; + private Acl acl; + private Acl defaultAcl; + + Builder(String id, String name, long createTime, long metageneration) { + this.id = id; + this.name = name; + this.createTime = createTime; + this.metageneration = metageneration; + } + + public Builder cors(Cors cors) { + this.cors = cors; + return this; + } + + public Builder acl(Acl acl) { + this.acl = acl; + return this; + } + + public Builder defaultAcl(Acl defaultAcl) { + this.defaultAcl = defaultAcl; + return this; + } + + public BucketInfo build() { + return new BucketInfo(this); + } + } + + private BucketInfo(Builder builder) { + id = builder.id; + name = builder.name; + createTime = builder.createTime; + metageneration = builder.metageneration; + cors = builder.cors; + acl = builder.acl; + defaultAcl = builder.defaultAcl; + } + + public String id() { + return id; + } + + public String name() { + return name; + } + + public Cors cors() { + return cors; + } + + public Acl acl() { + return acl; + } + + public Acl defaultObjectAcl() { + return defaultAcl; + } + + public Builder toBuilder() { + return builder(id, name, createTime, metageneration).cors(cors).acl(acl).defaultAcl(defaultAcl); + } + + public static Builder builder(String id, String name, long createTime, long metageneration) { + return new Builder(id, name, createTime, metageneration); + } +} diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java index 71e87da50fd3..c5a944cd6ac9 100644 --- a/src/main/java/com/google/gcloud/storage/Cors.java +++ b/src/main/java/com/google/gcloud/storage/Cors.java @@ -22,7 +22,7 @@ import java.net.URISyntaxException; import java.util.List; -public class Cors { +public final class Cors { private final Integer maxAgeSeconds; private final ImmutableList methods; @@ -73,16 +73,31 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) { return this; } + public Builder methods(List methods) { + this.methods = ImmutableList.copyOf(methods); + return this; + } + public Builder methods(Method... methods) { this.methods = ImmutableList.copyOf(methods); return this; } + public Builder origins(List origins) { + this.origins = ImmutableList.copyOf(origins); + return this; + } + public Builder origins(Origin... origins) { this.origins = ImmutableList.copyOf(origins); return this; } + public Builder responseHeaders(List headers) { + this.responseHeaders = ImmutableList.copyOf(headers); + return this; + } + public Builder responseHeaders(String... responseHeaders) { this.responseHeaders = ImmutableList.copyOf(responseHeaders); return this; @@ -116,6 +131,14 @@ public List responseHeaders() { return responseHeaders; } + public Builder toBuilder() { + return builder() + .maxAgeSeconds(maxAgeSeconds) + .methods(methods) + .origins(origins) + .responseHeaders(responseHeaders); + } + public static Builder builder() { return new Builder(); } diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/ObjectInfo.java new file mode 100644 index 000000000000..97183bc720ea --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/ObjectInfo.java @@ -0,0 +1,266 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; + +import java.util.List; +import java.util.Map; + +public class ObjectInfo { + + private final String bucket; + private final String name; + private final String contentType; + private final String cacheControl; + private final ImmutableList acl; + private final String owner; + private final long size; + private final String contentEncoding; + private final HashCode md5; + private final long crc32; + private final String mediaLink; + private final ImmutableMap metadata; + private final long generation; + private final long metageneration; + private final long deleteTime; + private final long updateTime; + + + public static final class Builder { + + private String bucket; + private String name; + private String contentType; + private String cacheControl; + private ImmutableList acl; + private String owner; + private long size; + private String contentEncoding; + private HashCode md5; + private long crc32; + private String mediaLink; + private ImmutableMap metadata; + private long generation; + private long metageneration; + private long deleteTime; + private long updateTime; + + private Builder() { + } + + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder contentType(String contentType) { + this.contentType = contentType; + return this; + } + + public Builder cacheControl(String cacheControl) { + this.cacheControl = cacheControl; + return this; + } + + public Builder acl(List acl) { + this.acl = ImmutableList.copyOf(acl); + return this; + } + + public Builder owner(String owner) { + this.owner = owner; + return this; + } + + public Builder size(long size) { + this.size = size; + return this; + } + + public Builder contentEncoding(String contentEncoding) { + this.contentEncoding = contentEncoding; + return this; + } + + Builder md5(HashCode md5) { + this.md5 = md5; + return this; + } + + public Builder md5(byte[] bytes) { + this.md5 = bytes == null ? null : Hashing.md5().hashBytes(bytes); + return this; + } + + public Builder crc32(long crc32) { + this.crc32 = crc32; + return this; + } + + public Builder mediaLink(String mediaLink) { + this.mediaLink = mediaLink; + return this; + } + + public Builder metadata(Map metadata) { + this.metadata = ImmutableMap.copyOf(metadata); + return this; + } + + public Builder generation(long generation) { + this.generation = generation; + return this; + } + + public Builder metageneration(long metageneration) { + this.metageneration = metageneration; + return this; + } + + public Builder deleteTime(long deleteTime) { + this.deleteTime = deleteTime; + return this; + } + + public Builder updateTime(long updateTime) { + this.updateTime = updateTime; + return this; + } + + public ObjectInfo build() { + return new ObjectInfo(this); + } + } + + private ObjectInfo(Builder builder) { + bucket = builder.bucket; + name = builder.name; + contentType = builder.contentType; + cacheControl = builder.cacheControl; + acl = builder.acl; + owner = builder.owner; + size = builder.size; + contentEncoding = builder.contentEncoding; + md5 = builder.md5; + crc32 = builder.crc32; + mediaLink = builder.mediaLink; + metadata = builder.metadata; + generation = builder.generation; + metageneration = builder.metageneration; + deleteTime = builder.deleteTime; + updateTime = builder.updateTime; + } + + public String bucket() { + return bucket; + } + + public String name() { + return name; + } + + public String contentType() { + return contentType; + } + + public String cacheControl() { + return cacheControl; + } + + public List acl() { + return acl; + } + + public String owner() { + return owner; + } + + public long size() { + return size; + } + + public String contentEncoding() { + return contentEncoding; + } + + public byte[] md5() { + return md5 == null ? null : md5.asBytes(); + } + + public long crc32() { + return crc32; + } + + public String mediaLink() { + return mediaLink; + } + + public Map metadata() { + return metadata; + } + + public long generation() { + return generation; + } + + public long metageneration() { + return metageneration; + } + + public long deleteTime() { + return deleteTime; + } + + public long updateTime() { + return updateTime; + } + + public Builder toBuilder() { + return builder() + .acl(acl) + .bucket(bucket) + .cacheControl(cacheControl) + .contentEncoding(contentEncoding) + .crc32(crc32) + .contentType(contentType) + .deleteTime(deleteTime) + .generation(generation) + .md5(md5) + .mediaLink(mediaLink) + .metadata(metadata) + .metageneration(metageneration) + .name(name) + .owner(owner) + .updateTime(updateTime) + .size(size); + + } + + public static Builder builder() { + return new Builder(); + } +} diff --git a/src/main/java/com/google/gcloud/storage/InputChannel.java b/src/main/java/com/google/gcloud/storage/ObjectReadChannel.java similarity index 83% rename from src/main/java/com/google/gcloud/storage/InputChannel.java rename to src/main/java/com/google/gcloud/storage/ObjectReadChannel.java index 901ccf048ee9..0dbb320685f1 100644 --- a/src/main/java/com/google/gcloud/storage/InputChannel.java +++ b/src/main/java/com/google/gcloud/storage/ObjectReadChannel.java @@ -21,15 +21,17 @@ import java.nio.channels.ReadableByteChannel; /** - * A readable byte channel for reading data from Google Cloud Storage. + * A channel for reading data from a Google Cloud Storage object. * * Implementations of this class may buffer data internally to reduce remote calls. * * This class is @{link Serializable}, which allows incremental reads. */ -public interface InputChannel extends ReadableByteChannel, Serializable, Closeable { +public interface ObjectReadChannel extends ReadableByteChannel, Serializable, Closeable { - StorageObject.Key key(); + String bucket(); + + String object(); int size(); diff --git a/src/main/java/com/google/gcloud/storage/OutputChannel.java b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java similarity index 83% rename from src/main/java/com/google/gcloud/storage/OutputChannel.java rename to src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java index cf45e292ce2c..8cc86d2c3e00 100644 --- a/src/main/java/com/google/gcloud/storage/OutputChannel.java +++ b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java @@ -21,13 +21,15 @@ import java.nio.channels.WritableByteChannel; /** - * A writable byte channel for writing data to Google Cloud Storage. + * A channel for writing data to a Google Cloud Storage object. * * Implementations of this class may further buffer data internally to reduce remote calls. * Written data will only be visible after calling {@link #close()}. * This class is serializable, to allow incremental writes. */ -public interface OutputChannel extends WritableByteChannel, Serializable, Closeable { +public interface ObjectWriteChannel extends WritableByteChannel, Serializable, Closeable { - StorageObject.Key key(); + String bucket(); + + String object(); } diff --git a/src/main/java/com/google/gcloud/storage/StorageObject.java b/src/main/java/com/google/gcloud/storage/StorageObject.java deleted file mode 100644 index 2a8dcac2f916..000000000000 --- a/src/main/java/com/google/gcloud/storage/StorageObject.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.storage; - -import java.nio.ByteBuffer; - -// TODO: add equals,hashCode, toString, serializable -public interface StorageObject { - - class Key { - - // TODO: add builder, factory method, toURL, from URL, equals,hashCode, toString, serializable - private final String bucket; - private final String name; - - - Key(Bucket bucket, String name) { - this.bucket = bucket.name(); - this.name = name; - } - - public String bucket() { - return bucket; - } - - public String name() { - return name; - } - } - - abstract class Builder { - - private Bucket bucket; - private Acl acl; - private String name; - private ByteBuffer content; - - public Builder() { - - } - - public Builder name(String name) { - this.name = name; - return this; - } - - public Builder bucket(Bucket bucket) { - this.bucket = bucket; - return this; - } - - public Builder acl(Acl acl) { - this.acl = acl; - return this; - } - - public Builder content(ByteBuffer content) { - this.content = content; - return this; - } - - public abstract StorageObject build(); - } - - boolean exists(); - - Key key(); - - Acl acl(); - - ByteBuffer content(); - - void save(); - - void delete(); - - InputChannel getInputChannel(); - - OutputChannel getOutputChannel(); -} diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 3fd0b6b3fa1b..77147b3621ea 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -18,9 +18,40 @@ import com.google.gcloud.Service; +import java.util.Iterator; + public interface StorageService extends Service { - Iterable listBuckets(); + Iterable list(); + + Iterable list(ListSettings settings); + + BucketInfo get(String bucket); + + Iterator get(String... bucket); + + ObjectInfo get(String bucket, String object); + + Iterator get(String bucket, String... object); + + void update(BucketInfo bucketInfo); + + void update(ObjectInfo objectInfo); + + void delete(String bucket, String... object); + + void compose(String bucket, Iterable sourceObjects, String destObject); + + void copy(String fromBucket, String fromObject, String destBucket, String destObject); + + + ObjectReadChannel getInputChannel(String bucket, String ObjectName); + + ObjectWriteChannel write(String ObjectName); + + // TODO: add listing + + ObjectReadChannel getInputChannel(); - Bucket getBucket(String bucketName); + ObjectWriteChannel getOutputChannel(); } From 125d6e9bf0365a76ed60534a96efeb5d7bf427b3 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 2 Mar 2015 17:36:08 -0800 Subject: [PATCH 02/48] wip --- .../java/com/google/gcloud/storage/Acl.java | 91 +++++++++---------- 1 file changed, 44 insertions(+), 47 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index aed45c58ffe5..280139035745 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -16,71 +16,68 @@ package com.google.gcloud.storage; -public final class Acl { +import java.io.Serializable; - private final String domain; - private final Entity entity; - private final String entityId; +public abstract class Acl implements Serializable { - String email(); + private static final long serialVersionUID = 6435575339887912222L; - String etag(); + private final Type type; + private final Role role; - String generation(); - - - ProjectTeam projectTeam(); - - String role(); - - class ProjectTeam { - // ProjectNumber: The project number. - //ProjectNumber string `json:"projectNumber,omitempty"` - - // Team: The team. Can be owners, editors, or viewers. - //Team string `json:"team,omitempty"` + public enum Role { + OWNER, + READER, + WRITER } - public static abstract class Entity { - - private final Type type; + public enum Type { + DOMAIN, + GROUP, + USER, + PROJECT + } - public static final Entity ALL_USERS = new Entity(Type.ALL_USERS); + public static class User extends Acl { - public enum Type { - USER_ID("user-userId"), - USER_EMAIL("user-emailAddress"), - GROUP_ID("group-groupId"), - GROUP_EMAIL("group-emailAddress"), - ALL_USERS("allUsers"), - ALL_AUTHENTICATED_USERS("allAuthenticatedUsers"); + private static final long serialVersionUID = 3076518036392737008L; - private final String value; + private String email; - Type(String value) { - this.value = value; - } + User(Role role, String email) { + super(Type.USER, role); + this.email = email; } - Entity(EntityType type) { - this.type = type; + String email() { + return email; } - } - - String domain(); - Entity entity(); - - String entityId(); + public static User forEmail(Role role, String email) { + return new User(role, email); + } - String email(); + public static User allUsers(Role role) { + return forEmail(role, "allUsers"); + } - String etag(); + public static User allAuthenticatedUsers(Role role) { + return forEmail(role, "allAuthenticatedUsers"); + } + } - String generation(); + + Acl(Type type, Role role) { + this.type = type; + this.role = role; + } - ProjectTeam projectTeam(); + public Type type() { + return type; + } - String role(); + public Role role() { + return role; + } } From 40e9739f5aa0ff541e0ff02febe246fcb4aa6dd8 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 3 Mar 2015 17:36:54 -0800 Subject: [PATCH 03/48] wip --- .../java/com/google/gcloud/storage/Acl.java | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index 280139035745..7cd251d5b43d 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -38,11 +38,33 @@ public enum Type { PROJECT } + public static class Builder { + private Builder() { + + } + + Acl build() + } + + public static class Domain extends Acl { + + private final String domain; + + Domain(Role role, String domain) { + super(Type.USER, role); + this.email = email; + } + + public static User domain(Role role, String domain) { + return new User(role, email); + } + } + public static class User extends Acl { private static final long serialVersionUID = 3076518036392737008L; - private String email; + private final String email; User(Role role, String email) { super(Type.USER, role); @@ -53,20 +75,20 @@ String email() { return email; } - public static User forEmail(Role role, String email) { + public static User email(Role role, String email) { return new User(role, email); } public static User allUsers(Role role) { - return forEmail(role, "allUsers"); + return email(role, "allUsers"); } public static User allAuthenticatedUsers(Role role) { - return forEmail(role, "allAuthenticatedUsers"); + return email(role, "allAuthenticatedUsers"); } } - + Acl(Type type, Role role) { this.type = type; this.role = role; @@ -80,4 +102,12 @@ public Type type() { public Role role() { return role; } + + public Builder toBuilder() { + + } + + public static Builder builder() { + return new Builder(); + } } From 79215c4877a4598aff8391ece9f3a37a8b927898 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 17 Mar 2015 17:41:00 -0700 Subject: [PATCH 04/48] wip --- .../java/com/google/gcloud/storage/Acl.java | 90 +++++++++++++------ .../google/gcloud/storage/ListOptions.java | 1 + .../google/gcloud/storage/StorageService.java | 2 +- 3 files changed, 66 insertions(+), 27 deletions(-) create mode 100644 src/main/java/com/google/gcloud/storage/ListOptions.java diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index 7cd251d5b43d..15d9444311d5 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -38,63 +38,109 @@ public enum Type { PROJECT } - public static class Builder { - private Builder() { + public static class Domain extends Acl { + + private static final long serialVersionUID = -3033025857280447253L; + private final String domain; + + Domain(String domain, Role role) { + super(Type.DOMAIN, role); + this.domain = domain; + } + public String domain() { + return domain; } - Acl build() + public static Domain of(String domain, Role role) { + return new Domain(domain, role); + } } - public static class Domain extends Acl { + public static class Group extends Acl { - private final String domain; + private static final long serialVersionUID = -1660987136294408826L; + private final String email; - Domain(Role role, String domain) { - super(Type.USER, role); + Group(String email, Role role) { + super(Type.GROUP, role); this.email = email; } - public static User domain(Role role, String domain) { - return new User(role, email); + public String email() { + return email; + } + + public static Group of(String email, Role role) { + return new Group(email, role); } } public static class User extends Acl { private static final long serialVersionUID = 3076518036392737008L; + private static final String ALL_USERS = "allUsers"; + private static final String ALL_AUTHENTICATED_USERS = "allAuthenticatedUsers"; private final String email; - User(Role role, String email) { + User(String email, Role role) { super(Type.USER, role); this.email = email; } - String email() { + public String email() { return email; } - public static User email(Role role, String email) { - return new User(role, email); + public static User of(String email, Role role) { + return new User(email, role); } - public static User allUsers(Role role) { - return email(role, "allUsers"); + public static User ofAllUsers(Role role) { + return of(ALL_USERS, role); } - public static User allAuthenticatedUsers(Role role) { - return email(role, "allAuthenticatedUsers"); + public static User ofAllAuthenticatedUsers(Role role) { + return of(ALL_AUTHENTICATED_USERS, role); } } + public static class Project extends Acl { + + private final String projectId; + private final ProjectRole pRole; + + enum ProjectRole { + OWNERS, + EDITORS, + VIEWERS + } + + Project(ProjectRole pRole, String projectId, Role role) { + super(Type.PROJECT, role); + this.pRole = pRole; + this.projectId = projectId; + } + + public ProjectRole projectRole() { + return pRole; + } + + public String projectId() { + return projectId; + } + + public static Project of(ProjectRole pRole, String projectId, Role role) { + return new Project(pRole, projectId, role); + } + } Acl(Type type, Role role) { this.type = type; this.role = role; } - public Type type() { return type; } @@ -102,12 +148,4 @@ public Type type() { public Role role() { return role; } - - public Builder toBuilder() { - - } - - public static Builder builder() { - return new Builder(); - } } diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java new file mode 100644 index 000000000000..9ca3d9120ff1 --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/ListOptions.java @@ -0,0 +1 @@ +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private final boolean recurse; private final String prefix; private final String cursor; private final Integer maxResults; public static class Builder { private boolean recurse; private String prefix; private String cursor; private int maxResults; public Builder() { } public Builder recurse(boolean recurse) { this.recurse = recurse; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } } private ListOptions(Builder builder) { recurse = builder.recurse; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; } public boolean recurse() { return recurse; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 77147b3621ea..48947537653b 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -24,7 +24,7 @@ public interface StorageService extends Service { Iterable list(); - Iterable list(ListSettings settings); + Iterable list(ListOptions settings); BucketInfo get(String bucket); From d15a68214c35cbfc852f14a67d00ac4547aa0b64 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 19 Mar 2015 17:34:15 -0700 Subject: [PATCH 05/48] work in progress --- .../com/google/gcloud/storage/Bucket.java | 24 ----- .../com/google/gcloud/storage/BucketImpl.java | 1 - .../com/google/gcloud/storage/BucketInfo.java | 96 +++++++++++++++---- .../gcloud/storage/StorageServiceImpl.java | 5 +- 4 files changed, 78 insertions(+), 48 deletions(-) delete mode 100644 src/main/java/com/google/gcloud/storage/Bucket.java delete mode 100644 src/main/java/com/google/gcloud/storage/BucketImpl.java diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java deleted file mode 100644 index 13e615e8e1b5..000000000000 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.storage; - -public interface Bucket { - - - BucketInfo info(); - -} diff --git a/src/main/java/com/google/gcloud/storage/BucketImpl.java b/src/main/java/com/google/gcloud/storage/BucketImpl.java deleted file mode 100644 index addf35b8f8de..000000000000 --- a/src/main/java/com/google/gcloud/storage/BucketImpl.java +++ /dev/null @@ -1 +0,0 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import com.google.gcloud.storage.StorageObject.Key; import java.util.Iterator; public class BucketImpl implements Bucket { private final StorageServiceImpl storageService; private final com.google.api.services.storage.model.Bucket model; BucketImpl(StorageServiceImpl storageService, com.google.api.services.storage.model.Bucket model) { this.storageService = storageService; this.model = model; } @Override public String id() { return null; } @Override public String name() { return null; } @Override public Cors cors() { return null; } @Override public Acl acl() { return null; } @Override public Acl defaultObjectAcl() { return null; } @Override public void updateCors(Cors cors) { } @Override public void updateAcl(Acl acl) { } @Override public void updateDefaultObjectAcl() { } @Override public void delete(String... objectName) { } @Override public void compose(Iterable sourceObjectNames, String destObjectName) { } @Override public void copy(String sourceObjectName, Key destObjectKey) { } @Override public StorageObject get(String objectName) { return null; } @Override public Iterator get(String... objectName) { return null; } @Override public ObjectReadChannel getInputChannel(String ObjectName) { return null; } @Override public ObjectWriteChannel getOutputChannel(String ObjectName) { return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java index 2bf3900002c3..d82229e4008d 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -16,50 +16,89 @@ package com.google.gcloud.storage; +import com.google.api.services.storage.model.Bucket; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; + +import java.util.List; + public final class BucketInfo { private final String id; private final String name; + private final String etag; private final long createTime; private final long metageneration; private final Cors cors; - private final Acl acl; - private final Acl defaultAcl; + private final ImmutableList acl; + private final ImmutableList defaultAcl; + private final Location location; + private final StorageClass storageClass; public enum StorageClass { DURABLE_REDUCED_AVAILABILITY, STANDARD } + public enum Location { + US, EU, ASIA + } + public final static class Builder { private final String id; private final String name; - private final long createTime; - private final long metageneration; + private StorageClass storageClass; + private Location location; + private String etag; + private Long createTime; + private Long metageneration; private Cors cors; - private Acl acl; - private Acl defaultAcl; + private Iterable acl; + private Iterable defaultAcl; - Builder(String id, String name, long createTime, long metageneration) { + Builder(String id, String name) { this.id = id; this.name = name; + } + + public Builder storageClass(StorageClass storageClass) { + this.storageClass = storageClass; + return this; + } + + public Builder location(Location location) { + this.location = location; + return this; + } + + Builder etag(String etag) { + this.etag = etag; + return this; + } + + Builder createTime(Long createTime) { this.createTime = createTime; + return this; + } + + Builder metageneration(Long metageneration) { this.metageneration = metageneration; + return this; } - public Builder cors(Cors cors) { + Builder cors(Cors cors) { this.cors = cors; return this; } - public Builder acl(Acl acl) { - this.acl = acl; + public Builder acl(Iterable acl) { + this.acl = ImmutableList.copyOf(acl); return this; } - public Builder defaultAcl(Acl defaultAcl) { - this.defaultAcl = defaultAcl; + public Builder defaultAcl(Iterable acl) { + this.defaultAcl = ImmutableList.copyOf(acl); return this; } @@ -71,11 +110,14 @@ public BucketInfo build() { private BucketInfo(Builder builder) { id = builder.id; name = builder.name; - createTime = builder.createTime; - metageneration = builder.metageneration; + etag = builder.etag; + createTime = MoreObjects.firstNonNull(builder.createTime, 0L); + metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L); + location = builder.location; + storageClass = builder.storageClass; cors = builder.cors; - acl = builder.acl; - defaultAcl = builder.defaultAcl; + acl = ImmutableList.copyOf(builder.acl); + defaultAcl = ImmutableList.copyOf(builder.defaultAcl); } public String id() { @@ -90,19 +132,31 @@ public Cors cors() { return cors; } - public Acl acl() { + public List acl() { return acl; } - public Acl defaultObjectAcl() { + public List defaultObjectAcl() { return defaultAcl; } public Builder toBuilder() { - return builder(id, name, createTime, metageneration).cors(cors).acl(acl).defaultAcl(defaultAcl); + return new Builder(id, name) + .createTime(createTime) + .etag(etag) + .metageneration(metageneration) + .cors(cors) + .acl(acl) + .defaultAcl(defaultAcl) + .location(location) + .storageClass(storageClass); + } + + void fromPb(Bucket bucket) { + } - public static Builder builder(String id, String name, long createTime, long metageneration) { - return new Builder(id, name, createTime, metageneration); + Bucket toPb() { + return n } } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 4c84d6f9d67b..8d45573e6916 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -16,6 +16,7 @@ package com.google.gcloud.storage; +import com.google.api.services.storage.model.Bucket; import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.gcloud.BaseService; @@ -33,7 +34,7 @@ final class StorageServiceImpl extends BaseService implem } @Override - public Iterable listBuckets() { + public Iterable listBuckets() { try { return Iterables.transform(storageRpc.buckets(), new Function() { @@ -47,7 +48,7 @@ public Iterable listBuckets() { } @Override - public Bucket getBucket(String bucket) { + public BucketInfo getBucket(String bucket) { try { return new BucketImpl(this, storageRpc.bucket(bucket)); } catch (IOException ex) { From 1188ac62c9f7c07561c7ee1b68c3bf57b33360da Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 1 Apr 2015 14:41:47 -0700 Subject: [PATCH 06/48] work in progress --- .../com/google/gcloud/storage/BucketInfo.java | 98 +++++++++++++++++-- .../java/com/google/gcloud/storage/Cors.java | 37 ++++--- .../google/gcloud/storage/ListOptions.java | 2 +- 3 files changed, 114 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java index d82229e4008d..ea79e78d325c 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -20,9 +20,12 @@ import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; +import java.io.Serializable; import java.util.List; -public final class BucketInfo { +public final class BucketInfo implements Serializable { + + private static final long serialVersionUID = -3946094202176916586L; private final String id; private final String name; @@ -32,12 +35,41 @@ public final class BucketInfo { private final Cors cors; private final ImmutableList acl; private final ImmutableList defaultAcl; - private final Location location; - private final StorageClass storageClass; + private final String location; + private final String storageClass; + + + public static final class StorageClass implements Serializable { + + private static final long serialVersionUID = 374002156285326563L; + + private final String value; + + public enum Type { + DURABLE_REDUCED_AVAILABILITY, + STANDARD, + OTHER + } + + private StorageClass(Type type) { + value = type.name(); + } - public enum StorageClass { - DURABLE_REDUCED_AVAILABILITY, - STANDARD + private StorageClass(String value) { + this.value = value; + } + + public static StorageClass standard() { + return new StorageClass(Type.STANDARD); + } + + public static StorageClass durableReducedAvailability() { + return new StorageClass(Type.DURABLE_REDUCED_AVAILABILITY); + } + + public static StorageClass other(String other) { + return new StorageClass(other); + } } public enum Location { @@ -48,8 +80,8 @@ public final static class Builder { private final String id; private final String name; - private StorageClass storageClass; - private Location location; + private String storageClass; + private String location; private String etag; private Long createTime; private Long metageneration; @@ -63,11 +95,19 @@ public final static class Builder { } public Builder storageClass(StorageClass storageClass) { + return storageClass(storageClass.name()); + } + + Builder storageClass(String storageClass) { this.storageClass = storageClass; return this; } public Builder location(Location location) { + return location(location.name()); + } + + Builder location(String location) { this.location = location; return this; } @@ -128,6 +168,26 @@ public String name() { return name; } + public String etag() { + return etag; + } + + public long createTime() { + return createTime; + } + + public long metageneration() { + return metageneration; + } + + public String lLocation() { + return location; + } + + public String storageClass() { + return storageClass; + } + public Cors cors() { return cors; } @@ -153,10 +213,30 @@ public Builder toBuilder() { } void fromPb(Bucket bucket) { + Builder builder = new Builder(bucket.getId(), bucket.getName()) + .createTime(bucket.getTimeCreated().getValue()) + .etag(bucket.getEtag()) + .metageneration(bucket.getMetageneration()) + .location(bucket.getLocation()) + .storageClass(bucket.getStorageClass()); + + cors = builder.cors; + acl = ImmutableList.copyOf(builder.acl); + defaultAcl = ImmutableList.copyOf(builder.defaultAcl); + } Bucket toPb() { - return n + id = builder.id; + name = builder.name; + etag = builder.etag; + createTime = MoreObjects.firstNonNull(builder.createTime, 0L); + metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L); + location = builder.location; + storageClass = builder.storageClass; + cors = builder.cors; + acl = ImmutableList.copyOf(builder.acl); + defaultAcl = ImmutableList.copyOf(builder.defaultAcl); } } diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java index c5a944cd6ac9..040985fd77e9 100644 --- a/src/main/java/com/google/gcloud/storage/Cors.java +++ b/src/main/java/com/google/gcloud/storage/Cors.java @@ -16,13 +16,19 @@ package com.google.gcloud.storage; +import com.google.api.services.storage.model.Bucket; +import com.google.common.base.Functions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import java.io.Serializable; import java.net.URI; import java.net.URISyntaxException; import java.util.List; -public final class Cors { +public final class Cors implements Serializable { + + private static final long serialVersionUID = -8637770919343335655L; private final Integer maxAgeSeconds; private final ImmutableList methods; @@ -33,7 +39,9 @@ public enum Method { ANY, GET, HEAD, PUT, POST, DELETE } - public static class Origin { + public static class Origin implements Serializable { + + private static final long serialVersionUID = -4447958124895577993L; private final URI uri; @@ -49,7 +57,6 @@ public Origin(String scheme, String host, int port) { } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } - } @Override @@ -61,8 +68,8 @@ public String toString() { public static final class Builder { private Integer maxAgeSeconds; - private ImmutableList methods; - private ImmutableList origins; + private ImmutableList methods; + private ImmutableList origins; private ImmutableList responseHeaders; private Builder() { @@ -73,22 +80,20 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) { return this; } - public Builder methods(List methods) { - this.methods = ImmutableList.copyOf(methods); - return this; + public Builder methods(Iterable methods) { + return methods(Iterables.transform(methods, Functions.toStringFunction())); } - public Builder methods(Method... methods) { + public Builder methods(Iterable methods) { this.methods = ImmutableList.copyOf(methods); return this; } - public Builder origins(List origins) { - this.origins = ImmutableList.copyOf(origins); - return this; + public Builder origins(Iterable origins) { + return origins(Iterables.transform(origins, Functions.toStringFunction())); } - public Builder origins(Origin... origins) { + public Builder origins(Iterable origins) { this.origins = ImmutableList.copyOf(origins); return this; } @@ -142,4 +147,10 @@ public Builder toBuilder() { public static Builder builder() { return new Builder(); } + + void fromPb(Bucket.Cors cors) { + Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds()); + cors.getMethod() + + } } diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java index 9ca3d9120ff1..2584a380ae40 100644 --- a/src/main/java/com/google/gcloud/storage/ListOptions.java +++ b/src/main/java/com/google/gcloud/storage/ListOptions.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private final boolean recurse; private final String prefix; private final String cursor; private final Integer maxResults; public static class Builder { private boolean recurse; private String prefix; private String cursor; private int maxResults; public Builder() { } public Builder recurse(boolean recurse) { this.recurse = recurse; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } } private ListOptions(Builder builder) { recurse = builder.recurse; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; } public boolean recurse() { return recurse; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; public Builder() { } public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } } \ No newline at end of file From c9611f21eb09abe166393d2d6278014e80055e20 Mon Sep 17 00:00:00 2001 From: ozarov Date: Wed, 1 Apr 2015 15:48:04 -0700 Subject: [PATCH 07/48] work in progress --- .../com/google/gcloud/storage/BucketInfo.java | 97 +++++++++++++------ 1 file changed, 67 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java index ea79e78d325c..59ecc73433af 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -16,6 +16,8 @@ package com.google.gcloud.storage; +import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull; + import com.google.api.services.storage.model.Bucket; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; @@ -35,8 +37,8 @@ public final class BucketInfo implements Serializable { private final Cors cors; private final ImmutableList acl; private final ImmutableList defaultAcl; - private final String location; - private final String storageClass; + private final Location location; + private final StorageClass storageClass; public static final class StorageClass implements Serializable { @@ -45,43 +47,84 @@ public static final class StorageClass implements Serializable { private final String value; - public enum Type { + public enum Options { DURABLE_REDUCED_AVAILABILITY, - STANDARD, - OTHER - } + STANDARD; - private StorageClass(Type type) { - value = type.name(); + private final StorageClass storageClass; + + Options() { + storageClass = new StorageClass(name()); + } } private StorageClass(String value) { - this.value = value; + this.value = checkNotNull(value); } public static StorageClass standard() { - return new StorageClass(Type.STANDARD); + return Options.STANDARD.storageClass; } public static StorageClass durableReducedAvailability() { - return new StorageClass(Type.DURABLE_REDUCED_AVAILABILITY); + return Options.DURABLE_REDUCED_AVAILABILITY.storageClass; } public static StorageClass other(String other) { return new StorageClass(other); } + + public String value() { + return value; + } } - public enum Location { - US, EU, ASIA + public static final class Location implements Serializable { + + private static final long serialVersionUID = 9073107666838637662L; + private final String value; + + public enum Options { + US, EU, ASIA; + + private final Location location; + + Options() { + location = new Location(name()); + } + } + + private Location(String value) { + this.value = checkNotNull(value); + } + + public static Location us() { + return Options.US.location; + } + + public static Location eu() { + return Options.EU.location; + } + + public static Location asia() { + return Options.ASIA.location; + } + + public static Location other(String other) { + return new Location(other); + } + + public String value() { + return value; + } } public final static class Builder { private final String id; private final String name; - private String storageClass; - private String location; + private StorageClass storageClass; + private Location location; private String etag; private Long createTime; private Long metageneration; @@ -95,19 +138,11 @@ public final static class Builder { } public Builder storageClass(StorageClass storageClass) { - return storageClass(storageClass.name()); - } - - Builder storageClass(String storageClass) { this.storageClass = storageClass; return this; } public Builder location(Location location) { - return location(location.name()); - } - - Builder location(String location) { this.location = location; return this; } @@ -180,11 +215,11 @@ public long metageneration() { return metageneration; } - public String lLocation() { + public Location location() { return location; } - public String storageClass() { + public StorageClass storageClass() { return storageClass; } @@ -216,17 +251,19 @@ void fromPb(Bucket bucket) { Builder builder = new Builder(bucket.getId(), bucket.getName()) .createTime(bucket.getTimeCreated().getValue()) .etag(bucket.getEtag()) - .metageneration(bucket.getMetageneration()) - .location(bucket.getLocation()) - .storageClass(bucket.getStorageClass()); + .metageneration(bucket.getMetageneration()); + //.location(bucket.getLocation()) + //.storageClass(bucket.getStorageClass()); + /* cors = builder.cors; acl = ImmutableList.copyOf(builder.acl); defaultAcl = ImmutableList.copyOf(builder.defaultAcl); - +*/ } + /* Bucket toPb() { id = builder.id; name = builder.name; @@ -238,5 +275,5 @@ Bucket toPb() { cors = builder.cors; acl = ImmutableList.copyOf(builder.acl); defaultAcl = ImmutableList.copyOf(builder.defaultAcl); - } + }*/ } From 43d0761936bd23f121b3e14fd6412609acdf5ab6 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 2 Apr 2015 14:28:15 -0700 Subject: [PATCH 08/48] work in progress --- .../com/google/gcloud/storage/BucketInfo.java | 54 +++++++++++----- .../java/com/google/gcloud/storage/Cors.java | 63 ++++++++++--------- 2 files changed, 72 insertions(+), 45 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java index 59ecc73433af..e59a604371fe 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -21,6 +21,7 @@ import com.google.api.services.storage.model.Bucket; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.io.Serializable; import java.util.List; @@ -44,34 +45,43 @@ public final class BucketInfo implements Serializable { public static final class StorageClass implements Serializable { private static final long serialVersionUID = 374002156285326563L; - + private static final ImmutableMap STRING_TO_OPTION; private final String value; - public enum Options { + public enum Option { DURABLE_REDUCED_AVAILABILITY, STANDARD; private final StorageClass storageClass; - Options() { + Option() { storageClass = new StorageClass(name()); } } + static { + ImmutableMap.Builder map = ImmutableMap.builder(); + for (Option option : Option.values()) { + map.put(option.name(), option); + } + STRING_TO_OPTION = map.build(); + } + private StorageClass(String value) { this.value = checkNotNull(value); } public static StorageClass standard() { - return Options.STANDARD.storageClass; + return Option.STANDARD.storageClass; } public static StorageClass durableReducedAvailability() { - return Options.DURABLE_REDUCED_AVAILABILITY.storageClass; + return Option.DURABLE_REDUCED_AVAILABILITY.storageClass; } - public static StorageClass other(String other) { - return new StorageClass(other); + public static StorageClass of(String value) { + Option option = STRING_TO_OPTION.get(value.toUpperCase()); + return option == null ? new StorageClass(value) : option.storageClass; } public String value() { @@ -82,36 +92,46 @@ public String value() { public static final class Location implements Serializable { private static final long serialVersionUID = 9073107666838637662L; + private static final ImmutableMap STRING_TO_OPTION; private final String value; - public enum Options { + public enum Option { US, EU, ASIA; private final Location location; - Options() { + Option() { location = new Location(name()); } } + static { + ImmutableMap.Builder map = ImmutableMap.builder(); + for (Option option : Option.values()) { + map.put(option.name(), option); + } + STRING_TO_OPTION = map.build(); + } + private Location(String value) { this.value = checkNotNull(value); } public static Location us() { - return Options.US.location; + return Option.US.location; } public static Location eu() { - return Options.EU.location; + return Option.EU.location; } public static Location asia() { - return Options.ASIA.location; + return Option.ASIA.location; } - public static Location other(String other) { - return new Location(other); + public static Location of(String value) { + Option option = STRING_TO_OPTION.get(value.toUpperCase()); + return option == null ? new Location(value) : option.location; } public String value() { @@ -251,9 +271,9 @@ void fromPb(Bucket bucket) { Builder builder = new Builder(bucket.getId(), bucket.getName()) .createTime(bucket.getTimeCreated().getValue()) .etag(bucket.getEtag()) - .metageneration(bucket.getMetageneration()); - //.location(bucket.getLocation()) - //.storageClass(bucket.getStorageClass()); + .metageneration(bucket.getMetageneration()) + .location(Location.of(bucket.getLocation())) + .storageClass(StorageClass.of(bucket.getStorageClass())); /* cors = builder.cors; diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java index 040985fd77e9..b8d45f12c501 100644 --- a/src/main/java/com/google/gcloud/storage/Cors.java +++ b/src/main/java/com/google/gcloud/storage/Cors.java @@ -16,8 +16,12 @@ package com.google.gcloud.storage; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.api.services.storage.model.Bucket; +import com.google.common.base.Function; import com.google.common.base.Functions; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -26,6 +30,8 @@ import java.net.URISyntaxException; import java.util.List; +import javax.annotation.Nullable; + public final class Cors implements Serializable { private static final long serialVersionUID = -8637770919343335655L; @@ -42,34 +48,44 @@ public enum Method { public static class Origin implements Serializable { private static final long serialVersionUID = -4447958124895577993L; + private static final String ANY_URI = "*"; + private final String value; - private final URI uri; + private static final Origin ANY = new Origin(ANY_URI); - public static final Origin ANY = new Origin(); + private Origin(String value) { + this.value = checkNotNull(value); + } - private Origin() { - uri = null; + public static Origin any() { + return ANY; } - public Origin(String scheme, String host, int port) { + public static Origin of(String scheme, String host, int port) { try { - this.uri = new URI(scheme, null, host, port, null, null, null); + of(new URI(scheme, null, host, port, null, null, null).toString()); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } } - @Override - public String toString() { - return uri == null ? "*" : uri.toString(); + public static Origin of(String value) { + if (ANY_URI.equals(value)) { + return any(); + } + return new Origin(value); + } + + public String value() { + return value; } } public static final class Builder { private Integer maxAgeSeconds; - private ImmutableList methods; - private ImmutableList origins; + private ImmutableList methods; + private ImmutableList origins; private ImmutableList responseHeaders; private Builder() { @@ -81,33 +97,20 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) { } public Builder methods(Iterable methods) { - return methods(Iterables.transform(methods, Functions.toStringFunction())); - } - - public Builder methods(Iterable methods) { this.methods = ImmutableList.copyOf(methods); return this; } public Builder origins(Iterable origins) { - return origins(Iterables.transform(origins, Functions.toStringFunction())); - } - - public Builder origins(Iterable origins) { this.origins = ImmutableList.copyOf(origins); return this; } - public Builder responseHeaders(List headers) { + public Builder responseHeaders(Iterable headers) { this.responseHeaders = ImmutableList.copyOf(headers); return this; } - public Builder responseHeaders(String... responseHeaders) { - this.responseHeaders = ImmutableList.copyOf(responseHeaders); - return this; - } - public Cors build() { return new Cors(this); } @@ -148,9 +151,13 @@ public static Builder builder() { return new Builder(); } - void fromPb(Bucket.Cors cors) { + Cors fromPb(Bucket.Cors cors) { Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds()); - cors.getMethod() - + builder.methods(Iterables.transform(cors.getMethod(), new Function() { + @Override public Method apply(String name) { + return Method.valueOf(name); + } + })); + builder } } From 7dcee4b7a136479b63bacf535a4c619823009c23 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 2 Apr 2015 17:40:57 -0700 Subject: [PATCH 09/48] wip --- .../java/com/google/gcloud/storage/Acl.java | 36 ++++++ .../com/google/gcloud/storage/BucketInfo.java | 106 +++++++++++++----- .../java/com/google/gcloud/storage/Cors.java | 42 +++++-- 3 files changed, 143 insertions(+), 41 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index 15d9444311d5..5e893b828b04 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -16,6 +16,10 @@ package com.google.gcloud.storage; +import com.google.api.services.storage.model.BucketAccessControl; +import com.google.api.services.storage.model.ObjectAccessControl; +import com.google.gcloud.storage.Acl.Project.ProjectRole; + import java.io.Serializable; public abstract class Acl implements Serializable { @@ -148,4 +152,36 @@ public Type type() { public Role role() { return role; } + + public static Acl fromPb(ObjectAccessControl objectAccessControl) { + Role role = Role.valueOf(objectAccessControl.getRole()); + return forEntity(objectAccessControl.getEntity(), role); + } + + private static Acl forEntity(String entity, Role role) { + if (entity.startsWith("user-")) { + return User.of(entity.substring(5), role); + } + if (entity.equals(User.ALL_USERS)) { + return User.ofAllUsers(role); + } + if (entity.equals(User.ALL_AUTHENTICATED_USERS)) { + return User.ofAllAuthenticatedUsers(role); + } + if (entity.startsWith("group-")) { + return Group.of(entity.substring(6), role); + } + if (entity.startsWith("project-")) { + int idx = entity.indexOf('-', 9); + String team = entity.substring(9, idx); + String projectId = entity.substring(idx + 1); + return Project.of(ProjectRole.valueOf(team.toUpperCase()), projectId, role); + } + return null; + } + + public static Acl fromPb(BucketAccessControl bucketAccessControl) { + Role role = Role.valueOf(bucketAccessControl.getRole()); + return forEntity(bucketAccessControl.getEntity(), role); + } } diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java index e59a604371fe..c12d54c1906a 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -17,11 +17,18 @@ package com.google.gcloud.storage; import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Iterables.transform; +import static com.google.common.collect.Lists.newArrayList; +import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.BucketAccessControl; +import com.google.api.services.storage.model.ObjectAccessControl; +import com.google.common.base.Function; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import java.io.Serializable; import java.util.List; @@ -35,7 +42,7 @@ public final class BucketInfo implements Serializable { private final String etag; private final long createTime; private final long metageneration; - private final Cors cors; + private final ImmutableList cors; private final ImmutableList acl; private final ImmutableList defaultAcl; private final Location location; @@ -84,6 +91,11 @@ public static StorageClass of(String value) { return option == null ? new StorageClass(value) : option.storageClass; } + @Override + public String toString() { + return value(); + } + public String value() { return value; } @@ -134,6 +146,11 @@ public static Location of(String value) { return option == null ? new Location(value) : option.location; } + @Override + public String toString() { + return value(); + } + public String value() { return value; } @@ -148,9 +165,9 @@ public final static class Builder { private String etag; private Long createTime; private Long metageneration; - private Cors cors; - private Iterable acl; - private Iterable defaultAcl; + private Iterable cors = ImmutableList.of(); + private Iterable acl = ImmutableList.of(); + private Iterable defaultAcl = ImmutableList.of(); Builder(String id, String name) { this.id = id; @@ -182,7 +199,7 @@ Builder metageneration(Long metageneration) { return this; } - Builder cors(Cors cors) { + Builder cors(Iterable cors) { this.cors = cors; return this; } @@ -210,7 +227,7 @@ private BucketInfo(Builder builder) { metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L); location = builder.location; storageClass = builder.storageClass; - cors = builder.cors; + cors = ImmutableList.copyOf(builder.cors); acl = ImmutableList.copyOf(builder.acl); defaultAcl = ImmutableList.copyOf(builder.defaultAcl); } @@ -243,7 +260,7 @@ public StorageClass storageClass() { return storageClass; } - public Cors cors() { + public List cors() { return cors; } @@ -251,7 +268,7 @@ public List acl() { return acl; } - public List defaultObjectAcl() { + public List defaultAcl() { return defaultAcl; } @@ -267,33 +284,64 @@ public Builder toBuilder() { .storageClass(storageClass); } - void fromPb(Bucket bucket) { + BucketInfo fromPb(Bucket bucket) { Builder builder = new Builder(bucket.getId(), bucket.getName()) .createTime(bucket.getTimeCreated().getValue()) .etag(bucket.getEtag()) .metageneration(bucket.getMetageneration()) .location(Location.of(bucket.getLocation())) - .storageClass(StorageClass.of(bucket.getStorageClass())); - - /* - cors = builder.cors; - acl = ImmutableList.copyOf(builder.acl); - defaultAcl = ImmutableList.copyOf(builder.defaultAcl); -*/ - + .storageClass(StorageClass.of(bucket.getStorageClass())) + .cors(transform(bucket.getCors(), new Function() { + @Override public Cors apply(Bucket.Cors cors) { + return Cors.fromPb(cors); + } + })) + .acl(transform(bucket.getAcl(), new Function() { + @Override public Acl apply(BucketAccessControl bucketAccessControl) { + return Acl.fromPb(bucketAccessControl); + } + })) + .defaultAcl(transform(bucket.getDefaultObjectAcl(), new Function() { + @Override public Acl apply(ObjectAccessControl objectAccessControl) { + return Acl.fromPb(objectAccessControl); + } + })); + return builder.build(); } - /* Bucket toPb() { - id = builder.id; - name = builder.name; - etag = builder.etag; - createTime = MoreObjects.firstNonNull(builder.createTime, 0L); - metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L); - location = builder.location; - storageClass = builder.storageClass; - cors = builder.cors; - acl = ImmutableList.copyOf(builder.acl); - defaultAcl = ImmutableList.copyOf(builder.defaultAcl); - }*/ + Bucket bucket = new Bucket(); + bucket.setId(id); + bucket.setName(name); + bucket.setEtag(etag); + if (createTime > 0) { + bucket.setTimeCreated(new DateTime(createTime)); + } + if (metageneration > 0) { + bucket.setMetageneration(metageneration); + } + if (location != null) { + bucket.setLocation(location.value()); + } + if (storageClass != null) { + bucket.setStorageClass(storageClass.value()); + } + bucket.setCors(newArrayList(Iterables.transform(cors, new Function() { + @Override public Bucket.Cors apply(Cors cors) { + return cors.toPb(); + } + }))); + bucket.setAcl(newArrayList(Iterables.transform(acl, new Function() { + @Override public BucketAccessControl apply(Acl acl) { + return new BucketAccessControl().setEntity(acl.toEntity()); + } + }))); + bucket.setDefaultObjectAcl( + newArrayList(Iterables.transform(defaultAcl, new Function() { + @Override public ObjectAccessControl apply(Acl acl) { + return new ObjectAccessControl().setEntity(acl.toEntity()); + } + }))); + return bucket; + } } diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java index b8d45f12c501..a58331d5063a 100644 --- a/src/main/java/com/google/gcloud/storage/Cors.java +++ b/src/main/java/com/google/gcloud/storage/Cors.java @@ -17,21 +17,19 @@ package com.google.gcloud.storage; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Iterables.transform; +import static com.google.common.collect.Lists.newArrayList; import com.google.api.services.storage.model.Bucket; import com.google.common.base.Function; import com.google.common.base.Functions; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import java.io.Serializable; import java.net.URI; import java.net.URISyntaxException; import java.util.List; -import javax.annotation.Nullable; - public final class Cors implements Serializable { private static final long serialVersionUID = -8637770919343335655L; @@ -63,7 +61,7 @@ public static Origin any() { public static Origin of(String scheme, String host, int port) { try { - of(new URI(scheme, null, host, port, null, null, null).toString()); + return of(new URI(scheme, null, host, port, null, null, null).toString()); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } @@ -76,6 +74,11 @@ public static Origin of(String value) { return new Origin(value); } + @Override + public String toString() { + return value(); + } + public String value() { return value; } @@ -151,13 +154,28 @@ public static Builder builder() { return new Builder(); } - Cors fromPb(Bucket.Cors cors) { + Bucket.Cors toPb() { + Bucket.Cors pb = new Bucket.Cors(); + pb.setMaxAgeSeconds(maxAgeSeconds); + pb.setResponseHeader(responseHeaders); + pb.setMethod(newArrayList(transform(methods(), Functions.toStringFunction()))); + pb.setOrigin(newArrayList(transform(origins(), Functions.toStringFunction()))); + return pb; + } + + static Cors fromPb(Bucket.Cors cors) { Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds()); - builder.methods(Iterables.transform(cors.getMethod(), new Function() { - @Override public Method apply(String name) { - return Method.valueOf(name); - } - })); - builder + builder.methods(transform(cors.getMethod(), new Function() { + @Override public Method apply(String name) { + return Method.valueOf(name.toUpperCase()); + } + })); + builder.origins(transform(cors.getOrigin(), new Function() { + @Override public Origin apply(String value) { + return Origin.of(value); + } + })); + builder.responseHeaders(cors.getResponseHeader()); + return builder.build(); } } From 59f0d8575d6ee8c59c944fa94fb0b6f27a99bcc6 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 9 Apr 2015 10:08:45 -0700 Subject: [PATCH 10/48] work in progress --- .../java/com/google/gcloud/storage/Acl.java | 52 ++++++++++++++++--- .../com/google/gcloud/storage/BucketInfo.java | 4 +- .../google/gcloud/storage/ListOptions.java | 2 +- .../com/google/gcloud/storage/ObjectInfo.java | 5 +- 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index 5e893b828b04..e4cb31995c03 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -56,6 +56,10 @@ public String domain() { return domain; } + String entity() { + return "domain-" + domain; + } + public static Domain of(String domain, Role role) { return new Domain(domain, role); } @@ -75,6 +79,10 @@ public String email() { return email; } + String entity() { + return "group-" + email; + } + public static Group of(String email, Role role) { return new Group(email, role); } @@ -97,6 +105,18 @@ public String email() { return email; } + String entity() { + switch (email) { + case ALL_AUTHENTICATED_USERS: + return ALL_AUTHENTICATED_USERS; + case ALL_USERS: + return ALL_USERS; + default: + return "user-" + email; + } + } + + public static User of(String email, Role role) { return new User(email, role); } @@ -135,6 +155,10 @@ public String projectId() { return projectId; } + String entity() { + return "project-" + pRole.name().toLowerCase() + "-" + projectId; + } + public static Project of(ProjectRole pRole, String projectId, Role role) { return new Project(pRole, projectId, role); } @@ -153,11 +177,32 @@ public Role role() { return role; } - public static Acl fromPb(ObjectAccessControl objectAccessControl) { + BucketAccessControl toBucketPb() { + BucketAccessControl bucketPb = new BucketAccessControl(); + bucketPb.setRole(role().toString()); + bucketPb.setEntity(entity()); + return bucketPb; + } + + ObjectAccessControl toObjectPb() { + ObjectAccessControl objectPb = new ObjectAccessControl(); + objectPb.setRole(role().toString()); + objectPb.setEntity(entity()); + return objectPb; + } + + abstract String entity(); + + static Acl fromPb(ObjectAccessControl objectAccessControl) { Role role = Role.valueOf(objectAccessControl.getRole()); return forEntity(objectAccessControl.getEntity(), role); } + static Acl fromPb(BucketAccessControl bucketAccessControl) { + Role role = Role.valueOf(bucketAccessControl.getRole()); + return forEntity(bucketAccessControl.getEntity(), role); + } + private static Acl forEntity(String entity, Role role) { if (entity.startsWith("user-")) { return User.of(entity.substring(5), role); @@ -179,9 +224,4 @@ private static Acl forEntity(String entity, Role role) { } return null; } - - public static Acl fromPb(BucketAccessControl bucketAccessControl) { - Role role = Role.valueOf(bucketAccessControl.getRole()); - return forEntity(bucketAccessControl.getEntity(), role); - } } diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java index c12d54c1906a..778d53330496 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -333,13 +333,13 @@ Bucket toPb() { }))); bucket.setAcl(newArrayList(Iterables.transform(acl, new Function() { @Override public BucketAccessControl apply(Acl acl) { - return new BucketAccessControl().setEntity(acl.toEntity()); + return acl.toBucketPb(); } }))); bucket.setDefaultObjectAcl( newArrayList(Iterables.transform(defaultAcl, new Function() { @Override public ObjectAccessControl apply(Acl acl) { - return new ObjectAccessControl().setEntity(acl.toEntity()); + return acl.toObjectPb(); } }))); return bucket; diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java index 2584a380ae40..b1b5d96f27da 100644 --- a/src/main/java/com/google/gcloud/storage/ListOptions.java +++ b/src/main/java/com/google/gcloud/storage/ListOptions.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; public Builder() { } public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; public Builder() { } public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/ObjectInfo.java index 97183bc720ea..47134d22ad21 100644 --- a/src/main/java/com/google/gcloud/storage/ObjectInfo.java +++ b/src/main/java/com/google/gcloud/storage/ObjectInfo.java @@ -21,10 +21,13 @@ import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; +import java.io.Serializable; import java.util.List; import java.util.Map; -public class ObjectInfo { +public class ObjectInfo implements Serializable { + + private static final long serialVersionUID = 2228487739943277159L; private final String bucket; private final String name; From 126b043ea9afefacc833ee5b1c17251684911422 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 9 Apr 2015 15:27:58 -0700 Subject: [PATCH 11/48] work in progress --- .../gcloud/examples/StorageExample.java | 123 ++++++++++++++++++ .../com/google/gcloud/storage/BucketInfo.java | 19 +++ .../com/google/gcloud/storage/ObjectInfo.java | 107 ++++++++++++--- .../google/gcloud/storage/StorageService.java | 23 +--- 4 files changed, 232 insertions(+), 40 deletions(-) create mode 100644 src/main/java/com/google/gcloud/examples/StorageExample.java diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java new file mode 100644 index 000000000000..87bf24d28772 --- /dev/null +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -0,0 +1,123 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.examples; + +import com.google.api.services.storage.Storage; +import com.google.gcloud.storage.BucketInfo; +import com.google.gcloud.storage.StorageService; +import com.google.gcloud.storage.StorageServiceFactory; +import com.google.gcloud.storage.StorageServiceOptions; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * An example of using the Google Cloud Storage. + *

+ * This example demonstrates a simple/typical usage. + *

+ * Steps needed for running the example:

    + *
  1. login using gcloud SDK - {@code gcloud auth login}.
    2. + *
  2. compile using maven - {@code mvn compile}
    3. + *
  3. run using maven - {@code mvn exec:java + * -Dexec.mainClass="com.google.gcloud.examples.StorageExample" + * -Dexec.args=" [] [list|get |put []|delete ...]"}
    4. + *
+ */ +public class StorageExample { + + private static final String DEFAULT_FOLDER = "_STORAGE_EXAMPLE"; + private static final String DEFAULT_ACTION = "list"; + private static final Map ACTIONS = new HashMap<>(); + + private static abstract class StorageAction { + + abstract void run(StorageService storage, BucketInfo bucket, String folder, String... args); + + protected String getRequiredParams() { + return ""; + } + + protected String fullPath(String folder, String file) { + StringBuilder stringBuilder = new StringBuilder(folder); + if (!folder.endsWith("/")) { + stringBuilder.append('/'); + } + return stringBuilder.append(file).toString(); + } + } + + + private static class DeleteAction extends StorageAction { + @Override + public void run(StorageService storage, BucketInfo bucket, String folder, String... args) { + for (String file : args) { + storage.delete(bucket.name(), fullPath(folder, file)); + } + } + + @Override + public String getRequiredParams() { + return "..."; + } + } + + static { + ACTIONS.put("delete", new DeleteAction()); + } + + public static void main(String... args) { + StorageAction action = null; + StorageService storage = null; + BucketInfo bucketInfo = null; + String folder = DEFAULT_FOLDER; + if (args.length > 0) { + String bucket = args[0]; + String actionName = DEFAULT_ACTION; + if (args.length > 1) { + folder = args[1]; + if (args.length > 2) { + actionName = args[2].toLowerCase(); + } + storage = StorageServiceFactory.getDefault(StorageServiceOptions.builder().build()); + bucketInfo = storage.get(bucket); + } + action = ACTIONS.get(actionName); + } + + if (action == null) { + StringBuilder actionAndParams = new StringBuilder(); + for (Map.Entry entry : ACTIONS.entrySet()) { + actionAndParams.append(entry.getKey()); + String param = entry.getValue().getRequiredParams(); + if (param != null && !param.isEmpty()) { + actionAndParams.append(' ').append(param); + } + actionAndParams.append('|'); + } + actionAndParams.setLength(actionAndParams.length() - 1); + System.out.printf("Usage: %s bucket [] [%s]%n", + StorageExample.class.getSimpleName(), actionAndParams); + return; + } + + args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{}; + action.run(bucketInfo, folder, args); + } + } +} diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java index 778d53330496..7f6de0300180 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -22,6 +22,7 @@ import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.Bucket.Lifecycle; import com.google.api.services.storage.model.BucketAccessControl; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.common.base.Function; @@ -39,6 +40,9 @@ public final class BucketInfo implements Serializable { private final String id; private final String name; + private final String owner; + private final String selfLink; + private final boolean versioningEnabled; private final String etag; private final long createTime; private final long metageneration; @@ -49,6 +53,15 @@ public final class BucketInfo implements Serializable { private final StorageClass storageClass; + public static abstract class AutoDeleteRule implements Serializable { + + public enum Option { + AGE, CREATE_BEFORE, VERSION_LIMIT, NOT_LIVE + } + + + } + public static final class StorageClass implements Serializable { private static final long serialVersionUID = 374002156285326563L; @@ -306,6 +319,12 @@ BucketInfo fromPb(Bucket bucket) { return Acl.fromPb(objectAccessControl); } })); + bucket.setOwner(); + bucket.setSelfLink(); + bucket.setWebsite(); + bucket.setVersioning(); + bucket.setLifecycle(new Lifecycle.Rule(). + return builder.build(); } diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/ObjectInfo.java index 47134d22ad21..05a0c901284a 100644 --- a/src/main/java/com/google/gcloud/storage/ObjectInfo.java +++ b/src/main/java/com/google/gcloud/storage/ObjectInfo.java @@ -16,12 +16,16 @@ package com.google.gcloud.storage; +import com.google.api.client.util.DateTime; +import com.google.api.services.storage.model.ObjectAccessControl; +import com.google.api.services.storage.model.StorageObject; +import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.hash.HashCode; -import com.google.common.hash.Hashing; +import com.google.common.collect.Lists; import java.io.Serializable; +import java.math.BigInteger; import java.util.List; import java.util.Map; @@ -37,8 +41,8 @@ public class ObjectInfo implements Serializable { private final String owner; private final long size; private final String contentEncoding; - private final HashCode md5; - private final long crc32; + private final String md5; + private final String crc32c; private final String mediaLink; private final ImmutableMap metadata; private final long generation; @@ -46,7 +50,6 @@ public class ObjectInfo implements Serializable { private final long deleteTime; private final long updateTime; - public static final class Builder { private String bucket; @@ -57,8 +60,8 @@ public static final class Builder { private String owner; private long size; private String contentEncoding; - private HashCode md5; - private long crc32; + private String md5; + private String crc32c; private String mediaLink; private ImmutableMap metadata; private long generation; @@ -109,18 +112,13 @@ public Builder contentEncoding(String contentEncoding) { return this; } - Builder md5(HashCode md5) { + Builder md5(String md5) { this.md5 = md5; return this; } - public Builder md5(byte[] bytes) { - this.md5 = bytes == null ? null : Hashing.md5().hashBytes(bytes); - return this; - } - - public Builder crc32(long crc32) { - this.crc32 = crc32; + public Builder crc32c(String crc32c) { + this.crc32c = crc32c; return this; } @@ -169,7 +167,7 @@ private ObjectInfo(Builder builder) { size = builder.size; contentEncoding = builder.contentEncoding; md5 = builder.md5; - crc32 = builder.crc32; + crc32c = builder.crc32c; mediaLink = builder.mediaLink; metadata = builder.metadata; generation = builder.generation; @@ -210,12 +208,12 @@ public String contentEncoding() { return contentEncoding; } - public byte[] md5() { - return md5 == null ? null : md5.asBytes(); + public String md5() { + return md5; } - public long crc32() { - return crc32; + public String crc32c() { + return crc32c; } public String mediaLink() { @@ -248,7 +246,7 @@ public Builder toBuilder() { .bucket(bucket) .cacheControl(cacheControl) .contentEncoding(contentEncoding) - .crc32(crc32) + .crc32c(crc32c) .contentType(contentType) .deleteTime(deleteTime) .generation(generation) @@ -266,4 +264,71 @@ public Builder toBuilder() { public static Builder builder() { return new Builder(); } + + StorageObject toPb() { + StorageObject storageObject = new StorageObject(); + storageObject.setAcl(Lists.transform(acl, + new Function() { + @Override + public ObjectAccessControl apply(Acl acl) { + return acl.toObjectPb(); + } + })); + storageObject.setBucket(bucket); + storageObject.setCacheControl(cacheControl); + storageObject.setContentEncoding(contentEncoding); + storageObject.setCrc32c(crc32c); + storageObject.setContentType(contentType); + storageObject.setTimeDeleted(new DateTime(deleteTime)); + storageObject.setGeneration(generation); + storageObject.setMd5Hash(md5); + storageObject.setMediaLink(mediaLink); + storageObject.setMetadata(metadata); + storageObject.setMetageneration(metageneration); + storageObject.setName(name); + storageObject.setOwner(owner); + storageObject.setUpdated(new DateTime(updateTime)); + storageObject.setSize(BigInteger.valueOf(size)); + storageObject.setContentDisposition(); + storageObject.setComponentCount(); + storageObject.setContentLanguage(); + storageObject.setEtag(); + storageObject.setId(); + storageObject.setSelfLink(); + storageObject.setStorageClass(); + return storageObject; + } + + static ObjectInfo fromPb(StorageObject storageObject) { + return builder() + .acl(Lists.transform(storageObject.getAcl(), + new Function() { + @Override public Acl apply(ObjectAccessControl objectAccessControl) { + return Acl.fromPb(objectAccessControl); + } + })) + .bucket(storageObject.getBucket()) + .cacheControl(storageObject.getCacheControl()) + .contentEncoding(storageObject.getContentEncoding()) + .crc32c(storageObject.getCrc32c()) + .contentType(storageObject.getContentType()) + .deleteTime(storageObject.getTimeDeleted().getValue()) + .generation(storageObject.getGeneration()) + .md5(storageObject.getMd5Hash()) + .mediaLink(storageObject.getMediaLink()) + .metadata(storageObject.getMetadata()) + .metageneration(storageObject.getMetageneration()) + .name(storageObject.getName()) + .owner(storageObject.getName()) + .updateTime(storageObject.getUpdated().getValue()) + .size(storageObject.getSize().longValue()) + .build(); + storageObject.setContentDisposition(); + storageObject.setComponentCount(); + storageObject.setContentLanguage(); + storageObject.setEtag(); + storageObject.setId(); + storageObject.setSelfLink(); + storageObject.setStorageClass(); + } } diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 48947537653b..f88f68cb331f 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -18,40 +18,25 @@ import com.google.gcloud.Service; -import java.util.Iterator; - public interface StorageService extends Service { - Iterable list(); - - Iterable list(ListOptions settings); - BucketInfo get(String bucket); - Iterator get(String... bucket); - ObjectInfo get(String bucket, String object); - Iterator get(String bucket, String... object); + Iterable list(ListOptions settings); void update(BucketInfo bucketInfo); void update(ObjectInfo objectInfo); - void delete(String bucket, String... object); + void delete(String bucket, String object); void compose(String bucket, Iterable sourceObjects, String destObject); void copy(String fromBucket, String fromObject, String destBucket, String destObject); + ObjectReadChannel newReader(String bucket, String ObjectName); - ObjectReadChannel getInputChannel(String bucket, String ObjectName); - - ObjectWriteChannel write(String ObjectName); - - // TODO: add listing - - ObjectReadChannel getInputChannel(); - - ObjectWriteChannel getOutputChannel(); + ObjectWriteChannel newWriter(String ObjectName); } From 4f65b2ddcfc6bfae3124f5b7acab75ac22694568 Mon Sep 17 00:00:00 2001 From: ozarov Date: Thu, 9 Apr 2015 18:18:11 -0700 Subject: [PATCH 12/48] work in progress --- .../java/com/google/gcloud/storage/Acl.java | 170 +++++++++--------- .../com/google/gcloud/storage/BucketInfo.java | 2 +- 2 files changed, 81 insertions(+), 91 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index e4cb31995c03..d2e2be0848f1 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -18,15 +18,14 @@ import com.google.api.services.storage.model.BucketAccessControl; import com.google.api.services.storage.model.ObjectAccessControl; -import com.google.gcloud.storage.Acl.Project.ProjectRole; import java.io.Serializable; -public abstract class Acl implements Serializable { +public final class Acl implements Serializable { private static final long serialVersionUID = 6435575339887912222L; - private final Type type; + private final Entity entity; private final Role role; public enum Role { @@ -35,105 +34,128 @@ public enum Role { WRITER } - public enum Type { + public enum EntityType { DOMAIN, GROUP, USER, PROJECT } - public static class Domain extends Acl { + public static abstract class Entity implements Serializable { - private static final long serialVersionUID = -3033025857280447253L; - private final String domain; + private static final long serialVersionUID = -2707407252771255840L; + private final EntityType type; + private final String value; - Domain(String domain, Role role) { - super(Type.DOMAIN, role); - this.domain = domain; + Entity(EntityType type, String value) { + this.type = type; + this.value = value; } - public String domain() { - return domain; + public EntityType type() { + return type; } - String entity() { - return "domain-" + domain; + protected String value() { + return value; } - public static Domain of(String domain, Role role) { - return new Domain(domain, role); + @Override + public String toString() { + return type.name().toLowerCase() + "-" + value(); + } + + static Entity fromPb(String entity) { + if (entity.startsWith("user-")) { + return new User(entity.substring(5)); + } + if (entity.equals(User.ALL_USERS)) { + return User.ofAllUsers(); + } + if (entity.equals(User.ALL_AUTHENTICATED_USERS)) { + return User.ofAllAuthenticatedUsers(); + } + if (entity.startsWith("group-")) { + return new Group(entity.substring(6)); + } + if (entity.startsWith("project-")) { + int idx = entity.indexOf('-', 9); + String team = entity.substring(9, idx); + String projectId = entity.substring(idx + 1); + return new Project(Project.ProjectRole.valueOf(team.toUpperCase()), projectId); + } + return null; } } - public static class Group extends Acl { + public static class Domain extends Entity { - private static final long serialVersionUID = -1660987136294408826L; - private final String email; + private static final long serialVersionUID = -3033025857280447253L; + + public Domain(String domain) { + super(EntityType.DOMAIN, domain); - Group(String email, Role role) { - super(Type.GROUP, role); - this.email = email; } - public String email() { - return email; + public String domain() { + return value(); } + } + + public static class Group extends Entity { + + private static final long serialVersionUID = -1660987136294408826L; - String entity() { - return "group-" + email; + public Group(String email) { + super(EntityType.GROUP, email); } - public static Group of(String email, Role role) { - return new Group(email, role); + public String email() { + return value(); } } - public static class User extends Acl { + public static class User extends Entity { private static final long serialVersionUID = 3076518036392737008L; private static final String ALL_USERS = "allUsers"; private static final String ALL_AUTHENTICATED_USERS = "allAuthenticatedUsers"; - private final String email; - - User(String email, Role role) { - super(Type.USER, role); - this.email = email; + public User(String email) { + super(EntityType.USER, email); } public String email() { - return email; + return value(); } - String entity() { - switch (email) { + @Override + public String toString() { + switch (value()) { case ALL_AUTHENTICATED_USERS: return ALL_AUTHENTICATED_USERS; case ALL_USERS: return ALL_USERS; default: - return "user-" + email; + break; } + return super.toString(); } - public static User of(String email, Role role) { - return new User(email, role); + public static User ofAllUsers() { + return new User(ALL_USERS); } - public static User ofAllUsers(Role role) { - return of(ALL_USERS, role); - } - - public static User ofAllAuthenticatedUsers(Role role) { - return of(ALL_AUTHENTICATED_USERS, role); + public static User ofAllAuthenticatedUsers() { + return new User(ALL_AUTHENTICATED_USERS); } } - public static class Project extends Acl { + public static class Project extends Entity { - private final String projectId; private final ProjectRole pRole; + private final String projectId; enum ProjectRole { OWNERS, @@ -141,8 +163,8 @@ enum ProjectRole { VIEWERS } - Project(ProjectRole pRole, String projectId, Role role) { - super(Type.PROJECT, role); + Project(ProjectRole pRole, String projectId) { + super(EntityType.PROJECT, pRole.name().toLowerCase() + "-" + projectId); this.pRole = pRole; this.projectId = projectId; } @@ -154,23 +176,15 @@ public ProjectRole projectRole() { public String projectId() { return projectId; } - - String entity() { - return "project-" + pRole.name().toLowerCase() + "-" + projectId; - } - - public static Project of(ProjectRole pRole, String projectId, Role role) { - return new Project(pRole, projectId, role); - } } - Acl(Type type, Role role) { - this.type = type; + public Acl(Entity entity, Role role) { + this.entity = entity; this.role = role; } - public Type type() { - return type; + public Entity entity() { + return entity; } public Role role() { @@ -180,48 +194,24 @@ public Role role() { BucketAccessControl toBucketPb() { BucketAccessControl bucketPb = new BucketAccessControl(); bucketPb.setRole(role().toString()); - bucketPb.setEntity(entity()); + bucketPb.setEntity(entity().toString()); return bucketPb; } ObjectAccessControl toObjectPb() { ObjectAccessControl objectPb = new ObjectAccessControl(); objectPb.setRole(role().toString()); - objectPb.setEntity(entity()); + objectPb.setEntity(entity().toString()); return objectPb; } - abstract String entity(); - static Acl fromPb(ObjectAccessControl objectAccessControl) { Role role = Role.valueOf(objectAccessControl.getRole()); - return forEntity(objectAccessControl.getEntity(), role); + return new Acl(Entity.fromPb(objectAccessControl.getEntity()), role); } static Acl fromPb(BucketAccessControl bucketAccessControl) { Role role = Role.valueOf(bucketAccessControl.getRole()); - return forEntity(bucketAccessControl.getEntity(), role); - } - - private static Acl forEntity(String entity, Role role) { - if (entity.startsWith("user-")) { - return User.of(entity.substring(5), role); - } - if (entity.equals(User.ALL_USERS)) { - return User.ofAllUsers(role); - } - if (entity.equals(User.ALL_AUTHENTICATED_USERS)) { - return User.ofAllAuthenticatedUsers(role); - } - if (entity.startsWith("group-")) { - return Group.of(entity.substring(6), role); - } - if (entity.startsWith("project-")) { - int idx = entity.indexOf('-', 9); - String team = entity.substring(9, idx); - String projectId = entity.substring(idx + 1); - return Project.of(ProjectRole.valueOf(team.toUpperCase()), projectId, role); - } - return null; + return new Acl(Entity.fromPb(bucketAccessControl.getEntity()), role); } } diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java index 7f6de0300180..085317e41961 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -40,7 +40,7 @@ public final class BucketInfo implements Serializable { private final String id; private final String name; - private final String owner; + private final Acl.Entity owner; private final String selfLink; private final boolean versioningEnabled; private final String etag; From a3cf9070aa1bb1dd5a956b2975d7c19dbd6ad122 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 10 Apr 2015 16:23:56 -0700 Subject: [PATCH 13/48] work in progress --- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../java/com/google/gcloud/storage/Acl.java | 56 ++-- .../com/google/gcloud/storage/BucketInfo.java | 290 ++++++++++++++++-- 3 files changed, 304 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index c53574986c4b..d4e6a245dea5 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import java.io.IOException; import java.util.List; public interface StorageRpc { List buckets() throws IOException; Bucket bucket(String name) throws IOException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import java.io.IOException; import java.util.List; public interface StorageRpc { List buckets() throws IOException; Bucket bucket(String name) throws IOException; void patch(Bucket bucket) throws IOException; void patch(StorageObject storageObject) throws IOException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index d2e2be0848f1..48bd6c787718 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -34,25 +34,26 @@ public enum Role { WRITER } - public enum EntityType { - DOMAIN, - GROUP, - USER, - PROJECT - } - public static abstract class Entity implements Serializable { private static final long serialVersionUID = -2707407252771255840L; - private final EntityType type; + private final Type type; private final String value; - Entity(EntityType type, String value) { + public enum Type { + DOMAIN, + GROUP, + USER, + PROJECT, + UNKNOWN + } + + Entity(Type type, String value) { this.type = type; this.value = value; } - public EntityType type() { + public Type type() { return type; } @@ -62,6 +63,10 @@ protected String value() { @Override public String toString() { + return toPb(); + } + + String toPb() { return type.name().toLowerCase() + "-" + value(); } @@ -84,7 +89,7 @@ static Entity fromPb(String entity) { String projectId = entity.substring(idx + 1); return new Project(Project.ProjectRole.valueOf(team.toUpperCase()), projectId); } - return null; + return new RawEntity(entity); } } @@ -93,7 +98,7 @@ public static class Domain extends Entity { private static final long serialVersionUID = -3033025857280447253L; public Domain(String domain) { - super(EntityType.DOMAIN, domain); + super(Type.DOMAIN, domain); } @@ -107,7 +112,7 @@ public static class Group extends Entity { private static final long serialVersionUID = -1660987136294408826L; public Group(String email) { - super(EntityType.GROUP, email); + super(Type.GROUP, email); } public String email() { @@ -122,7 +127,7 @@ public static class User extends Entity { private static final String ALL_AUTHENTICATED_USERS = "allAuthenticatedUsers"; public User(String email) { - super(EntityType.USER, email); + super(Type.USER, email); } public String email() { @@ -130,7 +135,7 @@ public String email() { } @Override - public String toString() { + String toPb() { switch (value()) { case ALL_AUTHENTICATED_USERS: return ALL_AUTHENTICATED_USERS; @@ -139,10 +144,9 @@ public String toString() { default: break; } - return super.toString(); + return super.toPb(); } - public static User ofAllUsers() { return new User(ALL_USERS); } @@ -164,7 +168,7 @@ enum ProjectRole { } Project(ProjectRole pRole, String projectId) { - super(EntityType.PROJECT, pRole.name().toLowerCase() + "-" + projectId); + super(Type.PROJECT, pRole.name().toLowerCase() + "-" + projectId); this.pRole = pRole; this.projectId = projectId; } @@ -178,6 +182,18 @@ public String projectId() { } } + public static class RawEntity extends Entity { + + RawEntity(String entity) { + super(Type.UNKNOWN, entity); + } + + @Override + String toPb() { + return value(); + } + } + public Acl(Entity entity, Role role) { this.entity = entity; this.role = role; @@ -200,8 +216,8 @@ BucketAccessControl toBucketPb() { ObjectAccessControl toObjectPb() { ObjectAccessControl objectPb = new ObjectAccessControl(); - objectPb.setRole(role().toString()); - objectPb.setEntity(entity().toString()); + objectPb.setRole(role().name()); + objectPb.setEntity(entity().toPb()); return objectPb; } diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java index 085317e41961..97f1bea1e244 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -18,18 +18,19 @@ import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Iterables.transform; -import static com.google.common.collect.Lists.newArrayList; import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.Bucket; -import com.google.api.services.storage.model.Bucket.Lifecycle; +import com.google.api.services.storage.model.Bucket.Lifecycle.Rule; +import com.google.api.services.storage.model.Bucket.Versioning; import com.google.api.services.storage.model.BucketAccessControl; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.common.base.Function; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.gcloud.storage.Acl.Entity; import java.io.Serializable; import java.util.List; @@ -43,6 +44,9 @@ public final class BucketInfo implements Serializable { private final Acl.Entity owner; private final String selfLink; private final boolean versioningEnabled; + private final String indexPage; + private final String notFoundPage; + private final ImmutableList deleteRules; private final String etag; private final long createTime; private final long metageneration; @@ -53,13 +57,151 @@ public final class BucketInfo implements Serializable { private final StorageClass storageClass; - public static abstract class AutoDeleteRule implements Serializable { + public static abstract class DeleteRule implements Serializable { - public enum Option { - AGE, CREATE_BEFORE, VERSION_LIMIT, NOT_LIVE + private static final long serialVersionUID = 3137971668395933033L; + private final Type type; + private static final String SUPPORTED_ACTION = "Delete"; + + public enum Type { + AGE, CREATE_BEFORE, NUM_NEWER_VERSIONS, IS_LIVE, UNKNOWN + } + + DeleteRule(Type type) { + this.type = type; + } + + public Type type() { + return type; + } + + Rule toPb() { + Rule rule = new Rule(); + rule.setAction(new Rule.Action().setType(SUPPORTED_ACTION)); + Rule.Condition condition = new Rule.Condition(); + populateCondition(condition); + rule.setCondition(condition); + return rule; + } + + abstract void populateCondition(Rule.Condition condition); + + static DeleteRule fromPb(Rule rule) { + if (rule.getAction() != null && SUPPORTED_ACTION.endsWith(rule.getAction().getType())) { + Rule.Condition condition = rule.getCondition(); + Integer age = condition.getAge(); + if (age != null) { + return new AgeDeleteRule(age); + } + DateTime dateTime = condition.getCreatedBefore(); + if (dateTime != null) { + return new CreatedBeforeDeleteRule(dateTime.getValue()); + } + Integer numNewerVersions = condition.getNumNewerVersions(); + if (numNewerVersions != null) { + return new NumNewerVersionsDeleteRule(numNewerVersions); + } + Boolean isLive = condition.getIsLive(); + if (isLive != null) { + return new IsLiveDeleteRule(isLive); + } + } + return new RawDeleteRule(rule); + } + } + + public static class AgeDeleteRule extends DeleteRule { + + private static final long serialVersionUID = 5697166940712116380L; + private final int daysToLive; + + public AgeDeleteRule(int daysToLive) { + super(Type.AGE); + this.daysToLive = daysToLive; + } + + public int daysToLive() { + return daysToLive; + } + + void populateCondition(Rule.Condition condition) { + condition.setAge(daysToLive); + } + } + + static class RawDeleteRule extends DeleteRule { + + private final Rule rule; + + RawDeleteRule(Rule rule) { + super(Type.UNKNOWN); + this.rule = rule; + } + + void populateCondition(Rule.Condition condition) { + throw new UnsupportedOperationException(); + } + + Rule toPb() { + return rule; } + } + + public static class CreatedBeforeDeleteRule extends DeleteRule { + private static final long serialVersionUID = 881692650279195867L; + private final long timeMillis; + public CreatedBeforeDeleteRule(long timeMillis) { + super(Type.CREATE_BEFORE); + this.timeMillis = timeMillis; + } + + public long timeMillis() { + return timeMillis; + } + + void populateCondition(Rule.Condition condition) { + condition.setCreatedBefore(new DateTime(timeMillis)); + } + } + + public static class NumNewerVersionsDeleteRule extends DeleteRule { + + private static final long serialVersionUID = -1955554976528303894L; + private final int numNewerVersions; + + public NumNewerVersionsDeleteRule(int numNewerVersions) { + super(Type.NUM_NEWER_VERSIONS); + this.numNewerVersions = numNewerVersions; + } + + public int numNewerVersions() { + return numNewerVersions; + } + + void populateCondition(Rule.Condition condition) { + condition.setNumNewerVersions(numNewerVersions); + } + } + + public static class IsLiveDeleteRule extends DeleteRule { + + private static final long serialVersionUID = -3502994563121313364L; + private final boolean isLive; + + public IsLiveDeleteRule(boolean isLive) { + super(Type.IS_LIVE); + this.isLive = isLive; + } + + public boolean isLive() { + return isLive; + } + + void populateCondition(Rule.Condition condition) { + condition.setIsLive(isLive); + } } public static final class StorageClass implements Serializable { @@ -173,6 +315,12 @@ public final static class Builder { private final String id; private final String name; + private Acl.Entity owner; + private String selfLink; + private boolean versioningEnabled; + private String indexPage; + private String notFoundPage; + private ImmutableList deleteRules = ImmutableList.of(); private StorageClass storageClass; private Location location; private String etag; @@ -187,6 +335,36 @@ public final static class Builder { this.name = name; } + Builder owner(Acl.Entity owner) { + this.owner = owner; + return this; + } + + Builder selfLink(String selfLink) { + this.selfLink = selfLink; + return this; + } + + public Builder versioningEnabled(boolean enable) { + this.versioningEnabled = enable; + return this; + } + + public Builder indexPage(String indexPage) { + this.indexPage = indexPage; + return this; + } + + public Builder notFoundPage(String notFoundPage) { + this.notFoundPage = notFoundPage; + return this; + } + + public Builder deleteRules(Iterable rules) { + this.deleteRules = ImmutableList.copyOf(rules); + return this; + } + public Builder storageClass(StorageClass storageClass) { this.storageClass = storageClass; return this; @@ -243,6 +421,12 @@ private BucketInfo(Builder builder) { cors = ImmutableList.copyOf(builder.cors); acl = ImmutableList.copyOf(builder.acl); defaultAcl = ImmutableList.copyOf(builder.defaultAcl); + owner = builder.owner; + selfLink = builder.selfLink; + versioningEnabled = builder.versioningEnabled; + indexPage = builder.indexPage; + notFoundPage = builder.notFoundPage; + deleteRules = ImmutableList.copyOf(builder.deleteRules); } public String id() { @@ -253,6 +437,30 @@ public String name() { return name; } + public Entity Owner() { + return owner; + } + + public String selfLink() { + return selfLink; + } + + public boolean versioningEnabled() { + return versioningEnabled; + } + + public String indexPage() { + return indexPage; + } + + public String notFoundPage() { + return notFoundPage; + } + + public ImmutableList deleteRules() { + return deleteRules; + } + public String etag() { return etag; } @@ -294,7 +502,13 @@ public Builder toBuilder() { .acl(acl) .defaultAcl(defaultAcl) .location(location) - .storageClass(storageClass); + .storageClass(storageClass) + .owner(owner) + .selfLink(selfLink) + .versioningEnabled(versioningEnabled) + .indexPage(indexPage) + .notFoundPage(notFoundPage) + .deleteRules(deleteRules); } BucketInfo fromPb(Bucket bucket) { @@ -318,13 +532,27 @@ BucketInfo fromPb(Bucket bucket) { @Override public Acl apply(ObjectAccessControl objectAccessControl) { return Acl.fromPb(objectAccessControl); } - })); - bucket.setOwner(); - bucket.setSelfLink(); - bucket.setWebsite(); - bucket.setVersioning(); - bucket.setLifecycle(new Lifecycle.Rule(). - + })) + .owner(Entity.fromPb(bucket.getOwner().getEntity())) + .selfLink(bucket.getSelfLink()); + Bucket.Versioning versioning = bucket.getVersioning(); + if (versioning != null) { + builder.versioningEnabled(MoreObjects.firstNonNull(versioning.getEnabled(), Boolean.FALSE)); + } + Bucket.Website website = bucket.getWebsite(); + if (website != null) { + builder.indexPage(website.getMainPageSuffix()); + builder.notFoundPage(website.getNotFoundPage()); + } + Bucket.Lifecycle lifecycle = bucket.getLifecycle(); + if (lifecycle != null) { + builder.deleteRules(transform(lifecycle.getRule(), + new Function() { + @Override public DeleteRule apply(Rule rule) { + return DeleteRule.fromPb(rule); + } + })); + } return builder.build(); } @@ -345,22 +573,38 @@ Bucket toPb() { if (storageClass != null) { bucket.setStorageClass(storageClass.value()); } - bucket.setCors(newArrayList(Iterables.transform(cors, new Function() { - @Override public Bucket.Cors apply(Cors cors) { + bucket.setCors(Lists.transform(cors, new Function() { + @Override + public Bucket.Cors apply(Cors cors) { return cors.toPb(); } - }))); - bucket.setAcl(newArrayList(Iterables.transform(acl, new Function() { - @Override public BucketAccessControl apply(Acl acl) { + })); + bucket.setAcl(Lists.transform(acl, new Function() { + @Override + public BucketAccessControl apply(Acl acl) { return acl.toBucketPb(); } - }))); + })); bucket.setDefaultObjectAcl( - newArrayList(Iterables.transform(defaultAcl, new Function() { - @Override public ObjectAccessControl apply(Acl acl) { + Lists.transform(defaultAcl, new Function() { + @Override + public ObjectAccessControl apply(Acl acl) { return acl.toObjectPb(); } - }))); + })); + bucket.setOwner(new Bucket.Owner().setEntity(owner.toPb())); + bucket.setSelfLink(selfLink); + bucket.setVersioning(new Versioning().setEnabled(versioningEnabled)); + Bucket.Website website = new Bucket.Website(); + website.setMainPageSuffix(indexPage); + website.setNotFoundPage(notFoundPage); + bucket.setWebsite(website); + Bucket.Lifecycle lifecycle = new Bucket.Lifecycle(); + lifecycle.setRule(Lists.transform(deleteRules, new Function() { + @Override public Rule apply(DeleteRule deleteRule) { + return deleteRule.toPb(); + } + })); return bucket; } } From 2cdc4e3dc02f029bf29b011f249013c4c70e339c Mon Sep 17 00:00:00 2001 From: ozarov Date: Sat, 11 Apr 2015 12:42:21 -0700 Subject: [PATCH 14/48] work in progress --- .../com/google/gcloud/storage/ObjectInfo.java | 141 ++++++++++++++---- 1 file changed, 108 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/ObjectInfo.java index 05a0c901284a..2e96a9163e93 100644 --- a/src/main/java/com/google/gcloud/storage/ObjectInfo.java +++ b/src/main/java/com/google/gcloud/storage/ObjectInfo.java @@ -34,13 +34,14 @@ public class ObjectInfo implements Serializable { private static final long serialVersionUID = 2228487739943277159L; private final String bucket; + private final String id; private final String name; - private final String contentType; + private final String selfLink; private final String cacheControl; private final ImmutableList acl; - private final String owner; + private final Acl.Entity owner; private final long size; - private final String contentEncoding; + private final String etag; private final String md5; private final String crc32c; private final String mediaLink; @@ -49,17 +50,28 @@ public class ObjectInfo implements Serializable { private final long metageneration; private final long deleteTime; private final long updateTime; + private final String contentType; + private final String contentEncoding; + private final String contentDisposition; + private final String contentLanguage; + private final int componentCount; public static final class Builder { private String bucket; + private String id; private String name; private String contentType; + private String contentEncoding; + private String contentDisposition; + private String contentLanguage; + private int componentCount; private String cacheControl; private ImmutableList acl; - private String owner; + private Acl.Entity owner; private long size; - private String contentEncoding; + private String etag; + private String selfLink; private String md5; private String crc32c; private String mediaLink; @@ -77,6 +89,11 @@ public Builder bucket(String bucket) { return this; } + Builder id(String id) { + this.id = id; + return this; + } + public Builder name(String name) { this.name = name; return this; @@ -87,6 +104,26 @@ public Builder contentType(String contentType) { return this; } + Builder contentDisposition(String contentDisposition) { + this.contentDisposition = contentDisposition; + return this; + } + + Builder contentLanguage(String contentLanguage) { + this.contentLanguage = contentLanguage; + return this; + } + + public Builder contentEncoding(String contentEncoding) { + this.contentEncoding = contentEncoding; + return this; + } + + Builder componentCount(int componentCount) { + this.componentCount = componentCount; + return this; + } + public Builder cacheControl(String cacheControl) { this.cacheControl = cacheControl; return this; @@ -97,7 +134,7 @@ public Builder acl(List acl) { return this; } - public Builder owner(String owner) { + public Builder owner(Acl.Entity owner) { this.owner = owner; return this; } @@ -107,8 +144,13 @@ public Builder size(long size) { return this; } - public Builder contentEncoding(String contentEncoding) { - this.contentEncoding = contentEncoding; + Builder etag(String etag) { + this.etag = etag; + return this; + } + + Builder selfLink(String selfLink) { + this.selfLink = selfLink; return this; } @@ -159,13 +201,19 @@ public ObjectInfo build() { private ObjectInfo(Builder builder) { bucket = builder.bucket; + id = builder.id; name = builder.name; - contentType = builder.contentType; cacheControl = builder.cacheControl; + contentEncoding = builder.contentEncoding; + contentType = builder.contentType; + contentDisposition = builder.contentDisposition; + contentLanguage = builder.contentLanguage; + componentCount = builder.componentCount; acl = builder.acl; owner = builder.owner; size = builder.size; - contentEncoding = builder.contentEncoding; + etag = builder.etag; + selfLink = builder.selfLink; md5 = builder.md5; crc32c = builder.crc32c; mediaLink = builder.mediaLink; @@ -180,12 +228,12 @@ public String bucket() { return bucket; } - public String name() { - return name; + public String id() { + return id; } - public String contentType() { - return contentType; + public String name() { + return name; } public String cacheControl() { @@ -196,7 +244,7 @@ public List acl() { return acl; } - public String owner() { + public Acl.Entity owner() { return owner; } @@ -204,10 +252,34 @@ public long size() { return size; } + public String contentType() { + return contentType; + } + public String contentEncoding() { return contentEncoding; } + public String contentDisposition() { + return contentDisposition; + } + + public String contentLanguage() { + return contentEncoding; + } + + public int componentCount() { + return componentCount; + } + + public String etag() { + return etag; + } + + public String selfLink() { + return selfLink; + } + public String md5() { return md5; } @@ -257,8 +329,13 @@ public Builder toBuilder() { .name(name) .owner(owner) .updateTime(updateTime) - .size(size); - + .size(size) + .contentDisposition(contentDisposition) + .componentCount(componentCount) + .contentLanguage(contentLanguage) + .etag(etag) + .id(id) + .selfLink(selfLink); } public static Builder builder() { @@ -286,16 +363,15 @@ public ObjectAccessControl apply(Acl acl) { storageObject.setMetadata(metadata); storageObject.setMetageneration(metageneration); storageObject.setName(name); - storageObject.setOwner(owner); + storageObject.setOwner(new StorageObject.Owner().setEntity(owner.toPb())); storageObject.setUpdated(new DateTime(updateTime)); storageObject.setSize(BigInteger.valueOf(size)); - storageObject.setContentDisposition(); - storageObject.setComponentCount(); - storageObject.setContentLanguage(); - storageObject.setEtag(); - storageObject.setId(); - storageObject.setSelfLink(); - storageObject.setStorageClass(); + storageObject.setContentDisposition(contentDisposition); + storageObject.setComponentCount(componentCount); + storageObject.setContentLanguage(contentLanguage); + storageObject.setEtag(etag); + storageObject.setId(id); + storageObject.setSelfLink(selfLink); return storageObject; } @@ -319,16 +395,15 @@ static ObjectInfo fromPb(StorageObject storageObject) { .metadata(storageObject.getMetadata()) .metageneration(storageObject.getMetageneration()) .name(storageObject.getName()) - .owner(storageObject.getName()) + .owner(Acl.Entity.fromPb(storageObject.getOwner().getEntity())) .updateTime(storageObject.getUpdated().getValue()) .size(storageObject.getSize().longValue()) + .contentDisposition(storageObject.getContentDisposition()) + .componentCount(storageObject.getComponentCount()) + .contentLanguage(storageObject.getContentLanguage()) + .etag(storageObject.getEtag()) + .id(storageObject.getId()) + .selfLink(storageObject.getSelfLink()) .build(); - storageObject.setContentDisposition(); - storageObject.setComponentCount(); - storageObject.setContentLanguage(); - storageObject.setEtag(); - storageObject.setId(); - storageObject.setSelfLink(); - storageObject.setStorageClass(); } } From 24f8a0b9187bd9c6f345c0e03a66d00c00125e7b Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 13 Apr 2015 17:41:04 -0700 Subject: [PATCH 15/48] work in progress --- .../gcloud/examples/StorageExample.java | 16 +- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../storage/{ObjectInfo.java => Blob.java} | 16 +- .../storage/{BucketInfo.java => Bucket.java} | 169 +++++++++--------- .../gcloud/storage/ObjectWriteChannel.java | 2 + .../google/gcloud/storage/StorageService.java | 23 +-- .../gcloud/storage/StorageServiceImpl.java | 58 ++++-- 8 files changed, 164 insertions(+), 124 deletions(-) rename src/main/java/com/google/gcloud/storage/{ObjectInfo.java => Blob.java} (96%) rename src/main/java/com/google/gcloud/storage/{BucketInfo.java => Bucket.java} (80%) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 87bf24d28772..ebfbeef3545f 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -16,8 +16,7 @@ package com.google.gcloud.examples; -import com.google.api.services.storage.Storage; -import com.google.gcloud.storage.BucketInfo; +import com.google.gcloud.storage.Bucket; import com.google.gcloud.storage.StorageService; import com.google.gcloud.storage.StorageServiceFactory; import com.google.gcloud.storage.StorageServiceOptions; @@ -47,7 +46,7 @@ public class StorageExample { private static abstract class StorageAction { - abstract void run(StorageService storage, BucketInfo bucket, String folder, String... args); + abstract void run(StorageService storage, Bucket bucket, String folder, String... args); protected String getRequiredParams() { return ""; @@ -65,7 +64,7 @@ protected String fullPath(String folder, String file) { private static class DeleteAction extends StorageAction { @Override - public void run(StorageService storage, BucketInfo bucket, String folder, String... args) { + public void run(StorageService storage, Bucket bucket, String folder, String... args) { for (String file : args) { storage.delete(bucket.name(), fullPath(folder, file)); } @@ -84,10 +83,10 @@ public String getRequiredParams() { public static void main(String... args) { StorageAction action = null; StorageService storage = null; - BucketInfo bucketInfo = null; + Bucket bucket = null; String folder = DEFAULT_FOLDER; if (args.length > 0) { - String bucket = args[0]; + String bucketName = args[0]; String actionName = DEFAULT_ACTION; if (args.length > 1) { folder = args[1]; @@ -95,7 +94,7 @@ public static void main(String... args) { actionName = args[2].toLowerCase(); } storage = StorageServiceFactory.getDefault(StorageServiceOptions.builder().build()); - bucketInfo = storage.get(bucket); + bucket = storage.get(bucketName); } action = ACTIONS.get(actionName); } @@ -117,7 +116,6 @@ public static void main(String... args) { } args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{}; - action.run(bucketInfo, folder, args); - } + action.run(storage, bucket, folder, args); } } diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 3295fa003530..44dc5df4e86a 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.gcloud.storage.StorageServiceOptions; import java.io.IOException; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory() .getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); } @Override public List buckets() throws IOException { Buckets buckets = storage.buckets().list(options.project()).execute(); return buckets.getItems(); } @Override public Bucket bucket(String bucket) throws IOException { return storage.buckets().get(bucket).execute(); } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.StorageServiceOptions; import java.io.IOException; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory() .getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); } @Override public List buckets() throws IOException { Buckets buckets = storage.buckets().list(options.project()).execute(); return buckets.getItems(); } @Override public List objects(String bucket, String prefix, String delimiter) throws IOException { return null; } @Override public Bucket get(String bucket) throws IOException { return storage.buckets().get(bucket).execute(); } @Override public StorageObject get(String bucket, String object) throws IOException { return storage.objects().get(bucket, object).execute(); } @Override public Bucket patch(Bucket bucket) throws IOException { return storage.buckets().patch(bucket.getName(), bucket).execute(); } @Override public StorageObject patch(StorageObject storageObject) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } @Override public void delete(String bucket, String object) throws IOException { storage.objects().delete(bucket, object).execute(); } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index d4e6a245dea5..9c650f5113fe 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import java.io.IOException; import java.util.List; public interface StorageRpc { List buckets() throws IOException; Bucket bucket(String name) throws IOException; void patch(Bucket bucket) throws IOException; void patch(StorageObject storageObject) throws IOException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import java.io.IOException; import java.util.List; public interface StorageRpc { List buckets() throws IOException; List objects(String bucket, String prefix, String delimiter) throws IOException; Bucket get(String bucket) throws IOException; StorageObject get(String bucket, String object) throws IOException; Bucket patch(Bucket bucket) throws IOException; StorageObject patch(StorageObject storageObject) throws IOException; //void delete(String bucket) throws IOException; void delete(String bucket, String object) throws IOException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/Blob.java similarity index 96% rename from src/main/java/com/google/gcloud/storage/ObjectInfo.java rename to src/main/java/com/google/gcloud/storage/Blob.java index 2e96a9163e93..e5fb3a76a6e2 100644 --- a/src/main/java/com/google/gcloud/storage/ObjectInfo.java +++ b/src/main/java/com/google/gcloud/storage/Blob.java @@ -19,6 +19,7 @@ import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.api.services.storage.model.StorageObject; +import com.google.api.services.storage.model.StorageObject.Owner; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -29,7 +30,7 @@ import java.util.List; import java.util.Map; -public class ObjectInfo implements Serializable { +public class Blob implements Serializable { private static final long serialVersionUID = 2228487739943277159L; @@ -194,12 +195,12 @@ public Builder updateTime(long updateTime) { return this; } - public ObjectInfo build() { - return new ObjectInfo(this); + public Blob build() { + return new Blob(this); } } - private ObjectInfo(Builder builder) { + private Blob(Builder builder) { bucket = builder.bucket; id = builder.id; name = builder.name; @@ -346,8 +347,7 @@ StorageObject toPb() { StorageObject storageObject = new StorageObject(); storageObject.setAcl(Lists.transform(acl, new Function() { - @Override - public ObjectAccessControl apply(Acl acl) { + @Override public ObjectAccessControl apply(Acl acl) { return acl.toObjectPb(); } })); @@ -363,7 +363,7 @@ public ObjectAccessControl apply(Acl acl) { storageObject.setMetadata(metadata); storageObject.setMetageneration(metageneration); storageObject.setName(name); - storageObject.setOwner(new StorageObject.Owner().setEntity(owner.toPb())); + storageObject.setOwner(new Owner().setEntity(owner.toPb())); storageObject.setUpdated(new DateTime(updateTime)); storageObject.setSize(BigInteger.valueOf(size)); storageObject.setContentDisposition(contentDisposition); @@ -375,7 +375,7 @@ public ObjectAccessControl apply(Acl acl) { return storageObject; } - static ObjectInfo fromPb(StorageObject storageObject) { + static Blob fromPb(StorageObject storageObject) { return builder() .acl(Lists.transform(storageObject.getAcl(), new Function() { diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/Bucket.java similarity index 80% rename from src/main/java/com/google/gcloud/storage/BucketInfo.java rename to src/main/java/com/google/gcloud/storage/Bucket.java index 97f1bea1e244..7d3944c0d9c0 100644 --- a/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -17,25 +17,26 @@ package com.google.gcloud.storage; import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.collect.Iterables.transform; +import static com.google.common.collect.Lists.transform; import com.google.api.client.util.DateTime; -import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.Bucket.Lifecycle; import com.google.api.services.storage.model.Bucket.Lifecycle.Rule; +import com.google.api.services.storage.model.Bucket.Owner; import com.google.api.services.storage.model.Bucket.Versioning; +import com.google.api.services.storage.model.Bucket.Website; import com.google.api.services.storage.model.BucketAccessControl; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.common.base.Function; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import com.google.gcloud.storage.Acl.Entity; import java.io.Serializable; import java.util.List; -public final class BucketInfo implements Serializable { +public final class Bucket implements Serializable { private static final long serialVersionUID = -3946094202176916586L; @@ -390,7 +391,7 @@ Builder metageneration(Long metageneration) { return this; } - Builder cors(Iterable cors) { + public Builder cors(Iterable cors) { this.cors = cors; return this; } @@ -405,12 +406,12 @@ public Builder defaultAcl(Iterable acl) { return this; } - public BucketInfo build() { - return new BucketInfo(this); + public Bucket build() { + return new Bucket(this); } } - private BucketInfo(Builder builder) { + private Bucket(Builder builder) { id = builder.id; name = builder.name; etag = builder.etag; @@ -511,100 +512,102 @@ public Builder toBuilder() { .deleteRules(deleteRules); } - BucketInfo fromPb(Bucket bucket) { - Builder builder = new Builder(bucket.getId(), bucket.getName()) - .createTime(bucket.getTimeCreated().getValue()) - .etag(bucket.getEtag()) - .metageneration(bucket.getMetageneration()) - .location(Location.of(bucket.getLocation())) - .storageClass(StorageClass.of(bucket.getStorageClass())) - .cors(transform(bucket.getCors(), new Function() { - @Override public Cors apply(Bucket.Cors cors) { - return Cors.fromPb(cors); - } - })) - .acl(transform(bucket.getAcl(), new Function() { - @Override public Acl apply(BucketAccessControl bucketAccessControl) { - return Acl.fromPb(bucketAccessControl); - } - })) - .defaultAcl(transform(bucket.getDefaultObjectAcl(), new Function() { - @Override public Acl apply(ObjectAccessControl objectAccessControl) { - return Acl.fromPb(objectAccessControl); - } - })) - .owner(Entity.fromPb(bucket.getOwner().getEntity())) - .selfLink(bucket.getSelfLink()); - Bucket.Versioning versioning = bucket.getVersioning(); - if (versioning != null) { - builder.versioningEnabled(MoreObjects.firstNonNull(versioning.getEnabled(), Boolean.FALSE)); - } - Bucket.Website website = bucket.getWebsite(); - if (website != null) { - builder.indexPage(website.getMainPageSuffix()); - builder.notFoundPage(website.getNotFoundPage()); - } - Bucket.Lifecycle lifecycle = bucket.getLifecycle(); - if (lifecycle != null) { - builder.deleteRules(transform(lifecycle.getRule(), - new Function() { - @Override public DeleteRule apply(Rule rule) { - return DeleteRule.fromPb(rule); - } - })); - } - return builder.build(); - } - - Bucket toPb() { - Bucket bucket = new Bucket(); - bucket.setId(id); - bucket.setName(name); - bucket.setEtag(etag); + com.google.api.services.storage.model.Bucket toPb() { + com.google.api.services.storage.model.Bucket bucketPb = + new com.google.api.services.storage.model.Bucket(); + bucketPb.setId(id); + bucketPb.setName(name); + bucketPb.setEtag(etag); if (createTime > 0) { - bucket.setTimeCreated(new DateTime(createTime)); + bucketPb.setTimeCreated(new DateTime(createTime)); } if (metageneration > 0) { - bucket.setMetageneration(metageneration); + bucketPb.setMetageneration(metageneration); } if (location != null) { - bucket.setLocation(location.value()); + bucketPb.setLocation(location.value()); } if (storageClass != null) { - bucket.setStorageClass(storageClass.value()); + bucketPb.setStorageClass(storageClass.value()); } - bucket.setCors(Lists.transform(cors, new Function() { - @Override - public Bucket.Cors apply(Cors cors) { - return cors.toPb(); - } - })); - bucket.setAcl(Lists.transform(acl, new Function() { - @Override - public BucketAccessControl apply(Acl acl) { + bucketPb.setCors( + transform(cors, new Function() { + @Override public com.google.api.services.storage.model.Bucket.Cors apply(Cors cors) { + return cors.toPb(); + } + })); + bucketPb.setAcl(transform(acl, new Function() { + @Override public BucketAccessControl apply(Acl acl) { return acl.toBucketPb(); } })); - bucket.setDefaultObjectAcl( - Lists.transform(defaultAcl, new Function() { - @Override - public ObjectAccessControl apply(Acl acl) { + bucketPb.setDefaultObjectAcl(transform(defaultAcl, + new Function() { + @Override public ObjectAccessControl apply(Acl acl) { return acl.toObjectPb(); } })); - bucket.setOwner(new Bucket.Owner().setEntity(owner.toPb())); - bucket.setSelfLink(selfLink); - bucket.setVersioning(new Versioning().setEnabled(versioningEnabled)); - Bucket.Website website = new Bucket.Website(); + bucketPb.setOwner(new Owner().setEntity(owner.toPb())); + bucketPb.setSelfLink(selfLink); + bucketPb.setVersioning(new Versioning().setEnabled(versioningEnabled)); + Website website = new Website(); website.setMainPageSuffix(indexPage); website.setNotFoundPage(notFoundPage); - bucket.setWebsite(website); - Bucket.Lifecycle lifecycle = new Bucket.Lifecycle(); - lifecycle.setRule(Lists.transform(deleteRules, new Function() { + bucketPb.setWebsite(website); + Lifecycle lifecycle = new Lifecycle(); + lifecycle.setRule(transform(deleteRules, new Function() { @Override public Rule apply(DeleteRule deleteRule) { return deleteRule.toPb(); } })); - return bucket; + return bucketPb; + } + + static Bucket fromPb(com.google.api.services.storage.model.Bucket bucketPb) { + Builder builder = new Builder(bucketPb.getId(), bucketPb.getName()) + .createTime(bucketPb.getTimeCreated().getValue()) + .etag(bucketPb.getEtag()) + .metageneration(bucketPb.getMetageneration()) + .location(Location.of(bucketPb.getLocation())) + .storageClass(StorageClass.of(bucketPb.getStorageClass())) + .cors(transform(bucketPb.getCors(), + new Function() { + @Override public Cors apply(com.google.api.services.storage.model.Bucket.Cors cors) { + return Cors.fromPb(cors); + } + })) + .acl(transform(bucketPb.getAcl(), + new Function() { + @Override public Acl apply(BucketAccessControl bucketAccessControl) { + return Acl.fromPb(bucketAccessControl); + } + })) + .defaultAcl(transform(bucketPb.getDefaultObjectAcl(), + new Function() { + @Override public Acl apply(ObjectAccessControl objectAccessControl) { + return Acl.fromPb(objectAccessControl); + } + })) + .owner(Entity.fromPb(bucketPb.getOwner().getEntity())) + .selfLink(bucketPb.getSelfLink()); + Versioning versioning = bucketPb.getVersioning(); + if (versioning != null) { + builder.versioningEnabled(MoreObjects.firstNonNull(versioning.getEnabled(), Boolean.FALSE)); + } + Website website = bucketPb.getWebsite(); + if (website != null) { + builder.indexPage(website.getMainPageSuffix()); + builder.notFoundPage(website.getNotFoundPage()); + } + Lifecycle lifecycle = bucketPb.getLifecycle(); + if (lifecycle != null) { + builder.deleteRules(transform(lifecycle.getRule(), + new Function() { + @Override public DeleteRule apply(Rule rule) { + return DeleteRule.fromPb(rule); + } + })); + } + return builder.build(); } } diff --git a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java index 8cc86d2c3e00..76d190d81a44 100644 --- a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java +++ b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java @@ -32,4 +32,6 @@ public interface ObjectWriteChannel extends WritableByteChannel, Serializable, C String bucket(); String object(); + + // todo: return ObjectInfo? upon close? } diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index f88f68cb331f..6ad5397384b6 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -20,23 +20,26 @@ public interface StorageService extends Service { - BucketInfo get(String bucket); + //todo: implement add/delete bucket + //todo: consider what to do with predefinedAcl - ObjectInfo get(String bucket, String object); + Bucket get(String bucket); - Iterable list(ListOptions settings); + Blob get(String bucket, String blob); - void update(BucketInfo bucketInfo); + Iterable list(ListOptions settings); - void update(ObjectInfo objectInfo); + Bucket update(Bucket bucket); - void delete(String bucket, String object); + Blob update(Blob blob); - void compose(String bucket, Iterable sourceObjects, String destObject); + void delete(String bucket, String blob); - void copy(String fromBucket, String fromObject, String destBucket, String destObject); + Blob compose(String bucket, Iterable srcBlobs, String destBlob); - ObjectReadChannel newReader(String bucket, String ObjectName); + Blob copy(String srcBucket, String srcBlob, String destBucket, String destBlob); - ObjectWriteChannel newWriter(String ObjectName); + ObjectReadChannel newReader(String bucket, String blob); + + ObjectWriteChannel newWriter(Blob blob); } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 96be7f634056..64ca284945da 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -16,9 +16,6 @@ package com.google.gcloud.storage; -import com.google.api.services.storage.model.Bucket; -import com.google.common.base.Function; -import com.google.common.collect.Iterables; import com.google.gcloud.BaseService; import com.google.gcloud.spi.StorageRpc; @@ -33,26 +30,63 @@ final class StorageServiceImpl extends BaseService implem storageRpc = options.storageRpc(); } + @Override - public Iterable listBuckets() { + public Bucket get(String bucket) { try { - return Iterables.transform(storageRpc.buckets(), - new Function() { - @Override public Bucket apply(com.google.api.services.storage.model.Bucket model) { - return new BucketImpl(StorageServiceImpl.this, model); - } - }); + return Bucket.fromPb(storageRpc.get(bucket)); } catch (IOException ex) { throw new StorageServiceException(ex); } } @Override - public BucketInfo getBucket(String bucket) { + public Blob get(String bucket, String object) { try { - return new BucketImpl(this, storageRpc.bucket(bucket)); + return Blob.fromPb(storageRpc.get(bucket, object)); } catch (IOException ex) { throw new StorageServiceException(ex); } } + + @Override + public Iterable list(ListOptions settings) { + return null; + } + + @Override + public Bucket update(Bucket bucket) { + return null; + } + + @Override + public Blob update(Blob blob) { + return null; + } + + @Override + public void delete(String bucket, String object) { + + } + + @Override + public Blob compose(String bucket, Iterable sourceObjects, String destObject) { + return null; + } + + @Override + public Blob copy(String fromBucket, String fromObject, String destBucket, + String destObject) { + return null; + } + + @Override + public ObjectReadChannel newReader(String bucket, String ObjectName) { + return null; + } + + @Override + public ObjectWriteChannel newWriter(Blob blob) { + return null; + } } From 483686366573c0ab988f9d3c7e807e89c8d086da Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 14 Apr 2015 14:54:27 -0700 Subject: [PATCH 16/48] work in progress --- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../java/com/google/gcloud/storage/Blob.java | 55 ++++++--- ...tReadChannel.java => BlobReadChannel.java} | 14 ++- .../com/google/gcloud/storage/Bucket.java | 60 ++++++---- .../java/com/google/gcloud/storage/Cors.java | 12 ++ .../google/gcloud/storage/ListOptions.java | 2 +- .../gcloud/storage/ObjectWriteChannel.java | 5 - .../google/gcloud/storage/StorageService.java | 34 ++++-- .../gcloud/storage/StorageServiceImpl.java | 110 +++++++++++++++--- 10 files changed, 216 insertions(+), 80 deletions(-) rename src/main/java/com/google/gcloud/storage/{ObjectReadChannel.java => BlobReadChannel.java} (82%) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 44dc5df4e86a..b61f96d23c06 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.StorageServiceOptions; import java.io.IOException; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory() .getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); } @Override public List buckets() throws IOException { Buckets buckets = storage.buckets().list(options.project()).execute(); return buckets.getItems(); } @Override public List objects(String bucket, String prefix, String delimiter) throws IOException { return null; } @Override public Bucket get(String bucket) throws IOException { return storage.buckets().get(bucket).execute(); } @Override public StorageObject get(String bucket, String object) throws IOException { return storage.objects().get(bucket, object).execute(); } @Override public Bucket patch(Bucket bucket) throws IOException { return storage.buckets().patch(bucket.getName(), bucket).execute(); } @Override public StorageObject patch(StorageObject storageObject) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } @Override public void delete(String bucket, String object) throws IOException { storage.objects().delete(bucket, object).execute(); } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.StorageServiceOptions; import java.io.IOException; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory() .getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() } @Override public List buckets() throws IOException { Buckets buckets = storage.buckets().list(options.project()).execute(); return buckets.getItems(); } @Override public List objects(String bucket, String prefix, String delimiter) throws IOException { return null; } @Override public Bucket get(String bucket) throws IOException { return storage.buckets().get(bucket).execute(); } @Override public StorageObject get(String bucket, String object) throws IOException { return storage.objects().get(bucket, object).execute(); } @Override public Bucket patch(Bucket bucket) throws IOException { return storage.buckets().patch(bucket.getName(), bucket).execute(); } @Override public StorageObject patch(StorageObject storageObject) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } @Override public void delete(String bucket, String object) throws IOException { storage.objects().delete(bucket, object).execute(); } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 9c650f5113fe..d74e8aeddc43 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import java.io.IOException; import java.util.List; public interface StorageRpc { List buckets() throws IOException; List objects(String bucket, String prefix, String delimiter) throws IOException; Bucket get(String bucket) throws IOException; StorageObject get(String bucket, String object) throws IOException; Bucket patch(Bucket bucket) throws IOException; StorageObject patch(StorageObject storageObject) throws IOException; //void delete(String bucket) throws IOException; void delete(String bucket, String object) throws IOException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.ObjectWriteChannel; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; public interface StorageRpc { Bucket create(Bucket bucket) throws IOException; StorageObject create(StorageObject object, ByteBuffer content) throws IOException; Iterator list() throws IOException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws IOException; Bucket get(Bucket bucket) throws IOException; StorageObject get(StorageObject object) throws IOException; Bucket patch(Bucket bucket) throws IOException; StorageObject patch(StorageObject storageObject) throws IOException; void delete(Bucket bucket) throws IOException; void delete(StorageObject object) throws IOException; StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException; StorageObject copy(StorageObject from, StorageObject to) throws IOException; BlobReadChannel reader(StorageObject from) throws IOException; ObjectWriteChannel writer(StorageObject to) throws IOException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java index e5fb3a76a6e2..d55673bcc168 100644 --- a/src/main/java/com/google/gcloud/storage/Blob.java +++ b/src/main/java/com/google/gcloud/storage/Blob.java @@ -16,6 +16,8 @@ package com.google.gcloud.storage; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.api.services.storage.model.StorageObject; @@ -34,6 +36,19 @@ public class Blob implements Serializable { private static final long serialVersionUID = 2228487739943277159L; + static final Function FROM_PB_FUNCTION = + new Function() { + @Override public Blob apply(StorageObject pb) { + return Blob.fromPb(pb); + } + }; + + static final Function TO_PB_FUNCTION = new Function() { + @Override public StorageObject apply(Blob blob) { + return blob.toPb(); + } + }; + private final String bucket; private final String id; private final String name; @@ -86,7 +101,7 @@ private Builder() { } public Builder bucket(String bucket) { - this.bucket = bucket; + this.bucket = checkNotNull(bucket); return this; } @@ -96,7 +111,7 @@ Builder id(String id) { } public Builder name(String name) { - this.name = name; + this.name = checkNotNull(name); return this; } @@ -201,9 +216,9 @@ public Blob build() { } private Blob(Builder builder) { - bucket = builder.bucket; + bucket = checkNotNull(builder.bucket); + name = checkNotNull(builder.name); id = builder.id; - name = builder.name; cacheControl = builder.cacheControl; contentEncoding = builder.contentEncoding; contentType = builder.contentType; @@ -314,33 +329,37 @@ public long updateTime() { } public Builder toBuilder() { - return builder() - .acl(acl) + return new Builder() .bucket(bucket) + .name(name) + .id(id) + .generation(generation) .cacheControl(cacheControl) .contentEncoding(contentEncoding) - .crc32c(crc32c) .contentType(contentType) - .deleteTime(deleteTime) - .generation(generation) + .contentDisposition(contentDisposition) + .contentLanguage(contentLanguage) + .componentCount(componentCount) + .crc32c(crc32c) .md5(md5) + .deleteTime(deleteTime) + .updateTime(updateTime) .mediaLink(mediaLink) .metadata(metadata) .metageneration(metageneration) - .name(name) + .acl(acl) .owner(owner) - .updateTime(updateTime) .size(size) - .contentDisposition(contentDisposition) - .componentCount(componentCount) - .contentLanguage(contentLanguage) .etag(etag) - .id(id) .selfLink(selfLink); } - public static Builder builder() { - return new Builder(); + public static Builder builder(Bucket bucket, String name) { + return builder(bucket.name(), name); + } + + public static Builder builder(String bucket, String name) { + return new Builder().bucket(bucket).name(name); } StorageObject toPb() { @@ -376,7 +395,7 @@ StorageObject toPb() { } static Blob fromPb(StorageObject storageObject) { - return builder() + return new Builder() .acl(Lists.transform(storageObject.getAcl(), new Function() { @Override public Acl apply(ObjectAccessControl objectAccessControl) { diff --git a/src/main/java/com/google/gcloud/storage/ObjectReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java similarity index 82% rename from src/main/java/com/google/gcloud/storage/ObjectReadChannel.java rename to src/main/java/com/google/gcloud/storage/BlobReadChannel.java index 0dbb320685f1..3757dec04e0b 100644 --- a/src/main/java/com/google/gcloud/storage/ObjectReadChannel.java +++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java @@ -27,13 +27,15 @@ * * This class is @{link Serializable}, which allows incremental reads. */ -public interface ObjectReadChannel extends ReadableByteChannel, Serializable, Closeable { +public interface BlobReadChannel extends ReadableByteChannel, Serializable, Closeable { - String bucket(); - - String object(); - - int size(); + /** + * Overridden to remove IOException. + * + * @see java.nio.channels.Channel#close() + */ + @Override + void close(); void seek(int position); } diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index 7d3944c0d9c0..4771506a2a35 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -58,6 +58,20 @@ public final class Bucket implements Serializable { private final StorageClass storageClass; + static final Function FROM_PB_FUNCTION = + new Function() { + @Override public Bucket apply(com.google.api.services.storage.model.Bucket pb) { + return Bucket.fromPb(pb); + } + }; + + static final Function TO_PB_FUNCTION = + new Function() { + @Override public com.google.api.services.storage.model.Bucket apply(Bucket bucket) { + return bucket.toPb(); + } + }; + public static abstract class DeleteRule implements Serializable { private static final long serialVersionUID = 3137971668395933033L; @@ -87,7 +101,7 @@ Rule toPb() { abstract void populateCondition(Rule.Condition condition); - static DeleteRule fromPb(Rule rule) { + private static DeleteRule fromPb(Rule rule) { if (rule.getAction() != null && SUPPORTED_ACTION.endsWith(rule.getAction().getType())) { Rule.Condition condition = rule.getCondition(); Integer age = condition.getAge(); @@ -314,8 +328,8 @@ public String value() { public final static class Builder { - private final String id; - private final String name; + private String id; + private String name; private Acl.Entity owner; private String selfLink; private boolean versioningEnabled; @@ -331,9 +345,17 @@ public final static class Builder { private Iterable acl = ImmutableList.of(); private Iterable defaultAcl = ImmutableList.of(); - Builder(String id, String name) { + private Builder() { + } + + public Builder name(String name) { + this.name = checkNotNull(name); + return this; + } + + Builder id(String id) { this.id = id; - this.name = name; + return this; } Builder owner(Acl.Entity owner) { @@ -413,7 +435,7 @@ public Bucket build() { private Bucket(Builder builder) { id = builder.id; - name = builder.name; + name = checkNotNull(builder.name); etag = builder.etag; createTime = MoreObjects.firstNonNull(builder.createTime, 0L); metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L); @@ -495,7 +517,9 @@ public List defaultAcl() { } public Builder toBuilder() { - return new Builder(id, name) + return new Builder() + .name(name) + .id(id) .createTime(createTime) .etag(etag) .metageneration(metageneration) @@ -512,6 +536,10 @@ public Builder toBuilder() { .deleteRules(deleteRules); } + public static Builder builder(String name) { + return new Builder().name(name); + } + com.google.api.services.storage.model.Bucket toPb() { com.google.api.services.storage.model.Bucket bucketPb = new com.google.api.services.storage.model.Bucket(); @@ -530,12 +558,7 @@ com.google.api.services.storage.model.Bucket toPb() { if (storageClass != null) { bucketPb.setStorageClass(storageClass.value()); } - bucketPb.setCors( - transform(cors, new Function() { - @Override public com.google.api.services.storage.model.Bucket.Cors apply(Cors cors) { - return cors.toPb(); - } - })); + bucketPb.setCors(transform(cors, Cors.TO_PB_FUNCTION)); bucketPb.setAcl(transform(acl, new Function() { @Override public BucketAccessControl apply(Acl acl) { return acl.toBucketPb(); @@ -564,18 +587,15 @@ com.google.api.services.storage.model.Bucket toPb() { } static Bucket fromPb(com.google.api.services.storage.model.Bucket bucketPb) { - Builder builder = new Builder(bucketPb.getId(), bucketPb.getName()) + Builder builder = new Builder() + .name(bucketPb.getName()) + .id(bucketPb.getId()) .createTime(bucketPb.getTimeCreated().getValue()) .etag(bucketPb.getEtag()) .metageneration(bucketPb.getMetageneration()) .location(Location.of(bucketPb.getLocation())) .storageClass(StorageClass.of(bucketPb.getStorageClass())) - .cors(transform(bucketPb.getCors(), - new Function() { - @Override public Cors apply(com.google.api.services.storage.model.Bucket.Cors cors) { - return Cors.fromPb(cors); - } - })) + .cors(transform(bucketPb.getCors(), Cors.FROM_PB_FUNCTION)) .acl(transform(bucketPb.getAcl(), new Function() { @Override public Acl apply(BucketAccessControl bucketAccessControl) { diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java index a58331d5063a..87d33348983d 100644 --- a/src/main/java/com/google/gcloud/storage/Cors.java +++ b/src/main/java/com/google/gcloud/storage/Cors.java @@ -34,6 +34,18 @@ public final class Cors implements Serializable { private static final long serialVersionUID = -8637770919343335655L; + static final Function FROM_PB_FUNCTION = new Function() { + @Override public Cors apply(Bucket.Cors pb) { + return Cors.fromPb(pb); + } + }; + + static final Function TO_PB_FUNCTION = new Function() { + @Override public Bucket.Cors apply(Cors cors) { + return cors.toPb(); + } + }; + private final Integer maxAgeSeconds; private final ImmutableList methods; private final ImmutableList origins; diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java index b1b5d96f27da..06637ed6aa3b 100644 --- a/src/main/java/com/google/gcloud/storage/ListOptions.java +++ b/src/main/java/com/google/gcloud/storage/ListOptions.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; public Builder() { } public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; private final boolean includeOlderVersions; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; private boolean includeOlderVersions; public Builder() { } public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } public Builder includeOlderVersions(boolean include) { this.includeOlderVersions = include; return this; } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; includeOlderVersions = builder.includeOlderVersions; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } public boolean includeOlderVersions() { return includeOlderVersions; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java index 76d190d81a44..518fb049718d 100644 --- a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java +++ b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java @@ -29,9 +29,4 @@ */ public interface ObjectWriteChannel extends WritableByteChannel, Serializable, Closeable { - String bucket(); - - String object(); - - // todo: return ObjectInfo? upon close? } diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 6ad5397384b6..6dfc8c48ef90 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -18,28 +18,42 @@ import com.google.gcloud.Service; +import java.nio.ByteBuffer; +import java.util.Iterator; + public interface StorageService extends Service { - //todo: implement add/delete bucket - //todo: consider what to do with predefinedAcl + // todo: consider what to do with predefinedAcl + // todo: consider supplying create/update/delete options (varargs) for + // ifGenerationMatch, ifGenerationNotMatch, ifMetagenerationMatch, + // ifMetagenerationNotMatch, ifSourceGenerationMatch, ifSourceGenerationNotMatch + // ifSourceMetagenerationMatch and ifSourceMetagenerationNotMatch + + Bucket create(Bucket bucket); + + Blob create(Blob blob, ByteBuffer content); - Bucket get(String bucket); + Bucket get(Bucket bucket); - Blob get(String bucket, String blob); + Blob get(Blob blob); - Iterable list(ListOptions settings); + Iterator list(); + + Iterator list(Bucket bucket, ListOptions settings); Bucket update(Bucket bucket); Blob update(Blob blob); - void delete(String bucket, String blob); + void delete(Bucket bucket); + + void delete(Blob blob); - Blob compose(String bucket, Iterable srcBlobs, String destBlob); + Blob compose(Bucket bucket, Iterable src, Blob dest); - Blob copy(String srcBucket, String srcBlob, String destBucket, String destBlob); + Blob copy(Blob src, Blob dest); - ObjectReadChannel newReader(String bucket, String blob); + BlobReadChannel readFrom(Blob blob); // todo: consider returning Blob - ObjectWriteChannel newWriter(Blob blob); + ObjectWriteChannel writeTo(Blob blob); // todo: consider returning Blob } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 64ca284945da..96b08398645c 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -16,10 +16,13 @@ package com.google.gcloud.storage; +import com.google.common.collect.Iterators; import com.google.gcloud.BaseService; import com.google.gcloud.spi.StorageRpc; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; final class StorageServiceImpl extends BaseService implements StorageService { @@ -32,61 +35,132 @@ final class StorageServiceImpl extends BaseService implem @Override - public Bucket get(String bucket) { + public Bucket create(Bucket bucket) { try { - return Bucket.fromPb(storageRpc.get(bucket)); + return Bucket.fromPb(storageRpc.create(bucket.toPb())); } catch (IOException ex) { throw new StorageServiceException(ex); } } @Override - public Blob get(String bucket, String object) { + public Blob create(Blob blob, ByteBuffer content) { try { - return Blob.fromPb(storageRpc.get(bucket, object)); + return Blob.fromPb(storageRpc.create(blob.toPb(), content)); } catch (IOException ex) { throw new StorageServiceException(ex); } } @Override - public Iterable list(ListOptions settings) { - return null; + public Bucket get(Bucket bucket) { + try { + return Bucket.fromPb(storageRpc.get(bucket.toPb())); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } + } + + @Override + public Blob get(Blob blob) { + try { + return Blob.fromPb(storageRpc.get(blob.toPb())); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } + } + + @Override + public Iterator list() { + try { + return Iterators.transform(storageRpc.list(), Bucket.FROM_PB_FUNCTION); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } + } + + @Override + public Iterator list(Bucket bucket, ListOptions settings) { + try { + String delimiter = settings.recursive() ? options().pathDelimiter() : null; + return Iterators.transform( + storageRpc.list(bucket.name(), settings.prefix(), delimiter, settings.cursor(), + settings.includeOlderVersions(), settings.maxResults()), + Blob.FROM_PB_FUNCTION); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } } @Override public Bucket update(Bucket bucket) { - return null; + try { + return Bucket.fromPb(storageRpc.patch(bucket.toPb())); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } } @Override public Blob update(Blob blob) { - return null; + try { + return Blob.fromPb(storageRpc.patch(blob.toPb())); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } } @Override - public void delete(String bucket, String object) { + public void delete(Bucket bucket) { + try { + storageRpc.delete(bucket.toPb()); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } + } + @Override + public void delete(Blob blob) { + try { + storageRpc.delete(blob.toPb()); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } } @Override - public Blob compose(String bucket, Iterable sourceObjects, String destObject) { - return null; + public Blob compose(Bucket bucket, Iterable src, Blob dest) { + try { + return Blob.fromPb(storageRpc.compose(bucket.name(), src, dest.toPb())); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } } @Override - public Blob copy(String fromBucket, String fromObject, String destBucket, - String destObject) { - return null; + public Blob copy(Blob src, Blob dest) { + try { + return Blob.fromPb(storageRpc.copy(src.toPb(), dest.toPb())); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } } @Override - public ObjectReadChannel newReader(String bucket, String ObjectName) { - return null; + public BlobReadChannel readFrom(Blob blob) { + try { + return storageRpc.reader(blob.toPb()); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } } @Override - public ObjectWriteChannel newWriter(Blob blob) { - return null; + public ObjectWriteChannel writeTo(Blob blob) { + try { + return storageRpc.writer(blob.toPb()); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } } } From fcb3dcd1c3482aff18e42a3cf1ec7cbc8cdcda7d Mon Sep 17 00:00:00 2001 From: ozarov Date: Tue, 14 Apr 2015 14:59:09 -0700 Subject: [PATCH 17/48] work in progress --- src/main/java/com/google/gcloud/spi/StorageRpc.java | 2 +- .../storage/{ObjectWriteChannel.java => BlobWriteChannel.java} | 2 +- src/main/java/com/google/gcloud/storage/StorageService.java | 2 +- src/main/java/com/google/gcloud/storage/StorageServiceImpl.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename src/main/java/com/google/gcloud/storage/{ObjectWriteChannel.java => BlobWriteChannel.java} (92%) diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index d74e8aeddc43..3e272eeb4606 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.ObjectWriteChannel; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; public interface StorageRpc { Bucket create(Bucket bucket) throws IOException; StorageObject create(StorageObject object, ByteBuffer content) throws IOException; Iterator list() throws IOException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws IOException; Bucket get(Bucket bucket) throws IOException; StorageObject get(StorageObject object) throws IOException; Bucket patch(Bucket bucket) throws IOException; StorageObject patch(StorageObject storageObject) throws IOException; void delete(Bucket bucket) throws IOException; void delete(StorageObject object) throws IOException; StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException; StorageObject copy(StorageObject from, StorageObject to) throws IOException; BlobReadChannel reader(StorageObject from) throws IOException; ObjectWriteChannel writer(StorageObject to) throws IOException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; public interface StorageRpc { Bucket create(Bucket bucket) throws IOException; StorageObject create(StorageObject object, ByteBuffer content) throws IOException; Iterator list() throws IOException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws IOException; Bucket get(Bucket bucket) throws IOException; StorageObject get(StorageObject object) throws IOException; Bucket patch(Bucket bucket) throws IOException; StorageObject patch(StorageObject storageObject) throws IOException; void delete(Bucket bucket) throws IOException; void delete(StorageObject object) throws IOException; StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException; StorageObject copy(StorageObject from, StorageObject to) throws IOException; BlobReadChannel reader(StorageObject from) throws IOException; BlobWriteChannel writer(StorageObject to) throws IOException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java similarity index 92% rename from src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java rename to src/main/java/com/google/gcloud/storage/BlobWriteChannel.java index 518fb049718d..a83ed661baca 100644 --- a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java +++ b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java @@ -27,6 +27,6 @@ * Written data will only be visible after calling {@link #close()}. * This class is serializable, to allow incremental writes. */ -public interface ObjectWriteChannel extends WritableByteChannel, Serializable, Closeable { +public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable { } diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 6dfc8c48ef90..2828f612b4ca 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -55,5 +55,5 @@ public interface StorageService extends Service { BlobReadChannel readFrom(Blob blob); // todo: consider returning Blob - ObjectWriteChannel writeTo(Blob blob); // todo: consider returning Blob + BlobWriteChannel writeTo(Blob blob); // todo: consider returning Blob } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 96b08398645c..eb9f78684efd 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -156,7 +156,7 @@ public BlobReadChannel readFrom(Blob blob) { } @Override - public ObjectWriteChannel writeTo(Blob blob) { + public BlobWriteChannel writeTo(Blob blob) { try { return storageRpc.writer(blob.toPb()); } catch (IOException ex) { From d3f035675c8cf17bfd8e76de4453f65813fa1f30 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 14 Apr 2015 15:57:24 -0700 Subject: [PATCH 18/48] work in progress --- src/main/java/com/google/gcloud/spi/StorageRpc.java | 2 +- .../com/google/gcloud/storage/StorageService.java | 9 ++++++--- .../google/gcloud/storage/StorageServiceImpl.java | 12 ++++++------ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 3e272eeb4606..2e851cdba981 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; public interface StorageRpc { Bucket create(Bucket bucket) throws IOException; StorageObject create(StorageObject object, ByteBuffer content) throws IOException; Iterator list() throws IOException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws IOException; Bucket get(Bucket bucket) throws IOException; StorageObject get(StorageObject object) throws IOException; Bucket patch(Bucket bucket) throws IOException; StorageObject patch(StorageObject storageObject) throws IOException; void delete(Bucket bucket) throws IOException; void delete(StorageObject object) throws IOException; StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException; StorageObject copy(StorageObject from, StorageObject to) throws IOException; BlobReadChannel reader(StorageObject from) throws IOException; BlobWriteChannel writer(StorageObject to) throws IOException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; public interface StorageRpc { Bucket create(Bucket bucket) throws IOException; StorageObject create(StorageObject object, ByteBuffer content) throws IOException; Iterator list() throws IOException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws IOException; Bucket get(String bucket) throws IOException; StorageObject get(String bucket, String object) throws IOException; Bucket patch(Bucket bucket) throws IOException; StorageObject patch(StorageObject storageObject) throws IOException; void delete(Bucket bucket) throws IOException; void delete(StorageObject object) throws IOException; StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException; StorageObject copy(StorageObject from, StorageObject to) throws IOException; BlobReadChannel reader(StorageObject from) throws IOException; BlobWriteChannel writer(StorageObject to) throws IOException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 2828f612b4ca..ff57f3fa2906 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -24,22 +24,25 @@ public interface StorageService extends Service { // todo: consider what to do with predefinedAcl + // todo: consider supplying create/update/delete options (varargs) for // ifGenerationMatch, ifGenerationNotMatch, ifMetagenerationMatch, // ifMetagenerationNotMatch, ifSourceGenerationMatch, ifSourceGenerationNotMatch // ifSourceMetagenerationMatch and ifSourceMetagenerationNotMatch + // todo: provide way for construct signed URLs - https://cloud.google.com/storage/docs/access-control#Signed-URLs + Bucket create(Bucket bucket); Blob create(Blob blob, ByteBuffer content); - Bucket get(Bucket bucket); + Bucket get(String bucket); - Blob get(Blob blob); + Blob get(String bucket, String blob); Iterator list(); - Iterator list(Bucket bucket, ListOptions settings); + Iterator list(String bucket, ListOptions settings); Bucket update(Bucket bucket); diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index eb9f78684efd..6df6800d31d5 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -53,18 +53,18 @@ public Blob create(Blob blob, ByteBuffer content) { } @Override - public Bucket get(Bucket bucket) { + public Bucket get(String bucket) { try { - return Bucket.fromPb(storageRpc.get(bucket.toPb())); + return Bucket.fromPb(storageRpc.get(bucket)); } catch (IOException ex) { throw new StorageServiceException(ex); } } @Override - public Blob get(Blob blob) { + public Blob get(String bucket, String blob) { try { - return Blob.fromPb(storageRpc.get(blob.toPb())); + return Blob.fromPb(storageRpc.get(bucket, blob)); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -80,11 +80,11 @@ public Iterator list() { } @Override - public Iterator list(Bucket bucket, ListOptions settings) { + public Iterator list(String bucket, ListOptions settings) { try { String delimiter = settings.recursive() ? options().pathDelimiter() : null; return Iterators.transform( - storageRpc.list(bucket.name(), settings.prefix(), delimiter, settings.cursor(), + storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(), settings.includeOlderVersions(), settings.maxResults()), Blob.FROM_PB_FUNCTION); } catch (IOException ex) { From 3171d8c4f1614fd30874f319d5ac21fd07e6fa66 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 15 Apr 2015 17:33:40 -0700 Subject: [PATCH 19/48] work in progress --- .../gcloud/examples/StorageExample.java | 5 +- .../google/gcloud/storage/StorageService.java | 334 +++++++++++++++++- .../gcloud/storage/StorageServiceImpl.java | 49 ++- 3 files changed, 349 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index ebfbeef3545f..f670cc7ac1db 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -16,6 +16,7 @@ package com.google.gcloud.examples; +import com.google.gcloud.storage.Blob; import com.google.gcloud.storage.Bucket; import com.google.gcloud.storage.StorageService; import com.google.gcloud.storage.StorageServiceFactory; @@ -61,12 +62,11 @@ protected String fullPath(String folder, String file) { } } - private static class DeleteAction extends StorageAction { @Override public void run(StorageService storage, Bucket bucket, String folder, String... args) { for (String file : args) { - storage.delete(bucket.name(), fullPath(folder, file)); + storage.delete(Blob.builder(bucket, fullPath(folder, file)).build()); } } @@ -78,6 +78,7 @@ public String getRequiredParams() { static { ACTIONS.put("delete", new DeleteAction()); + // todo: implement list, get and put } public static void main(String... args) { diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index ff57f3fa2906..5ce0c3d701bc 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -16,47 +16,345 @@ package com.google.gcloud.storage; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.collect.ImmutableList; import com.google.gcloud.Service; +import java.io.Serializable; import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.Set; public interface StorageService extends Service { - // todo: consider what to do with predefinedAcl + // todo: provide way for construct signed URLs - https://cloud.google.com/storage/docs/access-control#Signed-URLs - // todo: consider supplying create/update/delete options (varargs) for - // ifGenerationMatch, ifGenerationNotMatch, ifMetagenerationMatch, - // ifMetagenerationNotMatch, ifSourceGenerationMatch, ifSourceGenerationNotMatch - // ifSourceMetagenerationMatch and ifSourceMetagenerationNotMatch + enum PredefinedAcl { + AUTHENTICATED_READ("authenticatedRead"), + ALL_AUTHENTICATED_USERS("allAuthenticatedUsers"), + PRIVATE("private"), + PROJECT_PRIVATE("projectPrivate"), + PUBLIC_READ("publicRead"), + PUBLIC_READ_WRITE("publicReadWrite"), + BUCKET_OWNER_READ("bucketOwnerRead"), + BUCKET_OWNER_FULL_CONTROL("bucketOwnerFullControl"); - // todo: provide way for construct signed URLs - https://cloud.google.com/storage/docs/access-control#Signed-URLs + private final String entry; + + PredefinedAcl(String entry) { + this.entry = entry; + } + + String entry() { + return entry; + } + } + + class Option implements Serializable { + + private static final long serialVersionUID = -73199088766477208L; + + private final String name; + private final Object value; + + Option(String name, Object value) { + this.name = checkNotNull(name); + this.value = value; + } + + String name() { + return name; + } + + Object value() { + return value; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Option)) { + return false; + } + Option other = (Option) obj; + return Objects.equals(name, other.name) + && Objects.equals(value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(name, value); + } + } + + class BucketTargetOption extends Option { + + private static final long serialVersionUID = -5880204616982900975L; + + private BucketTargetOption(String name, Object value) { + super(name, value); + } + + public static BucketTargetOption predefinedAcl(PredefinedAcl acl) { + return new BucketTargetOption("predefinedAcl", acl.entry()); + } + + public static BucketTargetOption predefinedDefaultObjectAcl(PredefinedAcl acl) { + return new BucketTargetOption("predefinedDefaultObjectAcl", acl.entry()); + } + + public static BucketTargetOption metagenerationMatch(boolean match) { + return new BucketTargetOption("ifMetagenerationMatch", match); + } + } + + class BucketSourceOption extends Option { + + private static final long serialVersionUID = 5185657617120212117L; + + private BucketSourceOption(String name, Object value) { + super(name, value); + } + + public static BucketSourceOption metagenerationMatch(boolean match) { + return new BucketSourceOption("ifMetagenerationMatch", match); + } + } + + class BlobTargetOption extends Option { + + private static final long serialVersionUID = 214616862061934846L; + + private BlobTargetOption(String name, Object value) { + super(name, value); + } + + public static BlobTargetOption predefinedAcl(PredefinedAcl acl) { + return new BlobTargetOption("predefinedAcl", acl.entry()); + } + + public static BlobTargetOption generationMath(boolean match) { + return new BlobTargetOption("ifMetagenerationMatch", match); + } + + public static BlobTargetOption metagenerationMatch(boolean match) { + return new BlobTargetOption("ifMetagenerationMatch", match); + } + } + + class BlobSourceOption extends Option { + + private static final long serialVersionUID = -3712768261070182991L; + + private BlobSourceOption(String name, Object value) { + super(name, value); + } + + public static BlobSourceOption generationMath(boolean match) { + return new BlobSourceOption("ifGenerationMatch", match); + } + + public static BlobSourceOption metagenerationMatch(boolean match) { + return new BlobSourceOption("ifMetagenerationMatch", match); + } + } + + class ComposeRequest implements Serializable { + + private static final long serialVersionUID = -7385681353748590911L; + + private final String sourceBucket; + private final List sourceBlobs; + private final Blob target; + private final List targetOptions; + + private static class SourceBlob implements Serializable { + + private static final long serialVersionUID = 4094962795951990439L; + + final String blob; + final String generation; + + SourceBlob(String blob) { + this(blob, null); + } + + SourceBlob(String blob, String generation) { + this.blob = blob; + this.generation = generation; + } + } + + public static class Builder { + + private String bucket; + private List sourceBlobs = new LinkedList<>(); + private Blob target; + private Set targetOptions = new LinkedHashSet<>(); + + public Builder sourceBucket(String bucket) { + this.bucket = bucket; + return this; + } + + public Builder addSource(Iterable blobs) { + for (String blob : blobs) { + sourceBlobs.add(new SourceBlob(blob)); + } + return this; + } + + public Builder addSource(String... blobs) { + return addSource(Arrays.asList(blobs)); + } + + public Builder addSource(String blob, String generation) { + sourceBlobs.add(new SourceBlob(blob, generation)); + return this; + } + + public Builder target(Blob target) { + this.target = target; + return this; + } + + public Builder targetOptions(BlobTargetOption... options) { + Collections.addAll(targetOptions, options); + return this; + } + + public ComposeRequest build() { + return new ComposeRequest(this); + } + } + + private ComposeRequest(Builder builder) { + sourceBucket = checkNotNull(builder.bucket); + sourceBlobs = ImmutableList.copyOf(builder.sourceBlobs); + target = checkNotNull(builder.target); + targetOptions = ImmutableList.copyOf(builder.targetOptions); + } + + String sourceBucket() { + return sourceBucket; + } + + List sourceBlobs() { + return sourceBlobs; + } + + Blob target() { + return target; + } + + List targetOptions() { + return targetOptions; + } + + public static Builder builder() { + return new Builder(); + } + } + + class CopyRequest implements Serializable { + + private static final long serialVersionUID = -2606508373751748775L; + + private final Blob source; + private final List sourceOptions; + private final Blob target; + private final List targetOptions; + + public static class Builder { + + private Blob source; + private Set sourceOptions; + private Blob target; + private Set targetOptions; + + public void source(Blob source) { + this.source = source; + } + + public Builder sourceOptions(BlobSourceOption... options) { + Collections.addAll(sourceOptions, options); + return this; + } + + public Builder target(Blob target) { + this.target = target; + return this; + } + + public Builder targetOptions(BlobTargetOption... options) { + Collections.addAll(targetOptions, options); + return this; + } + + public CopyRequest build() { + return new CopyRequest(this); + } + } + + private CopyRequest(Builder builder) { + source = checkNotNull(builder.source); + sourceOptions = ImmutableList.copyOf(builder.sourceOptions); + target = checkNotNull(builder.target); + targetOptions = ImmutableList.copyOf(builder.targetOptions); + } + + Blob source() { + return source; + } + + public List sourceOptions() { + return sourceOptions; + } + + Blob target() { + return target; + } + + List targetOptions() { + return targetOptions; + } + + public static Builder builder() { + return new Builder(); + } + } - Bucket create(Bucket bucket); + Bucket create(Bucket bucket, BucketTargetOption... option); - Blob create(Blob blob, ByteBuffer content); + Blob create(Blob blob, ByteBuffer content, BlobTargetOption... option); - Bucket get(String bucket); + Bucket get(String bucket, BucketSourceOption... options); - Blob get(String bucket, String blob); + Blob get(String bucket, String blob, BlobSourceOption... options); Iterator list(); Iterator list(String bucket, ListOptions settings); - Bucket update(Bucket bucket); + Bucket update(Bucket bucket, BucketTargetOption... option); - Blob update(Blob blob); + Blob update(Blob blob, BlobTargetOption... option); - void delete(Bucket bucket); + void delete(Bucket bucket, BucketSourceOption... option); - void delete(Blob blob); + void delete(Blob blob, BlobSourceOption... option); - Blob compose(Bucket bucket, Iterable src, Blob dest); + Blob compose(ComposeRequest composeRequest); - Blob copy(Blob src, Blob dest); + Blob copy(CopyRequest copyRequest); - BlobReadChannel readFrom(Blob blob); // todo: consider returning Blob + BlobReadChannel readFrom(Blob blob, BlobSourceOption... option); - BlobWriteChannel writeTo(Blob blob); // todo: consider returning Blob + BlobWriteChannel writeTo(Blob blob, BlobTargetOption... option); } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 6df6800d31d5..78acb40d5b9a 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -31,11 +31,17 @@ final class StorageServiceImpl extends BaseService implem StorageServiceImpl(StorageServiceOptions options) { super(options); storageRpc = options.storageRpc(); - } + // todo: like Datastore distinct exception to retriable and non-retriable + // https://cloud.google.com/storage/docs/json_api/v1/status-codes + // todo: Use retry helper on retriable failures + + // todo: consider options + // todo: replace nulls with Value.asNull (per toPb) + } @Override - public Bucket create(Bucket bucket) { + public Bucket create(Bucket bucket, BucketTargetOption... options) { try { return Bucket.fromPb(storageRpc.create(bucket.toPb())); } catch (IOException ex) { @@ -44,7 +50,7 @@ public Bucket create(Bucket bucket) { } @Override - public Blob create(Blob blob, ByteBuffer content) { + public Blob create(Blob blob, ByteBuffer content, BlobTargetOption... options) { try { return Blob.fromPb(storageRpc.create(blob.toPb(), content)); } catch (IOException ex) { @@ -53,7 +59,7 @@ public Blob create(Blob blob, ByteBuffer content) { } @Override - public Bucket get(String bucket) { + public Bucket get(String bucket, BucketSourceOption... options) { try { return Bucket.fromPb(storageRpc.get(bucket)); } catch (IOException ex) { @@ -62,7 +68,7 @@ public Bucket get(String bucket) { } @Override - public Blob get(String bucket, String blob) { + public Blob get(String bucket, String blob, BlobSourceOption... options) { try { return Blob.fromPb(storageRpc.get(bucket, blob)); } catch (IOException ex) { @@ -93,7 +99,7 @@ public Iterator list(String bucket, ListOptions settings) { } @Override - public Bucket update(Bucket bucket) { + public Bucket update(Bucket bucket, BucketTargetOption... options) { try { return Bucket.fromPb(storageRpc.patch(bucket.toPb())); } catch (IOException ex) { @@ -102,7 +108,7 @@ public Bucket update(Bucket bucket) { } @Override - public Blob update(Blob blob) { + public Blob update(Blob blob, BlobTargetOption... options) { try { return Blob.fromPb(storageRpc.patch(blob.toPb())); } catch (IOException ex) { @@ -111,7 +117,7 @@ public Blob update(Blob blob) { } @Override - public void delete(Bucket bucket) { + public void delete(Bucket bucket, BucketSourceOption... options) { try { storageRpc.delete(bucket.toPb()); } catch (IOException ex) { @@ -120,7 +126,7 @@ public void delete(Bucket bucket) { } @Override - public void delete(Blob blob) { + public void delete(Blob blob, BlobSourceOption... options) { try { storageRpc.delete(blob.toPb()); } catch (IOException ex) { @@ -129,25 +135,30 @@ public void delete(Blob blob) { } @Override - public Blob compose(Bucket bucket, Iterable src, Blob dest) { - try { - return Blob.fromPb(storageRpc.compose(bucket.name(), src, dest.toPb())); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public Blob compose(ComposeRequest composeRequest) { + // todo: implement (consider having XXXRequest/XXXResponse for all SPI/Rpc requests +// try { + +// +// return Blob.fromPb(storageRpc.compose(composeRequest.sourceBucket(), +// composeRequest.sourceBlobs(), composeRequest.target())); +// } catch (IOException ex) { +// throw new StorageServiceException(ex); +// } + throw new UnsupportedOperationException("bla"); } @Override - public Blob copy(Blob src, Blob dest) { + public Blob copy(CopyRequest copyRequest) { try { - return Blob.fromPb(storageRpc.copy(src.toPb(), dest.toPb())); + return Blob.fromPb(storageRpc.copy(copyRequest.source().toPb(), copyRequest.target().toPb())); } catch (IOException ex) { throw new StorageServiceException(ex); } } @Override - public BlobReadChannel readFrom(Blob blob) { + public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) { try { return storageRpc.reader(blob.toPb()); } catch (IOException ex) { @@ -156,7 +167,7 @@ public BlobReadChannel readFrom(Blob blob) { } @Override - public BlobWriteChannel writeTo(Blob blob) { + public BlobWriteChannel writeTo(Blob blob, BlobTargetOption... options) { try { return storageRpc.writer(blob.toPb()); } catch (IOException ex) { From f88107c6c1f8afa24cfaeface9157373f1ce1d19 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 17 Apr 2015 17:40:18 -0700 Subject: [PATCH 20/48] work in progress --- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../java/com/google/gcloud/storage/Blob.java | 125 +++++++------ .../com/google/gcloud/storage/Bucket.java | 169 ++++++++++-------- .../com/google/gcloud/storage/Option.java | 61 +++++++ .../google/gcloud/storage/StorageService.java | 61 ++----- .../gcloud/storage/StorageServiceImpl.java | 49 ++--- .../gcloud/storage/StorageServiceOptions.java | 11 +- .../com/google/gcloud/storage/Validator.java | 38 ++++ 8 files changed, 322 insertions(+), 194 deletions(-) create mode 100644 src/main/java/com/google/gcloud/storage/Option.java create mode 100644 src/main/java/com/google/gcloud/storage/Validator.java diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 2e851cdba981..74b0d55aa02e 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; public interface StorageRpc { Bucket create(Bucket bucket) throws IOException; StorageObject create(StorageObject object, ByteBuffer content) throws IOException; Iterator list() throws IOException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws IOException; Bucket get(String bucket) throws IOException; StorageObject get(String bucket, String object) throws IOException; Bucket patch(Bucket bucket) throws IOException; StorageObject patch(StorageObject storageObject) throws IOException; void delete(Bucket bucket) throws IOException; void delete(StorageObject object) throws IOException; StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException; StorageObject copy(StorageObject from, StorageObject to) throws IOException; BlobReadChannel reader(StorageObject from) throws IOException; BlobWriteChannel writer(StorageObject to) throws IOException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; import java.util.List; public interface StorageRpc { Bucket create(Bucket bucket, Option... options) throws IOException; StorageObject create(StorageObject object, ByteBuffer content, Option... options) throws IOException; Iterator list() throws IOException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws IOException; Bucket get(String bucket, Option... options) throws IOException; StorageObject get(String bucket, String object, Option... options) throws IOException; Bucket patch(Bucket bucket, Option... options) throws IOException; StorageObject patch(StorageObject storageObject, Option... options) throws IOException; void delete(Bucket bucket, Option... options) throws IOException; void delete(StorageObject object, Option... options) throws IOException; StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws IOException; StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws IOException; BlobReadChannel reader(StorageObject from, Option... options) throws IOException; BlobWriteChannel writer(StorageObject to, Option... options) throws IOException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java index d55673bcc168..04553a8c744e 100644 --- a/src/main/java/com/google/gcloud/storage/Blob.java +++ b/src/main/java/com/google/gcloud/storage/Blob.java @@ -54,23 +54,23 @@ public class Blob implements Serializable { private final String name; private final String selfLink; private final String cacheControl; - private final ImmutableList acl; + private final List acl; private final Acl.Entity owner; - private final long size; + private final Long size; private final String etag; private final String md5; private final String crc32c; private final String mediaLink; - private final ImmutableMap metadata; - private final long generation; - private final long metageneration; - private final long deleteTime; - private final long updateTime; + private final Map metadata; + private final Long generation; + private final Long metageneration; + private final Long deleteTime; + private final Long updateTime; private final String contentType; private final String contentEncoding; private final String contentDisposition; private final String contentLanguage; - private final int componentCount; + private final Integer componentCount; public static final class Builder { @@ -80,22 +80,22 @@ public static final class Builder { private String contentType; private String contentEncoding; private String contentDisposition; - private String contentLanguage; - private int componentCount; + private String contentLanguage; + private Integer componentCount; private String cacheControl; private ImmutableList acl; private Acl.Entity owner; - private long size; + private Long size; private String etag; private String selfLink; private String md5; private String crc32c; private String mediaLink; private ImmutableMap metadata; - private long generation; - private long metageneration; - private long deleteTime; - private long updateTime; + private Long generation; + private Long metageneration; + private Long deleteTime; + private Long updateTime; private Builder() { } @@ -135,7 +135,7 @@ public Builder contentEncoding(String contentEncoding) { return this; } - Builder componentCount(int componentCount) { + Builder componentCount(Integer componentCount) { this.componentCount = componentCount; return this; } @@ -155,7 +155,7 @@ public Builder owner(Acl.Entity owner) { return this; } - public Builder size(long size) { + public Builder size(Long size) { this.size = size; return this; } @@ -190,34 +190,36 @@ public Builder metadata(Map metadata) { return this; } - public Builder generation(long generation) { + public Builder generation(Long generation) { this.generation = generation; return this; } - public Builder metageneration(long metageneration) { + public Builder metageneration(Long metageneration) { this.metageneration = metageneration; return this; } - public Builder deleteTime(long deleteTime) { + public Builder deleteTime(Long deleteTime) { this.deleteTime = deleteTime; return this; } - public Builder updateTime(long updateTime) { + public Builder updateTime(Long updateTime) { this.updateTime = updateTime; return this; } public Blob build() { + checkNotNull(bucket); + checkNotNull(name); return new Blob(this); } } private Blob(Builder builder) { - bucket = checkNotNull(builder.bucket); - name = checkNotNull(builder.name); + bucket = builder.bucket; + name = builder.name; id = builder.id; cacheControl = builder.cacheControl; contentEncoding = builder.contentEncoding; @@ -264,7 +266,7 @@ public Acl.Entity owner() { return owner; } - public long size() { + public Long size() { return size; } @@ -284,7 +286,7 @@ public String contentLanguage() { return contentEncoding; } - public int componentCount() { + public Integer componentCount() { return componentCount; } @@ -312,19 +314,19 @@ public Map metadata() { return metadata; } - public long generation() { + public Long generation() { return generation; } - public long metageneration() { + public Long metageneration() { return metageneration; } - public long deleteTime() { + public Long deleteTime() { return deleteTime; } - public long updateTime() { + public Long updateTime() { return updateTime; } @@ -364,27 +366,36 @@ public static Builder builder(String bucket, String name) { StorageObject toPb() { StorageObject storageObject = new StorageObject(); - storageObject.setAcl(Lists.transform(acl, - new Function() { - @Override public ObjectAccessControl apply(Acl acl) { - return acl.toObjectPb(); - } - })); + if (acl != null) { + storageObject.setAcl(Lists.transform(acl, new Function() { + @Override public ObjectAccessControl apply(Acl acl) { + return acl.toObjectPb(); + } + })); + } + if (deleteTime != null) { + storageObject.setTimeDeleted(new DateTime(deleteTime)); + } + if (updateTime != null) { + storageObject.setUpdated(new DateTime(updateTime)); + } + if (size != null) { + storageObject.setSize(BigInteger.valueOf(size)); + } + if (owner != null) { + storageObject.setOwner(new Owner().setEntity(owner.toPb())); + } storageObject.setBucket(bucket); storageObject.setCacheControl(cacheControl); storageObject.setContentEncoding(contentEncoding); storageObject.setCrc32c(crc32c); storageObject.setContentType(contentType); - storageObject.setTimeDeleted(new DateTime(deleteTime)); storageObject.setGeneration(generation); storageObject.setMd5Hash(md5); storageObject.setMediaLink(mediaLink); storageObject.setMetadata(metadata); storageObject.setMetageneration(metageneration); storageObject.setName(name); - storageObject.setOwner(new Owner().setEntity(owner.toPb())); - storageObject.setUpdated(new DateTime(updateTime)); - storageObject.setSize(BigInteger.valueOf(size)); storageObject.setContentDisposition(contentDisposition); storageObject.setComponentCount(componentCount); storageObject.setContentLanguage(contentLanguage); @@ -395,34 +406,44 @@ StorageObject toPb() { } static Blob fromPb(StorageObject storageObject) { - return new Builder() - .acl(Lists.transform(storageObject.getAcl(), - new Function() { - @Override public Acl apply(ObjectAccessControl objectAccessControl) { - return Acl.fromPb(objectAccessControl); - } - })) + Builder builder = new Builder() .bucket(storageObject.getBucket()) .cacheControl(storageObject.getCacheControl()) .contentEncoding(storageObject.getContentEncoding()) .crc32c(storageObject.getCrc32c()) .contentType(storageObject.getContentType()) - .deleteTime(storageObject.getTimeDeleted().getValue()) .generation(storageObject.getGeneration()) .md5(storageObject.getMd5Hash()) .mediaLink(storageObject.getMediaLink()) .metadata(storageObject.getMetadata()) .metageneration(storageObject.getMetageneration()) .name(storageObject.getName()) - .owner(Acl.Entity.fromPb(storageObject.getOwner().getEntity())) - .updateTime(storageObject.getUpdated().getValue()) - .size(storageObject.getSize().longValue()) .contentDisposition(storageObject.getContentDisposition()) .componentCount(storageObject.getComponentCount()) .contentLanguage(storageObject.getContentLanguage()) .etag(storageObject.getEtag()) .id(storageObject.getId()) - .selfLink(storageObject.getSelfLink()) - .build(); + .selfLink(storageObject.getSelfLink()); + if (storageObject.getTimeDeleted() != null) { + builder.deleteTime(storageObject.getTimeDeleted().getValue()); + } + if (storageObject.getUpdated() != null) { + builder.updateTime(storageObject.getUpdated().getValue()); + } + if (storageObject.getSize() != null) { + builder.size(storageObject.getSize().longValue()); + } + if (storageObject.getOwner() != null) { + builder.owner(Acl.Entity.fromPb(storageObject.getOwner().getEntity())); + } + if (storageObject.getAcl() != null) { + builder.acl(Lists.transform(storageObject.getAcl(), new Function() { + @Override + public Acl apply(ObjectAccessControl objectAccessControl) { + return Acl.fromPb(objectAccessControl); + } + })); + } + return builder.build(); } } diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index 4771506a2a35..55586acc3cbb 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -28,7 +28,6 @@ import com.google.api.services.storage.model.BucketAccessControl; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.common.base.Function; -import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.Acl.Entity; @@ -44,16 +43,16 @@ public final class Bucket implements Serializable { private final String name; private final Acl.Entity owner; private final String selfLink; - private final boolean versioningEnabled; + private final Boolean versioningEnabled; private final String indexPage; private final String notFoundPage; - private final ImmutableList deleteRules; + private final List deleteRules; private final String etag; - private final long createTime; - private final long metageneration; - private final ImmutableList cors; - private final ImmutableList acl; - private final ImmutableList defaultAcl; + private final Long createTime; + private final Long metageneration; + private final List cors; + private final List acl; + private final List defaultAcl; private final Location location; private final StorageClass storageClass; @@ -332,18 +331,18 @@ public final static class Builder { private String name; private Acl.Entity owner; private String selfLink; - private boolean versioningEnabled; + private Boolean versioningEnabled; private String indexPage; private String notFoundPage; - private ImmutableList deleteRules = ImmutableList.of(); + private ImmutableList deleteRules; private StorageClass storageClass; private Location location; private String etag; private Long createTime; private Long metageneration; - private Iterable cors = ImmutableList.of(); - private Iterable acl = ImmutableList.of(); - private Iterable defaultAcl = ImmutableList.of(); + private ImmutableList cors; + private ImmutableList acl; + private ImmutableList defaultAcl; private Builder() { } @@ -368,7 +367,7 @@ Builder selfLink(String selfLink) { return this; } - public Builder versioningEnabled(boolean enable) { + public Builder versioningEnabled(Boolean enable) { this.versioningEnabled = enable; return this; } @@ -414,7 +413,7 @@ Builder metageneration(Long metageneration) { } public Builder cors(Iterable cors) { - this.cors = cors; + this.cors = ImmutableList.copyOf(cors); return this; } @@ -429,27 +428,28 @@ public Builder defaultAcl(Iterable acl) { } public Bucket build() { + checkNotNull(name); return new Bucket(this); } } private Bucket(Builder builder) { id = builder.id; - name = checkNotNull(builder.name); + name = builder.name; etag = builder.etag; - createTime = MoreObjects.firstNonNull(builder.createTime, 0L); - metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L); + createTime = builder.createTime; + metageneration = builder.metageneration; location = builder.location; storageClass = builder.storageClass; - cors = ImmutableList.copyOf(builder.cors); - acl = ImmutableList.copyOf(builder.acl); - defaultAcl = ImmutableList.copyOf(builder.defaultAcl); + cors = builder.cors; + acl = builder.acl; + defaultAcl = builder.defaultAcl; owner = builder.owner; selfLink = builder.selfLink; versioningEnabled = builder.versioningEnabled; indexPage = builder.indexPage; notFoundPage = builder.notFoundPage; - deleteRules = ImmutableList.copyOf(builder.deleteRules); + deleteRules = builder.deleteRules; } public String id() { @@ -468,7 +468,7 @@ public String selfLink() { return selfLink; } - public boolean versioningEnabled() { + public Boolean versioningEnabled() { return versioningEnabled; } @@ -480,7 +480,7 @@ public String notFoundPage() { return notFoundPage; } - public ImmutableList deleteRules() { + public List deleteRules() { return deleteRules; } @@ -488,11 +488,11 @@ public String etag() { return etag; } - public long createTime() { + public Long createTime() { return createTime; } - public long metageneration() { + public Long metageneration() { return metageneration; } @@ -546,10 +546,10 @@ com.google.api.services.storage.model.Bucket toPb() { bucketPb.setId(id); bucketPb.setName(name); bucketPb.setEtag(etag); - if (createTime > 0) { + if (createTime != null) { bucketPb.setTimeCreated(new DateTime(createTime)); } - if (metageneration > 0) { + if (metageneration != null) { bucketPb.setMetageneration(metageneration); } if (location != null) { @@ -558,31 +558,44 @@ com.google.api.services.storage.model.Bucket toPb() { if (storageClass != null) { bucketPb.setStorageClass(storageClass.value()); } - bucketPb.setCors(transform(cors, Cors.TO_PB_FUNCTION)); - bucketPb.setAcl(transform(acl, new Function() { - @Override public BucketAccessControl apply(Acl acl) { - return acl.toBucketPb(); - } - })); - bucketPb.setDefaultObjectAcl(transform(defaultAcl, - new Function() { - @Override public ObjectAccessControl apply(Acl acl) { - return acl.toObjectPb(); - } - })); - bucketPb.setOwner(new Owner().setEntity(owner.toPb())); + if (cors != null) { + bucketPb.setCors(transform(cors, Cors.TO_PB_FUNCTION)); + } + if (acl != null) { + bucketPb.setAcl(transform(acl, new Function() { + @Override public BucketAccessControl apply(Acl acl) { + return acl.toBucketPb(); + } + })); + } + if (defaultAcl != null) { + bucketPb.setDefaultObjectAcl(transform(defaultAcl, new Function() { + @Override public ObjectAccessControl apply(Acl acl) { + return acl.toObjectPb(); + } + })); + } + if (owner != null) { + bucketPb.setOwner(new Owner().setEntity(owner.toPb())); + } bucketPb.setSelfLink(selfLink); - bucketPb.setVersioning(new Versioning().setEnabled(versioningEnabled)); - Website website = new Website(); - website.setMainPageSuffix(indexPage); - website.setNotFoundPage(notFoundPage); - bucketPb.setWebsite(website); - Lifecycle lifecycle = new Lifecycle(); - lifecycle.setRule(transform(deleteRules, new Function() { - @Override public Rule apply(DeleteRule deleteRule) { - return deleteRule.toPb(); - } - })); + if (versioningEnabled != null) { + bucketPb.setVersioning(new Versioning().setEnabled(versioningEnabled)); + } + if (indexPage != null || notFoundPage != null) { + Website website = new Website(); + website.setMainPageSuffix(indexPage); + website.setNotFoundPage(notFoundPage); + bucketPb.setWebsite(website); + } + if (deleteRules != null) { + Lifecycle lifecycle = new Lifecycle(); + lifecycle.setRule(transform(deleteRules, new Function() { + @Override public Rule apply(DeleteRule deleteRule) { + return deleteRule.toPb(); + } + })); + } return bucketPb; } @@ -590,38 +603,46 @@ static Bucket fromPb(com.google.api.services.storage.model.Bucket bucketPb) { Builder builder = new Builder() .name(bucketPb.getName()) .id(bucketPb.getId()) - .createTime(bucketPb.getTimeCreated().getValue()) .etag(bucketPb.getEtag()) .metageneration(bucketPb.getMetageneration()) + .createTime(bucketPb.getTimeCreated().getValue()) .location(Location.of(bucketPb.getLocation())) - .storageClass(StorageClass.of(bucketPb.getStorageClass())) - .cors(transform(bucketPb.getCors(), Cors.FROM_PB_FUNCTION)) - .acl(transform(bucketPb.getAcl(), - new Function() { - @Override public Acl apply(BucketAccessControl bucketAccessControl) { - return Acl.fromPb(bucketAccessControl); - } - })) - .defaultAcl(transform(bucketPb.getDefaultObjectAcl(), - new Function() { - @Override public Acl apply(ObjectAccessControl objectAccessControl) { - return Acl.fromPb(objectAccessControl); - } - })) - .owner(Entity.fromPb(bucketPb.getOwner().getEntity())) .selfLink(bucketPb.getSelfLink()); - Versioning versioning = bucketPb.getVersioning(); - if (versioning != null) { - builder.versioningEnabled(MoreObjects.firstNonNull(versioning.getEnabled(), Boolean.FALSE)); + if (bucketPb.getStorageClass() != null) { + builder.storageClass(StorageClass.of(bucketPb.getStorageClass())); + } + if (bucketPb.getCors() != null) { + builder.cors(transform(bucketPb.getCors(), Cors.FROM_PB_FUNCTION)); + } + if (bucketPb.getAcl() != null) { + builder.acl(transform(bucketPb.getAcl(), new Function() { + @Override public Acl apply(BucketAccessControl bucketAccessControl) { + return Acl.fromPb(bucketAccessControl); + } + })); + } + if (bucketPb.getDefaultObjectAcl() != null) { + builder.defaultAcl(transform(bucketPb.getDefaultObjectAcl(), + new Function() { + @Override + public Acl apply(ObjectAccessControl objectAccessControl) { + return Acl.fromPb(objectAccessControl); + } + })); + } + if (bucketPb.getOwner() != null) { + builder.owner(Entity.fromPb(bucketPb.getOwner().getEntity())); + } + if (bucketPb.getVersioning() != null) { + builder.versioningEnabled(bucketPb.getVersioning().getEnabled()); } Website website = bucketPb.getWebsite(); if (website != null) { builder.indexPage(website.getMainPageSuffix()); builder.notFoundPage(website.getNotFoundPage()); } - Lifecycle lifecycle = bucketPb.getLifecycle(); - if (lifecycle != null) { - builder.deleteRules(transform(lifecycle.getRule(), + if (bucketPb.getLifecycle() != null && bucketPb.getLifecycle().getRule() != null) { + builder.deleteRules(transform(bucketPb.getLifecycle().getRule(), new Function() { @Override public DeleteRule apply(Rule rule) { return DeleteRule.fromPb(rule); diff --git a/src/main/java/com/google/gcloud/storage/Option.java b/src/main/java/com/google/gcloud/storage/Option.java new file mode 100644 index 000000000000..cdbf00015b72 --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/Option.java @@ -0,0 +1,61 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Base class for Storage operation option + */ +public class Option implements Serializable { + + private static final long serialVersionUID = -73199088766477208L; + + private final String name; + private final Object value; + + Option(String name, Object value) { + this.name = checkNotNull(name); + this.value = value; + } + + public String name() { + return name; + } + + public Object value() { + return value; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Option)) { + return false; + } + Option other = (Option) obj; + return Objects.equals(name, other.name) + && Objects.equals(value, other.value); + } + + @Override + public int hashCode() { + return Objects.hash(name, value); + } +} diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 5ce0c3d701bc..c78056c57168 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -17,6 +17,7 @@ package com.google.gcloud.storage; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.gcloud.storage.Validator.checkBlobOptions; import com.google.common.collect.ImmutableList; import com.google.gcloud.Service; @@ -29,7 +30,6 @@ import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; -import java.util.Objects; import java.util.Set; public interface StorageService extends Service { @@ -57,42 +57,6 @@ String entry() { } } - class Option implements Serializable { - - private static final long serialVersionUID = -73199088766477208L; - - private final String name; - private final Object value; - - Option(String name, Object value) { - this.name = checkNotNull(name); - this.value = value; - } - - String name() { - return name; - } - - Object value() { - return value; - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof Option)) { - return false; - } - Option other = (Option) obj; - return Objects.equals(name, other.name) - && Objects.equals(value, other.value); - } - - @Override - public int hashCode() { - return Objects.hash(name, value); - } - } - class BucketTargetOption extends Option { private static final long serialVersionUID = -5880204616982900975L; @@ -140,7 +104,7 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) { } public static BlobTargetOption generationMath(boolean match) { - return new BlobTargetOption("ifMetagenerationMatch", match); + return new BlobTargetOption("ifGenerationMatch", match); } public static BlobTargetOption metagenerationMatch(boolean match) { @@ -174,18 +138,18 @@ class ComposeRequest implements Serializable { private final Blob target; private final List targetOptions; - private static class SourceBlob implements Serializable { + static class SourceBlob implements Serializable { private static final long serialVersionUID = 4094962795951990439L; final String blob; - final String generation; + final Long generation; SourceBlob(String blob) { this(blob, null); } - SourceBlob(String blob, String generation) { + SourceBlob(String blob, Long generation) { this.blob = blob; this.generation = generation; } @@ -214,8 +178,8 @@ public Builder addSource(String... blobs) { return addSource(Arrays.asList(blobs)); } - public Builder addSource(String blob, String generation) { - sourceBlobs.add(new SourceBlob(blob, generation)); + public Builder addSource(String blob, long matchGeneration) { + sourceBlobs.add(new SourceBlob(blob, matchGeneration)); return this; } @@ -230,14 +194,17 @@ public Builder targetOptions(BlobTargetOption... options) { } public ComposeRequest build() { + checkNotNull(bucket); + checkNotNull(target); + checkBlobOptions("Target", target, targetOptions); return new ComposeRequest(this); } } private ComposeRequest(Builder builder) { - sourceBucket = checkNotNull(builder.bucket); + sourceBucket = builder.bucket; sourceBlobs = ImmutableList.copyOf(builder.sourceBlobs); - target = checkNotNull(builder.target); + target = builder.target; targetOptions = ImmutableList.copyOf(builder.targetOptions); } @@ -298,6 +265,10 @@ public Builder targetOptions(BlobTargetOption... options) { } public CopyRequest build() { + checkNotNull(source); + checkNotNull(target); + checkBlobOptions("Source", source, sourceOptions); + checkBlobOptions("Target", target, targetOptions); return new CopyRequest(this); } } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 78acb40d5b9a..f5c25443eeea 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -16,13 +16,16 @@ package com.google.gcloud.storage; +import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; import com.google.gcloud.BaseService; import com.google.gcloud.spi.StorageRpc; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; +import java.util.List; final class StorageServiceImpl extends BaseService implements StorageService { @@ -43,7 +46,7 @@ final class StorageServiceImpl extends BaseService implem @Override public Bucket create(Bucket bucket, BucketTargetOption... options) { try { - return Bucket.fromPb(storageRpc.create(bucket.toPb())); + return Bucket.fromPb(storageRpc.create(bucket.toPb(), options)); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -52,7 +55,7 @@ public Bucket create(Bucket bucket, BucketTargetOption... options) { @Override public Blob create(Blob blob, ByteBuffer content, BlobTargetOption... options) { try { - return Blob.fromPb(storageRpc.create(blob.toPb(), content)); + return Blob.fromPb(storageRpc.create(blob.toPb(), content, options)); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -61,7 +64,7 @@ public Blob create(Blob blob, ByteBuffer content, BlobTargetOption... options) { @Override public Bucket get(String bucket, BucketSourceOption... options) { try { - return Bucket.fromPb(storageRpc.get(bucket)); + return Bucket.fromPb(storageRpc.get(bucket, options)); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -70,7 +73,7 @@ public Bucket get(String bucket, BucketSourceOption... options) { @Override public Blob get(String bucket, String blob, BlobSourceOption... options) { try { - return Blob.fromPb(storageRpc.get(bucket, blob)); + return Blob.fromPb(storageRpc.get(bucket, blob, options)); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -101,7 +104,7 @@ public Iterator list(String bucket, ListOptions settings) { @Override public Bucket update(Bucket bucket, BucketTargetOption... options) { try { - return Bucket.fromPb(storageRpc.patch(bucket.toPb())); + return Bucket.fromPb(storageRpc.patch(bucket.toPb(), options)); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -110,7 +113,7 @@ public Bucket update(Bucket bucket, BucketTargetOption... options) { @Override public Blob update(Blob blob, BlobTargetOption... options) { try { - return Blob.fromPb(storageRpc.patch(blob.toPb())); + return Blob.fromPb(storageRpc.patch(blob.toPb(), options)); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -119,7 +122,7 @@ public Blob update(Blob blob, BlobTargetOption... options) { @Override public void delete(Bucket bucket, BucketSourceOption... options) { try { - storageRpc.delete(bucket.toPb()); + storageRpc.delete(bucket.toPb(), options); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -128,7 +131,7 @@ public void delete(Bucket bucket, BucketSourceOption... options) { @Override public void delete(Blob blob, BlobSourceOption... options) { try { - storageRpc.delete(blob.toPb()); + storageRpc.delete(blob.toPb(), options); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -136,22 +139,26 @@ public void delete(Blob blob, BlobSourceOption... options) { @Override public Blob compose(ComposeRequest composeRequest) { - // todo: implement (consider having XXXRequest/XXXResponse for all SPI/Rpc requests -// try { - -// -// return Blob.fromPb(storageRpc.compose(composeRequest.sourceBucket(), -// composeRequest.sourceBlobs(), composeRequest.target())); -// } catch (IOException ex) { -// throw new StorageServiceException(ex); -// } - throw new UnsupportedOperationException("bla"); + List sources = Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size()); + for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) { + sources.add(Blob.builder(composeRequest.sourceBucket(), sourceBlob.blob) + .generation(sourceBlob.generation) + .build() + .toPb()); + } + try { + return Blob.fromPb(storageRpc.compose(sources, composeRequest.target().toPb(), + composeRequest.targetOptions())); + } catch (IOException ex) { + throw new StorageServiceException(ex); + } } @Override public Blob copy(CopyRequest copyRequest) { try { - return Blob.fromPb(storageRpc.copy(copyRequest.source().toPb(), copyRequest.target().toPb())); + return Blob.fromPb(storageRpc.copy(copyRequest.source().toPb(), copyRequest.sourceOptions(), + copyRequest.target().toPb(), copyRequest.targetOptions())); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -160,7 +167,7 @@ public Blob copy(CopyRequest copyRequest) { @Override public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) { try { - return storageRpc.reader(blob.toPb()); + return storageRpc.reader(blob.toPb(), options); } catch (IOException ex) { throw new StorageServiceException(ex); } @@ -169,7 +176,7 @@ public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) { @Override public BlobWriteChannel writeTo(Blob blob, BlobTargetOption... options) { try { - return storageRpc.writer(blob.toPb()); + return storageRpc.writer(blob.toPb(), options); } catch (IOException ex) { throw new StorageServiceException(ex); } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java index a1b61bcc00d4..10b54d79e413 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java @@ -30,6 +30,7 @@ public class StorageServiceOptions extends ServiceOptions { private static final String GCS_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control"; private static final Set SCOPES = ImmutableSet.of(GCS_SCOPE); private static final String DEFAULT_PATH_DELIMITER = "/"; + private static final String PROJECT_ENV_NAME = "default_project_id"; private final StorageRpc storageRpc; private final String project; @@ -72,7 +73,7 @@ public StorageServiceOptions build() { private StorageServiceOptions(Builder builder) { super(builder); pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER); - project = builder.project != null ? builder.project : getAppEngineProjectId(); + project = builder.project != null ? builder.project : defaultProject(); Preconditions.checkArgument(project != null, "Missing required project id"); storageRpc = MoreObjects.firstNonNull(builder.storageRpc, ServiceRpcProvider.storage(this)); } @@ -99,6 +100,14 @@ public Builder toBuilder() { return new Builder(this); } + private static String defaultProject() { + String projectId = System.getProperty(PROJECT_ENV_NAME, System.getenv(PROJECT_ENV_NAME)); + if (projectId == null) { + projectId = getAppEngineProjectId(); + } + return projectId != null ? projectId : googleCloudProjectId(); + } + public static Builder builder() { return new Builder(); } diff --git a/src/main/java/com/google/gcloud/storage/Validator.java b/src/main/java/com/google/gcloud/storage/Validator.java new file mode 100644 index 000000000000..0db8fd3be16e --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/Validator.java @@ -0,0 +1,38 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * Utility to validate Storage type/values. + */ +public class Validator { + + static void checkBlobOptions(String name, Blob blob, Iterable options) { + for (Option option : options) { + switch (option.name()) { + case "ifGenerationMatch": + checkArgument(blob.generation() != 0, "%s blob is missing generation", name); + break; + case "ifMetagenerationMatch": + checkArgument(blob.metageneration() != 0, "%s blob is missing metageneration", name); + break; + } + } + } +} From 8ef30c8163ff6add0d5e3ef8c784d6926fc3066d Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 21 Apr 2015 11:35:32 -0700 Subject: [PATCH 21/48] work in progress --- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../google/gcloud/storage/StorageService.java | 45 +++- .../storage/StorageServiceException.java | 2 +- .../gcloud/storage/StorageServiceFactory.java | 7 +- .../gcloud/storage/StorageServiceImpl.java | 221 ++++++++++-------- 6 files changed, 175 insertions(+), 104 deletions(-) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index b61f96d23c06..41bff074c523 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.StorageServiceOptions; import java.io.IOException; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory() .getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() } @Override public List buckets() throws IOException { Buckets buckets = storage.buckets().list(options.project()).execute(); return buckets.getItems(); } @Override public List objects(String bucket, String prefix, String delimiter) throws IOException { return null; } @Override public Bucket get(String bucket) throws IOException { return storage.buckets().get(bucket).execute(); } @Override public StorageObject get(String bucket, String object) throws IOException { return storage.objects().get(bucket, object).execute(); } @Override public Bucket patch(Bucket bucket) throws IOException { return storage.buckets().patch(bucket.getName(), bucket).execute(); } @Override public StorageObject patch(StorageObject storageObject) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } @Override public void delete(String bucket, String object) throws IOException { storage.objects().delete(bucket, object).execute(); } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Iterator; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false); translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Option... options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Option... options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(String bucket, Option... options) { try { return storage.buckets().get(bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(String bucket, String object, Option... options) { try { return storage.objects().get(bucket, object).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Option... options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Option... options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Option... options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Option... options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException { try { return null; } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException { try { return null; } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 74b0d55aa02e..681f6206db96 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; import java.util.List; public interface StorageRpc { Bucket create(Bucket bucket, Option... options) throws IOException; StorageObject create(StorageObject object, ByteBuffer content, Option... options) throws IOException; Iterator list() throws IOException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws IOException; Bucket get(String bucket, Option... options) throws IOException; StorageObject get(String bucket, String object, Option... options) throws IOException; Bucket patch(Bucket bucket, Option... options) throws IOException; StorageObject patch(StorageObject storageObject, Option... options) throws IOException; void delete(Bucket bucket, Option... options) throws IOException; void delete(StorageObject object, Option... options) throws IOException; StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws IOException; StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws IOException; BlobReadChannel reader(StorageObject from, Option... options) throws IOException; BlobWriteChannel writer(StorageObject to, Option... options) throws IOException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import java.util.Iterator; import java.util.List; public interface StorageRpc { Bucket create(Bucket bucket, Option... options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Option... options) throws StorageServiceException; Iterator list() throws StorageServiceException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws StorageServiceException; Bucket get(String bucket, Option... options) throws StorageServiceException; StorageObject get(String bucket, String object, Option... options) throws StorageServiceException; Bucket patch(Bucket bucket, Option... options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Option... options) throws StorageServiceException; void delete(Bucket bucket, Option... options) throws StorageServiceException; void delete(StorageObject object, Option... options) throws StorageServiceException; StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException; StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index c78056c57168..5271319b7246 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -23,7 +23,6 @@ import com.google.gcloud.Service; import java.io.Serializable; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -301,31 +300,73 @@ public static Builder builder() { } } + /** + * @throws StorageServiceException upon failure + */ Bucket create(Bucket bucket, BucketTargetOption... option); - Blob create(Blob blob, ByteBuffer content, BlobTargetOption... option); + /** + * @throws StorageServiceException upon failure + */ + Blob create(Blob blob, byte[] content, BlobTargetOption... option); + /** + * @throws StorageServiceException upon failure + */ Bucket get(String bucket, BucketSourceOption... options); + /** + * @throws StorageServiceException upon failure + */ Blob get(String bucket, String blob, BlobSourceOption... options); + /** + * @throws StorageServiceException upon failure + */ Iterator list(); + /** + * @throws StorageServiceException upon failure + */ Iterator list(String bucket, ListOptions settings); + /** + * @throws StorageServiceException upon failure + */ Bucket update(Bucket bucket, BucketTargetOption... option); + /** + * @throws StorageServiceException upon failure + */ Blob update(Blob blob, BlobTargetOption... option); + /** + * @throws StorageServiceException upon failure + */ void delete(Bucket bucket, BucketSourceOption... option); + /** + * @throws StorageServiceException upon failure + */ void delete(Blob blob, BlobSourceOption... option); + /** + * @throws StorageServiceException upon failure + */ Blob compose(ComposeRequest composeRequest); + /** + * @throws StorageServiceException upon failure + */ Blob copy(CopyRequest copyRequest); + /** + * @throws StorageServiceException upon failure + */ BlobReadChannel readFrom(Blob blob, BlobSourceOption... option); + /** + * @throws StorageServiceException upon failure + */ BlobWriteChannel writeTo(Blob blob, BlobTargetOption... option); } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceException.java b/src/main/java/com/google/gcloud/storage/StorageServiceException.java index 8d181b182637..4a3438819250 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceException.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceException.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.IOException; public class StorageServiceException extends RuntimeException { StorageServiceException(IOException ex) { super(ex); } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; /** * Storage service exception. * * @see Google Cloud Storage error codes */ public class StorageServiceException extends RuntimeException { private static final long serialVersionUID = -3748432005065428084L; private final int code; private final boolean retryable; public StorageServiceException(int code, String message, boolean retryable) { super(message); this.code = code; this.retryable = retryable; } /** * Returns the code associated with this exception. */ public int code() { return code; } public boolean retryable() { return retryable; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java index 7f94334b3c08..2c2053b72d6b 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java @@ -21,14 +21,13 @@ public abstract class StorageServiceFactory { private static final StorageServiceFactory INSTANCE = new StorageServiceFactory() { - @Override - public StorageService get(StorageServiceOptions options) { + @Override public StorageService get(StorageServiceOptions options) { return new StorageServiceImpl(options); } }; - public static StorageService getDefault(StorageServiceOptions options) { - return INSTANCE.get(options); + public static StorageServiceFactory instance() { + return INSTANCE; } public abstract StorageService get(StorageServiceOptions options); diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index f5c25443eeea..fa0898bc5fc7 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -16,169 +16,200 @@ package com.google.gcloud.storage; +import static com.google.gcloud.RetryHelper.runWithRetries; + import com.google.api.services.storage.model.StorageObject; +import com.google.common.base.MoreObjects; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.gcloud.BaseService; +import com.google.gcloud.ExceptionHandler; +import com.google.gcloud.ExceptionHandler.Interceptor; +import com.google.gcloud.RetryParams; import com.google.gcloud.spi.StorageRpc; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Iterator; import java.util.List; +import java.util.concurrent.Callable; final class StorageServiceImpl extends BaseService implements StorageService { + private static final Interceptor EXCEPTION_HANDLER_INTERCEPTOR = + new Interceptor() { + + private static final long serialVersionUID = -7758580330857881124L; + + @Override + public RetryResult afterEval(Exception exception, RetryResult retryResult) { + return null; + } + + @Override + public RetryResult beforeEval(Exception exception) { + if (exception instanceof StorageServiceException) { + boolean retriable = ((StorageServiceException) exception).retryable(); + return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT; + } + return null; + } + }; + private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder() + .abortOn(RuntimeException.class) + .interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build(); + private final StorageRpc storageRpc; + private final RetryParams retryParams; StorageServiceImpl(StorageServiceOptions options) { super(options); storageRpc = options.storageRpc(); - - // todo: like Datastore distinct exception to retriable and non-retriable - // https://cloud.google.com/storage/docs/json_api/v1/status-codes - // todo: Use retry helper on retriable failures - - // todo: consider options + retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries()); // todo: replace nulls with Value.asNull (per toPb) } @Override - public Bucket create(Bucket bucket, BucketTargetOption... options) { - try { - return Bucket.fromPb(storageRpc.create(bucket.toPb(), options)); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public Bucket create(Bucket bucket, final BucketTargetOption... options) { + final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); + return Bucket.fromPb(runWithRetries( + new Callable() { + @Override public com.google.api.services.storage.model.Bucket call() { + return storageRpc.create(bucketPb, options); + } + }, retryParams, EXCEPTION_HANDLER)); } @Override - public Blob create(Blob blob, ByteBuffer content, BlobTargetOption... options) { - try { - return Blob.fromPb(storageRpc.create(blob.toPb(), content, options)); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public Blob create(Blob blob, final byte[] content, final BlobTargetOption... options) { + final StorageObject blobPb = blob.toPb(); + return Blob.fromPb(runWithRetries(new Callable() { + @Override public StorageObject call() { + return storageRpc.create(blobPb, content, options); + } + }, retryParams, EXCEPTION_HANDLER)); } @Override - public Bucket get(String bucket, BucketSourceOption... options) { - try { - return Bucket.fromPb(storageRpc.get(bucket, options)); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public Bucket get(final String bucket, final BucketSourceOption... options) { + return Bucket.fromPb(runWithRetries( + new Callable() { + @Override public com.google.api.services.storage.model.Bucket call() { + return storageRpc.get(bucket, options); + } + }, retryParams, EXCEPTION_HANDLER)); } @Override - public Blob get(String bucket, String blob, BlobSourceOption... options) { - try { - return Blob.fromPb(storageRpc.get(bucket, blob, options)); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public Blob get(final String bucket, final String blob, final BlobSourceOption... options) { + return Blob.fromPb(runWithRetries(new Callable() { + @Override public StorageObject call() { + return storageRpc.get(bucket, blob, options); + } + }, retryParams, EXCEPTION_HANDLER)); } @Override public Iterator list() { - try { - return Iterators.transform(storageRpc.list(), Bucket.FROM_PB_FUNCTION); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + return Iterators.transform(runWithRetries( + new Callable>() { + @Override public Iterator call() { + return storageRpc.list(); + } + }, retryParams, EXCEPTION_HANDLER), + Bucket.FROM_PB_FUNCTION); } @Override public Iterator list(String bucket, ListOptions settings) { - try { - String delimiter = settings.recursive() ? options().pathDelimiter() : null; - return Iterators.transform( - storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(), - settings.includeOlderVersions(), settings.maxResults()), - Blob.FROM_PB_FUNCTION); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + // todo implement paging (with retries) with if limit is not given or > X + String delimiter = settings.recursive() ? options().pathDelimiter() : null; + return Iterators.transform( + storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(), + settings.includeOlderVersions(), settings.maxResults()), + Blob.FROM_PB_FUNCTION); } @Override - public Bucket update(Bucket bucket, BucketTargetOption... options) { - try { - return Bucket.fromPb(storageRpc.patch(bucket.toPb(), options)); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public Bucket update(Bucket bucket, final BucketTargetOption... options) { + final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); + return Bucket.fromPb(runWithRetries( + new Callable() { + @Override public com.google.api.services.storage.model.Bucket call() { + return storageRpc.patch(bucketPb, options); + } + }, retryParams, EXCEPTION_HANDLER)); } @Override - public Blob update(Blob blob, BlobTargetOption... options) { - try { - return Blob.fromPb(storageRpc.patch(blob.toPb(), options)); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public Blob update(Blob blob, final BlobTargetOption... options) { + final StorageObject storageObject = blob.toPb(); + return Blob.fromPb(runWithRetries(new Callable() { + @Override public StorageObject call() { + return storageRpc.patch(storageObject, options); + } + }, retryParams, EXCEPTION_HANDLER)); } @Override - public void delete(Bucket bucket, BucketSourceOption... options) { - try { - storageRpc.delete(bucket.toPb(), options); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public void delete(Bucket bucket, final BucketSourceOption... options) { + final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); + runWithRetries(new Callable() { + @Override public Void call() { + storageRpc.delete(bucketPb, options); + return null; + } + }, retryParams, EXCEPTION_HANDLER); } @Override - public void delete(Blob blob, BlobSourceOption... options) { - try { - storageRpc.delete(blob.toPb(), options); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public void delete(Blob blob, final BlobSourceOption... options) { + final StorageObject storageObject = blob.toPb(); + runWithRetries(new Callable() { + @Override public Void call() { + storageRpc.delete(storageObject, options); + return null; + } + }, retryParams, EXCEPTION_HANDLER); } @Override - public Blob compose(ComposeRequest composeRequest) { - List sources = Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size()); + public Blob compose(final ComposeRequest composeRequest) { + final List sources = + Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size()); for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) { sources.add(Blob.builder(composeRequest.sourceBucket(), sourceBlob.blob) .generation(sourceBlob.generation) .build() .toPb()); } - try { - return Blob.fromPb(storageRpc.compose(sources, composeRequest.target().toPb(), - composeRequest.targetOptions())); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + final StorageObject target = composeRequest.target().toPb(); + return Blob.fromPb(runWithRetries(new Callable() { + @Override public StorageObject call() { + return storageRpc.compose(sources, target, composeRequest.targetOptions()); + } + }, retryParams, EXCEPTION_HANDLER)); } @Override - public Blob copy(CopyRequest copyRequest) { - try { - return Blob.fromPb(storageRpc.copy(copyRequest.source().toPb(), copyRequest.sourceOptions(), - copyRequest.target().toPb(), copyRequest.targetOptions())); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + public Blob copy(final CopyRequest copyRequest) { + final StorageObject source = copyRequest.source().toPb(); + final StorageObject target = copyRequest.target().toPb(); + return Blob.fromPb(runWithRetries(new Callable() { + @Override public StorageObject call() { + return storageRpc.copy(source, copyRequest.sourceOptions(), target, + copyRequest.targetOptions()); + } + }, retryParams, EXCEPTION_HANDLER)); } @Override public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) { - try { - return storageRpc.reader(blob.toPb(), options); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + // todo: Use retry helper on retriable failures + return storageRpc.reader(blob.toPb(), options); } @Override public BlobWriteChannel writeTo(Blob blob, BlobTargetOption... options) { - try { - return storageRpc.writer(blob.toPb(), options); - } catch (IOException ex) { - throw new StorageServiceException(ex); - } + // todo: Use retry helper on retriable failures + return storageRpc.writer(blob.toPb(), options); } } From 89a50240860944602a489f20a89c1f07ef946ae5 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 21 Apr 2015 11:42:59 -0700 Subject: [PATCH 22/48] work in progress --- src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 41bff074c523..7ae46db6cc99 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Iterator; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false); translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Option... options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Option... options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(String bucket, Option... options) { try { return storage.buckets().get(bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(String bucket, String object, Option... options) { try { return storage.objects().get(bucket, object).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Option... options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Option... options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Option... options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Option... options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException { try { return null; } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException { try { return null; } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false); translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Option... options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Option... options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(String bucket, Option... options) { try { return storage.buckets().get(bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(String bucket, String object, Option... options) { try { return storage.objects().get(bucket, object).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Option... options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Option... options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Option... options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Option... options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest try { return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException { try { return storage.objects() .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file From 2841929248e58d8c1cc9af128b224d37efd10daf Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 21 Apr 2015 17:35:00 -0700 Subject: [PATCH 23/48] work in progress --- .../gcloud/examples/StorageExample.java | 171 +++++++++++++----- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../java/com/google/gcloud/storage/Blob.java | 6 +- .../com/google/gcloud/storage/Bucket.java | 4 + .../google/gcloud/storage/ListOptions.java | 2 +- .../google/gcloud/storage/StorageService.java | 4 +- .../gcloud/storage/StorageServiceImpl.java | 10 +- 8 files changed, 141 insertions(+), 60 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index f670cc7ac1db..3c018f40b7d2 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -18,12 +18,14 @@ import com.google.gcloud.storage.Blob; import com.google.gcloud.storage.Bucket; +import com.google.gcloud.storage.ListOptions; import com.google.gcloud.storage.StorageService; import com.google.gcloud.storage.StorageServiceFactory; import com.google.gcloud.storage.StorageServiceOptions; import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; /** @@ -36,87 +38,156 @@ *
  • compile using maven - {@code mvn compile}
    • *
  • run using maven - {@code mvn exec:java * -Dexec.mainClass="com.google.gcloud.examples.StorageExample" - * -Dexec.args=" [] [list|get |put []|delete ...]"}
    • + * -Dexec.args="list []|info [ []]|get |upload []|delete "} * */ public class StorageExample { - private static final String DEFAULT_FOLDER = "_STORAGE_EXAMPLE"; - private static final String DEFAULT_ACTION = "list"; private static final Map ACTIONS = new HashMap<>(); - private static abstract class StorageAction { + private static abstract class StorageAction { - abstract void run(StorageService storage, Bucket bucket, String folder, String... args); + abstract void run(StorageService storage, T request); - protected String getRequiredParams() { + abstract T parse(String... args); + + protected String params() { return ""; } + } + + private static abstract class BlobAction extends StorageAction { - protected String fullPath(String folder, String file) { - StringBuilder stringBuilder = new StringBuilder(folder); - if (!folder.endsWith("/")) { - stringBuilder.append('/'); + @Override + Blob parse(String... args) { + if (args.length != 2) { + throw new IllegalArgumentException(); } - return stringBuilder.append(file).toString(); + return Blob.of(args[0], args[1]); + } + + @Override + public String params() { + return " "; } } - private static class DeleteAction extends StorageAction { + private static class InfoAction extends BlobAction { + @Override + public void run(StorageService storage, Blob blob) { + if (blob.name().isEmpty()) { + System.out.println(storage.get(Bucket.of(blob.bucket()))); + } else { + System.out.println(storage.get(blob)); + } + } + @Override - public void run(StorageService storage, Bucket bucket, String folder, String... args) { - for (String file : args) { - storage.delete(Blob.builder(bucket, fullPath(folder, file)).build()); + Blob parse(String... args) { + if (args.length < 2) { + return Blob.of(args[0], ""); } + return super.parse(args); } @Override - public String getRequiredParams() { - return "..."; + public String params() { + return " []"; } } - static { - ACTIONS.put("delete", new DeleteAction()); - // todo: implement list, get and put + private static class DeleteAction extends BlobAction { + @Override + public void run(StorageService storage, Blob blob) { + storage.delete(blob); + } + + @Override + public String params() { + return " "; + } } - public static void main(String... args) { - StorageAction action = null; - StorageService storage = null; - Bucket bucket = null; - String folder = DEFAULT_FOLDER; - if (args.length > 0) { - String bucketName = args[0]; - String actionName = DEFAULT_ACTION; - if (args.length > 1) { - folder = args[1]; - if (args.length > 2) { - actionName = args[2].toLowerCase(); - } - storage = StorageServiceFactory.getDefault(StorageServiceOptions.builder().build()); - bucket = storage.get(bucketName); + private static class ListAction extends StorageAction { + + @Override + String parse(String... args) { + if (args.length == 0) { + return null; + } + if (args.length == 1) { + return args[0]; } - action = ACTIONS.get(actionName); + throw new IllegalArgumentException(); } - if (action == null) { - StringBuilder actionAndParams = new StringBuilder(); - for (Map.Entry entry : ACTIONS.entrySet()) { - actionAndParams.append(entry.getKey()); - String param = entry.getValue().getRequiredParams(); - if (param != null && !param.isEmpty()) { - actionAndParams.append(' ').append(param); + @Override + public void run(StorageService storage, String bucket) { + if (bucket == null) { + Iterator buckets = storage.list(); + while (buckets.hasNext()) { + System.out.println(buckets.next()); + } + } else { + Iterator blobs = storage.list(bucket, ListOptions.of()); + while (blobs.hasNext()) { + System.out.println(blobs.next()); } - actionAndParams.append('|'); } - actionAndParams.setLength(actionAndParams.length() - 1); - System.out.printf("Usage: %s bucket [] [%s]%n", - StorageExample.class.getSimpleName(), actionAndParams); - return; } - args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{}; - action.run(storage, bucket, folder, args); + @Override + public String params() { + return "[]"; + } + } + + static { + ACTIONS.put("info", new InfoAction()); + ACTIONS.put("delete", new DeleteAction()); + ACTIONS.put("list", new ListAction()); + //ACTIONS.put("upload", new UploadAction()); + // todo: implement get + } + + public static void printUsage() { + StringBuilder actionAndParams = new StringBuilder(); + for (Map.Entry entry : ACTIONS.entrySet()) { + actionAndParams.append(entry.getKey()); + + String param = entry.getValue().params(); + if (param != null && !param.isEmpty()) { + actionAndParams.append(' ').append(param); + } + actionAndParams.append('|'); + } + actionAndParams.setLength(actionAndParams.length() - 1); + System.out.printf("Usage: %s [%s]%n", + StorageExample.class.getSimpleName(), actionAndParams); + } + public static void main(String... args) { + if (args.length == 0) { + System.out.println("Missing required action"); + printUsage(); + return; + } + StorageAction action = ACTIONS.get(args[0]); + if (action == null) { + System.out.println("Unrecognized action '" + args[0] + "'"); + printUsage(); + return; + } + StorageService storage = + StorageServiceFactory.instance().get(StorageServiceOptions.builder().build()); + args = args.length > 1 ? Arrays.copyOfRange(args, 1, args.length): new String []{}; + Object request; + try { + request = action.parse(args); + } catch (Exception ex) { + System.out.println("Invalid input for action '" + args[0] + "'"); + System.out.println("Expected: " + action.params()); + return; + } + action.run(storage, request); } } diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 7ae46db6cc99..e2ad7e19b431 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false); translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Option... options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Option... options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(String bucket, Option... options) { try { return storage.buckets().get(bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(String bucket, String object, Option... options) { try { return storage.objects().get(bucket, object).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Option... options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Option... options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Option... options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Option... options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest try { return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException { try { return storage.objects() .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false); translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Option... options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Option... options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Option... options) { try { return storage.buckets().get(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Option... options) { try { return storage.objects().get(object.getBucket(), object.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Option... options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Option... options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Option... options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Option... options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest try { return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException { try { return storage.objects() .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 681f6206db96..d051438244d2 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import java.util.Iterator; import java.util.List; public interface StorageRpc { Bucket create(Bucket bucket, Option... options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Option... options) throws StorageServiceException; Iterator list() throws StorageServiceException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws StorageServiceException; Bucket get(String bucket, Option... options) throws StorageServiceException; StorageObject get(String bucket, String object, Option... options) throws StorageServiceException; Bucket patch(Bucket bucket, Option... options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Option... options) throws StorageServiceException; void delete(Bucket bucket, Option... options) throws StorageServiceException; void delete(StorageObject object, Option... options) throws StorageServiceException; StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException; StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import java.util.Iterator; import java.util.List; public interface StorageRpc { Bucket create(Bucket bucket, Option... options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Option... options) throws StorageServiceException; Iterator list() throws StorageServiceException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws StorageServiceException; Bucket get(Bucket bucket, Option... options) throws StorageServiceException; StorageObject get(StorageObject object, Option... options) throws StorageServiceException; Bucket patch(Bucket bucket, Option... options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Option... options) throws StorageServiceException; void delete(Bucket bucket, Option... options) throws StorageServiceException; void delete(StorageObject object, Option... options) throws StorageServiceException; StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException; StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java index 04553a8c744e..0e7496a72a06 100644 --- a/src/main/java/com/google/gcloud/storage/Blob.java +++ b/src/main/java/com/google/gcloud/storage/Blob.java @@ -356,7 +356,11 @@ public Builder toBuilder() { .selfLink(selfLink); } - public static Builder builder(Bucket bucket, String name) { + public static Blob of(String bucket, String name) { + return builder(bucket, name).build(); + } + + public static Builder builder(Bucket bucket, String name) { return builder(bucket.name(), name); } diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index 55586acc3cbb..6bfe63e6bd9c 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -536,6 +536,10 @@ public Builder toBuilder() { .deleteRules(deleteRules); } + public static Bucket of(String name) { + return builder(name).build(); + } + public static Builder builder(String name) { return new Builder().name(name); } diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java index 06637ed6aa3b..05257babca00 100644 --- a/src/main/java/com/google/gcloud/storage/ListOptions.java +++ b/src/main/java/com/google/gcloud/storage/ListOptions.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; private final boolean includeOlderVersions; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; private boolean includeOlderVersions; public Builder() { } public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } public Builder includeOlderVersions(boolean include) { this.includeOlderVersions = include; return this; } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; includeOlderVersions = builder.includeOlderVersions; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } public boolean includeOlderVersions() { return includeOlderVersions; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; private final boolean includeOlderVersions; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; private boolean includeOlderVersions; private Builder() { } public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } public Builder includeOlderVersions(boolean include) { this.includeOlderVersions = include; return this; } public ListOptions build() { return new ListOptions(this); } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; includeOlderVersions = builder.includeOlderVersions; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } public boolean includeOlderVersions() { return includeOlderVersions; } public static ListOptions of() { return builder().build(); } public static Builder builder() { return new Builder(); } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 5271319b7246..ca3752e08b28 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -313,12 +313,12 @@ public static Builder builder() { /** * @throws StorageServiceException upon failure */ - Bucket get(String bucket, BucketSourceOption... options); + Bucket get(Bucket bucket, BucketSourceOption... options); /** * @throws StorageServiceException upon failure */ - Blob get(String bucket, String blob, BlobSourceOption... options); + Blob get(Blob blob, BlobSourceOption... options); /** * @throws StorageServiceException upon failure diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index fa0898bc5fc7..7d653a7fb25a 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -89,20 +89,22 @@ public Blob create(Blob blob, final byte[] content, final BlobTargetOption... op } @Override - public Bucket get(final String bucket, final BucketSourceOption... options) { + public Bucket get(Bucket bucket, final BucketSourceOption... options) { + final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); return Bucket.fromPb(runWithRetries( new Callable() { @Override public com.google.api.services.storage.model.Bucket call() { - return storageRpc.get(bucket, options); + return storageRpc.get(bucketPb, options); } }, retryParams, EXCEPTION_HANDLER)); } @Override - public Blob get(final String bucket, final String blob, final BlobSourceOption... options) { + public Blob get(Blob blob, final BlobSourceOption... options) { + final StorageObject storedObject = blob.toPb(); return Blob.fromPb(runWithRetries(new Callable() { @Override public StorageObject call() { - return storageRpc.get(bucket, blob, options); + return storageRpc.get(storedObject, options); } }, retryParams, EXCEPTION_HANDLER)); } From 803527d695eafdfaa71a0597cab53a135c8512c3 Mon Sep 17 00:00:00 2001 From: ozarov Date: Tue, 21 Apr 2015 20:18:37 -0700 Subject: [PATCH 24/48] work in progress --- .../gcloud/examples/StorageExample.java | 91 +++++++++++++++---- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../java/com/google/gcloud/storage/Acl.java | 4 +- .../java/com/google/gcloud/storage/Blob.java | 6 ++ .../com/google/gcloud/storage/Bucket.java | 6 ++ 5 files changed, 87 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 3c018f40b7d2..2fbaee752373 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -16,13 +16,11 @@ package com.google.gcloud.examples; -import com.google.gcloud.storage.Blob; -import com.google.gcloud.storage.Bucket; -import com.google.gcloud.storage.ListOptions; -import com.google.gcloud.storage.StorageService; -import com.google.gcloud.storage.StorageServiceFactory; -import com.google.gcloud.storage.StorageServiceOptions; +import com.google.gcloud.storage.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; @@ -38,7 +36,7 @@ *
  • compile using maven - {@code mvn compile}
    • *
  • run using maven - {@code mvn exec:java * -Dexec.mainClass="com.google.gcloud.examples.StorageExample" - * -Dexec.args="list []|info [ []]|get |upload []|delete "}
    • + * -Dexec.args="project_id list []|info [ []]|get |upload []|delete "} * */ public class StorageExample { @@ -47,15 +45,39 @@ public class StorageExample { private static abstract class StorageAction { - abstract void run(StorageService storage, T request); + abstract void run(StorageService storage, T request) throws Exception; - abstract T parse(String... args); + abstract T parse(String... args) throws IllegalArgumentException; protected String params() { return ""; } } + private static class Tuple { + + private final X x; + private final Y y; + + private Tuple(X x, Y y) { + this.x = x; + this.y = y; + } + + static Tuple of(X x, Y y) { + return new Tuple<>(x, y); + } + + X first() { + return x; + } + + Y second() { + return y; + } + } + + private static abstract class BlobAction extends StorageAction { @Override @@ -142,11 +164,39 @@ public String params() { } } + private static class UploadAction extends StorageAction> { + @Override + public void run(StorageService storage, Tuple tuple) throws Exception { + if (Files.size(tuple.first()) > 1_000_000) { + // todo upload via streaming API + throw new IllegalArgumentException("file is too big"); + } else { + byte[] bytes = Files.readAllBytes(tuple.first()); + System.out.println(storage.create(tuple.second(), bytes)); + } + } + + @Override + Tuple parse(String... args) { + if (args.length < 2 || args.length > 3) { + throw new IllegalArgumentException(); + } + Path path = Paths.get(args[0]); + String blob = args.length < 3 ? path.getFileName().toString() : args[2]; + return Tuple.of(path, Blob.of(args[1], blob)); + } + + @Override + public String params() { + return " []"; + } + } + static { ACTIONS.put("info", new InfoAction()); ACTIONS.put("delete", new DeleteAction()); ACTIONS.put("list", new ListAction()); - //ACTIONS.put("upload", new UploadAction()); + ACTIONS.put("upload", new UploadAction()); // todo: implement get } @@ -165,26 +215,29 @@ public static void printUsage() { System.out.printf("Usage: %s [%s]%n", StorageExample.class.getSimpleName(), actionAndParams); } - public static void main(String... args) { - if (args.length == 0) { - System.out.println("Missing required action"); + + @SuppressWarnings("unchecked") + public static void main(String... args) throws Exception { + if (args.length < 2) { + System.out.println("Missing required project id and action"); printUsage(); return; } - StorageAction action = ACTIONS.get(args[0]); + String projectId = args[0]; + StorageAction action = ACTIONS.get(args[1]); if (action == null) { - System.out.println("Unrecognized action '" + args[0] + "'"); + System.out.println("Unrecognized action '" + args[1] + "'"); printUsage(); return; } - StorageService storage = - StorageServiceFactory.instance().get(StorageServiceOptions.builder().build()); - args = args.length > 1 ? Arrays.copyOfRange(args, 1, args.length): new String []{}; + StorageServiceOptions options = StorageServiceOptions.builder().project(args[0]).build(); + StorageService storage = StorageServiceFactory.instance().get(options); + args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{}; Object request; try { request = action.parse(args); } catch (Exception ex) { - System.out.println("Invalid input for action '" + args[0] + "'"); + System.out.println("Invalid input for action '" + args[1] + "'"); System.out.println("Expected: " + action.params()); return; } diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index e2ad7e19b431..33f3d33924ab 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false); translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Option... options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Option... options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Option... options) { try { return storage.buckets().get(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Option... options) { try { return storage.objects().get(object.getBucket(), object.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Option... options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Option... options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Option... options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Option... options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest try { return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException { try { return storage.objects() .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.*; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); // todo: based on code consider if error is retryable translated = new StorageServiceException(details.getCode(), details.getMessage(), false); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Option... options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Option... options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Option... options) { try { return storage.buckets().get(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Option... options) { try { return storage.objects().get(object.getBucket(), object.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Option... options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Option... options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Option... options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Option... options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest try { return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException { try { return storage.objects() .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index 48bd6c787718..cb79e158d5f0 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -84,8 +84,8 @@ static Entity fromPb(String entity) { return new Group(entity.substring(6)); } if (entity.startsWith("project-")) { - int idx = entity.indexOf('-', 9); - String team = entity.substring(9, idx); + int idx = entity.indexOf('-', 8); + String team = entity.substring(8, idx); String projectId = entity.substring(idx + 1); return new Project(Project.ProjectRole.valueOf(team.toUpperCase()), projectId); } diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java index 0e7496a72a06..e6b6b150ced4 100644 --- a/src/main/java/com/google/gcloud/storage/Blob.java +++ b/src/main/java/com/google/gcloud/storage/Blob.java @@ -23,6 +23,7 @@ import com.google.api.services.storage.model.StorageObject; import com.google.api.services.storage.model.StorageObject.Owner; import com.google.common.base.Function; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -356,6 +357,11 @@ public Builder toBuilder() { .selfLink(selfLink); } + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("bucket", bucket).add("name", name).toString(); + } + public static Blob of(String bucket, String name) { return builder(bucket, name).build(); } diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index 6bfe63e6bd9c..1c3077e41bb6 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -28,6 +28,7 @@ import com.google.api.services.storage.model.BucketAccessControl; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.common.base.Function; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.Acl.Entity; @@ -536,6 +537,11 @@ public Builder toBuilder() { .deleteRules(deleteRules); } + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("name", name).toString(); + } + public static Bucket of(String name) { return builder(name).build(); } From f327dad7fc163181aa7049d687c5361ff5b82101 Mon Sep 17 00:00:00 2001 From: ozarov Date: Wed, 22 Apr 2015 17:37:14 -0700 Subject: [PATCH 25/48] work in progress --- .../com/google/gcloud/AuthCredentials.java | 18 +- .../com/google/gcloud/ExceptionHandler.java | 29 ++- .../java/com/google/gcloud/RetryHelper.java | 10 +- .../java/com/google/gcloud/RetryParams.java | 8 +- .../com/google/gcloud/ServiceOptions.java | 2 +- .../gcloud/examples/StorageExample.java | 41 ++-- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../google/gcloud/spi/StorageRpcFactory.java | 2 +- .../java/com/google/gcloud/storage/Acl.java | 14 +- .../java/com/google/gcloud/storage/Blob.java | 24 ++- .../gcloud/storage/BlobReadChannel.java | 4 +- .../gcloud/storage/BlobWriteChannel.java | 6 +- .../com/google/gcloud/storage/Bucket.java | 27 +-- .../java/com/google/gcloud/storage/Cors.java | 15 +- .../google/gcloud/storage/ListOptions.java | 2 +- .../com/google/gcloud/storage/Option.java | 31 ++- .../google/gcloud/storage/StorageService.java | 60 +++--- .../storage/StorageServiceException.java | 2 +- .../gcloud/storage/StorageServiceFactory.java | 9 +- .../gcloud/storage/StorageServiceImpl.java | 183 ++++++++++++------ .../gcloud/storage/StorageServiceOptions.java | 3 +- .../com/google/gcloud/storage/Validator.java | 22 ++- .../google/gcloud/storage/package-info.java | 1 + .../google/gcloud/ExceptionHandlerTest.java | 3 +- 25 files changed, 314 insertions(+), 206 deletions(-) diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java index 0514c26a4e7e..3e6c3651c576 100644 --- a/src/main/java/com/google/gcloud/AuthCredentials.java +++ b/src/main/java/com/google/gcloud/AuthCredentials.java @@ -42,8 +42,8 @@ public abstract class AuthCredentials { private static class AppEngineAuthCredentials extends AuthCredentials { @Override - protected HttpRequestInitializer httpRequestInitializer( - HttpTransport transport, Set scopes) { + protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, + Set scopes) { return new AppIdentityCredential(scopes); } } @@ -85,8 +85,8 @@ public static AuthCredentials createForAppEngine() { return new AppEngineAuthCredentials(); } - public static AuthCredentials createForComputeEngine() - throws IOException, GeneralSecurityException { + public static AuthCredentials createForComputeEngine() throws IOException, + GeneralSecurityException { final ComputeCredential cred = getComputeCredential(); return new AuthCredentials() { @Override @@ -100,10 +100,12 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, /** * Returns the Application Default Credentials. * - *

    Returns the Application Default Credentials which are credentials that identify and - * authorize the whole application. This is the built-in service account if running on Google - * Compute Engine or the credentials file from the path in the environment variable - * GOOGLE_APPLICATION_CREDENTIALS.

    + *

    + * Returns the Application Default Credentials which are credentials that identify and authorize + * the whole application. This is the built-in service account if running on Google Compute Engine + * or the credentials file from the path in the environment variable + * GOOGLE_APPLICATION_CREDENTIALS. + *

    * * @return the credentials instance. * @throws IOException if the credentials cannot be created in the current environment. diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java index 0e3f73d590fd..e4799934fa9c 100644 --- a/src/main/java/com/google/gcloud/ExceptionHandler.java +++ b/src/main/java/com/google/gcloud/ExceptionHandler.java @@ -49,8 +49,7 @@ public interface Interceptor extends Serializable { enum RetryResult { - RETRY(true), - ABORT(false); + RETRY(true), ABORT(false); private final boolean booleanValue; @@ -67,9 +66,9 @@ boolean booleanValue() { * This method is called before exception evaluation and could short-circuit the process. * * @param exception the exception that is being evaluated - * @return {@link RetryResult} to indicate if the exception should be ignored - * ({@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), - * or evaluation should proceed ({@code null}). + * @return {@link RetryResult} to indicate if the exception should be ignored ( + * {@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), or evaluation + * should proceed ({@code null}). */ RetryResult beforeEval(Exception exception); @@ -78,9 +77,9 @@ boolean booleanValue() { * * @param exception the exception that is being evaluated * @param retryResult the result of the evaluation so far. - * @return {@link RetryResult} to indicate if the exception should be ignored - * ({@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), - * or evaluation should proceed ({@code null}). + * @return {@link RetryResult} to indicate if the exception should be ignored ( + * {@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), or evaluation + * should proceed ({@code null}). */ RetryResult afterEval(Exception exception, RetryResult retryResult); } @@ -96,14 +95,12 @@ public static class Builder { private final ImmutableSet.Builder> nonRetriableExceptions = ImmutableSet.builder(); - private Builder() { - } + private Builder() {} /** - * Adds the exception handler interceptors. - * Call order will be maintained. - + * Adds the exception handler interceptors. Call order will be maintained. + * * @param interceptors the interceptors for this exception handler * @return the Builder for chaining */ @@ -214,7 +211,7 @@ private static RetryInfo findMostSpecificRetryInfo(Set retryInfo, Class exception) { for (RetryInfo current : retryInfo) { if (current.exception.isAssignableFrom(exception)) { - RetryInfo match = findMostSpecificRetryInfo(current.children, exception); + RetryInfo match = findMostSpecificRetryInfo(current.children, exception); return match == null ? current : match; } } @@ -239,8 +236,8 @@ void verifyCaller(Callable callable) { for (Class exceptionOrError : callMethod.getExceptionTypes()) { Preconditions.checkArgument(Exception.class.isAssignableFrom(exceptionOrError), "Callable method exceptions must be derived from Exception"); - @SuppressWarnings("unchecked") Class exception = - (Class) exceptionOrError; + @SuppressWarnings("unchecked") + Class exception = (Class) exceptionOrError; Preconditions.checkArgument(findMostSpecificRetryInfo(retryInfo, exception) != null, "Declared exception '" + exception + "' is not covered by exception handler"); } diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java index 7e354a4145d3..7b47209cd3ff 100644 --- a/src/main/java/com/google/gcloud/RetryHelper.java +++ b/src/main/java/com/google/gcloud/RetryHelper.java @@ -35,9 +35,9 @@ import java.util.logging.Logger; /** - * Utility class for retrying operations. - * For more details about the parameters, see {@link RetryParams}. - * If the request is never successful, a {@link RetriesExhaustedException} will be thrown. + * Utility class for retrying operations. For more details about the parameters, see + * {@link RetryParams}. If the request is never successful, a {@link RetriesExhaustedException} will + * be thrown. * * @param return value of the closure that is being run with retries */ @@ -201,8 +201,8 @@ private V doRetry() throws RetryHelperException { } long sleepDurationMillis = getSleepDuration(params, attemptNumber); if (log.isLoggable(Level.FINE)) { - log.fine(this + ": Attempt #" + attemptNumber + " failed [" + exception + "], sleeping for " - + sleepDurationMillis + " ms"); + log.fine(this + ": Attempt #" + attemptNumber + " failed [" + exception + + "], sleeping for " + sleepDurationMillis + " ms"); } try { Thread.sleep(sleepDurationMillis); diff --git a/src/main/java/com/google/gcloud/RetryParams.java b/src/main/java/com/google/gcloud/RetryParams.java index c7f25926eb9f..0d38aea77ab0 100644 --- a/src/main/java/com/google/gcloud/RetryParams.java +++ b/src/main/java/com/google/gcloud/RetryParams.java @@ -25,10 +25,10 @@ import java.util.Objects; /** - * Parameters for configuring retries with an exponential backoff. - * Initial request is executed immediately. If the request fails but passes the - * {@link ExceptionHandler} criteria the calling thread sleeps for {@code initialRetryDelayMillis}. - * Each subsequent failure the sleep interval is calculated as: + * Parameters for configuring retries with an exponential backoff. Initial request is executed + * immediately. If the request fails but passes the {@link ExceptionHandler} criteria the calling + * thread sleeps for {@code initialRetryDelayMillis}. Each subsequent failure the sleep interval is + * calculated as: *

    * {@code retryDelayBackoffFactor ^ attempts * initialRetryDelayMillis} but would be upper-bounded * to {@code maxRetryDelayMillis} diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index bbc8ef935a6b..36cdf64aa7f8 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -146,7 +146,7 @@ protected static String googleCloudProjectId() { URLConnection connection = url.openConnection(); connection.setRequestProperty("X-Google-Metadata-Request", "True"); try (BufferedReader reader = - new BufferedReader(new InputStreamReader(connection.getInputStream(), UTF_8))) { + new BufferedReader(new InputStreamReader(connection.getInputStream(), UTF_8))) { return reader.readLine(); } } catch (IOException ignore) { diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 2fbaee752373..f65c8cdd02c2 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -18,6 +18,7 @@ import com.google.gcloud.storage.*; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -31,12 +32,14 @@ *

    * This example demonstrates a simple/typical usage. *

    - * Steps needed for running the example:

      + * Steps needed for running the example: + *
        *
      1. login using gcloud SDK - {@code gcloud auth login}.
        2. *
      2. compile using maven - {@code mvn compile}
        3. - *
      3. run using maven - {@code mvn exec:java - * -Dexec.mainClass="com.google.gcloud.examples.StorageExample" - * -Dexec.args="project_id list []|info [ []]|get |upload []|delete "}
        4. + *
      4. run using maven - + * {@code mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" + * -Dexec.args="project_id list []|info [ []]|get |upload []|delete "} + *
        5. *
      */ public class StorageExample { @@ -123,11 +126,6 @@ private static class DeleteAction extends BlobAction { public void run(StorageService storage, Blob blob) { storage.delete(blob); } - - @Override - public String params() { - return " "; - } } private static class ListAction extends StorageAction { @@ -168,7 +166,7 @@ private static class UploadAction extends StorageAction> { @Override public void run(StorageService storage, Tuple tuple) throws Exception { if (Files.size(tuple.first()) > 1_000_000) { - // todo upload via streaming API + // todo: upload via streaming API throw new IllegalArgumentException("file is too big"); } else { byte[] bytes = Files.readAllBytes(tuple.first()); @@ -192,12 +190,28 @@ public String params() { } } + private static class GetAction extends BlobAction { + @Override + public void run(StorageService storage, Blob blob) { + blob = storage.get(blob); + if (blob == null) { + System.out.println("No such object"); + } + if (blob.size() < 1_000_000) { + System.out.println(new String(storage.load(blob), StandardCharsets.UTF_8)); + } else { + // todo: download via streaming API + throw new IllegalArgumentException("file is too big"); + } + } + } + static { ACTIONS.put("info", new InfoAction()); ACTIONS.put("delete", new DeleteAction()); ACTIONS.put("list", new ListAction()); ACTIONS.put("upload", new UploadAction()); - // todo: implement get + ACTIONS.put("get", new GetAction()); } public static void printUsage() { @@ -212,8 +226,7 @@ public static void printUsage() { actionAndParams.append('|'); } actionAndParams.setLength(actionAndParams.length() - 1); - System.out.printf("Usage: %s [%s]%n", - StorageExample.class.getSimpleName(), actionAndParams); + System.out.printf("Usage: %s [%s]%n", StorageExample.class.getSimpleName(), actionAndParams); } @SuppressWarnings("unchecked") @@ -232,7 +245,7 @@ public static void main(String... args) throws Exception { } StorageServiceOptions options = StorageServiceOptions.builder().project(args[0]).build(); StorageService storage = StorageServiceFactory.instance().get(options); - args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{}; + args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length) : new String[] {}; Object request; try { request = action.parse(args); diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 33f3d33924ab..41d4359c73d4 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.*; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.List; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); // todo: based on code consider if error is retryable translated = new StorageServiceException(details.getCode(), details.getMessage(), false); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Option... options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Option... options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Option... options) { try { return storage.buckets().get(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Option... options) { try { return storage.objects().get(object.getBucket(), object.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Option... options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Option... options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Option... options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Option... options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest try { return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException { try { return storage.objects() .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.Map; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); // todo: based on code consider if error is retryable translated = new StorageServiceException(details.getCode(), details.getMessage(), false); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets().get(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects().get(object.getBucket(), object.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest try { return storage.objects().compose(target.getBucket(), target.getName(), null) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName()); if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) { getRequest.setIfGenerationMatch(from.getGeneration()); } else { getRequest.setIfGenerationNotMatch(from.getGeneration()); } } if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) { getRequest.setIfMetagenerationMatch(from.getMetageneration()); } else { getRequest.setIfMetagenerationNotMatch(from.getMetageneration()); } } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index d051438244d2..17646392686f 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Option; import com.google.gcloud.storage.StorageServiceException; import java.util.Iterator; import java.util.List; public interface StorageRpc { Bucket create(Bucket bucket, Option... options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Option... options) throws StorageServiceException; Iterator list() throws StorageServiceException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws StorageServiceException; Bucket get(Bucket bucket, Option... options) throws StorageServiceException; StorageObject get(StorageObject object, Option... options) throws StorageServiceException; Bucket patch(Bucket bucket, Option... options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Option... options) throws StorageServiceException; void delete(Bucket bucket, Option... options) throws StorageServiceException; void delete(StorageObject object, Option... options) throws StorageServiceException; StorageObject compose(Iterable src, StorageObject destination, List destinationOptions) throws StorageServiceException; StorageObject copy(StorageObject from, List blobSourceOptions, StorageObject to, List blobTargetOptions) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.Iterator; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_GENERATION_MATCH("ifGenerationMatch"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Iterator list() throws StorageServiceException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; void delete(Bucket bucket, Map options) throws StorageServiceException; void delete(StorageObject object, Map options) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java b/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java index e12f036a3fe3..75b618f607ae 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.gcloud.storage.StorageServiceOptions; public interface StorageRpcFactory extends ServiceRpcFactory { } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.gcloud.storage.StorageServiceOptions; public interface StorageRpcFactory extends ServiceRpcFactory { } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index cb79e158d5f0..3058386d7201 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -29,9 +29,7 @@ public final class Acl implements Serializable { private final Role role; public enum Role { - OWNER, - READER, - WRITER + OWNER, READER, WRITER } public static abstract class Entity implements Serializable { @@ -41,11 +39,7 @@ public static abstract class Entity implements Serializable { private final String value; public enum Type { - DOMAIN, - GROUP, - USER, - PROJECT, - UNKNOWN + DOMAIN, GROUP, USER, PROJECT, UNKNOWN } Entity(Type type, String value) { @@ -162,9 +156,7 @@ public static class Project extends Entity { private final String projectId; enum ProjectRole { - OWNERS, - EDITORS, - VIEWERS + OWNERS, EDITORS, VIEWERS } Project(ProjectRole pRole, String projectId) { diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java index e6b6b150ced4..6cb8e0eb8ae6 100644 --- a/src/main/java/com/google/gcloud/storage/Blob.java +++ b/src/main/java/com/google/gcloud/storage/Blob.java @@ -39,16 +39,18 @@ public class Blob implements Serializable { static final Function FROM_PB_FUNCTION = new Function() { - @Override public Blob apply(StorageObject pb) { + @Override + public Blob apply(StorageObject pb) { return Blob.fromPb(pb); } }; static final Function TO_PB_FUNCTION = new Function() { - @Override public StorageObject apply(Blob blob) { - return blob.toPb(); - } - }; + @Override + public StorageObject apply(Blob blob) { + return blob.toPb(); + } + }; private final String bucket; private final String id; @@ -98,8 +100,7 @@ public static final class Builder { private Long deleteTime; private Long updateTime; - private Builder() { - } + private Builder() {} public Builder bucket(String bucket) { this.bucket = checkNotNull(bucket); @@ -366,7 +367,7 @@ public static Blob of(String bucket, String name) { return builder(bucket, name).build(); } - public static Builder builder(Bucket bucket, String name) { + public static Builder builder(Bucket bucket, String name) { return builder(bucket.name(), name); } @@ -378,7 +379,8 @@ StorageObject toPb() { StorageObject storageObject = new StorageObject(); if (acl != null) { storageObject.setAcl(Lists.transform(acl, new Function() { - @Override public ObjectAccessControl apply(Acl acl) { + @Override + public ObjectAccessControl apply(Acl acl) { return acl.toObjectPb(); } })); @@ -425,7 +427,6 @@ static Blob fromPb(StorageObject storageObject) { .generation(storageObject.getGeneration()) .md5(storageObject.getMd5Hash()) .mediaLink(storageObject.getMediaLink()) - .metadata(storageObject.getMetadata()) .metageneration(storageObject.getMetageneration()) .name(storageObject.getName()) .contentDisposition(storageObject.getContentDisposition()) @@ -434,6 +435,9 @@ static Blob fromPb(StorageObject storageObject) { .etag(storageObject.getEtag()) .id(storageObject.getId()) .selfLink(storageObject.getSelfLink()); + if (storageObject.getMetadata() != null) { + builder.metadata(storageObject.getMetadata()); + } if (storageObject.getTimeDeleted() != null) { builder.deleteTime(storageObject.getTimeDeleted().getValue()); } diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java index 3757dec04e0b..32b68b3e97f6 100644 --- a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java +++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java @@ -23,9 +23,9 @@ /** * A channel for reading data from a Google Cloud Storage object. * - * Implementations of this class may buffer data internally to reduce remote calls. + * Implementations of this class may buffer data internally to reduce remote calls. * - * This class is @{link Serializable}, which allows incremental reads. + * This class is @{link Serializable}, which allows incremental reads. */ public interface BlobReadChannel extends ReadableByteChannel, Serializable, Closeable { diff --git a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java index a83ed661baca..77ce84a8ea7a 100644 --- a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java +++ b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java @@ -23,9 +23,9 @@ /** * A channel for writing data to a Google Cloud Storage object. * - * Implementations of this class may further buffer data internally to reduce remote calls. - * Written data will only be visible after calling {@link #close()}. - * This class is serializable, to allow incremental writes. + * Implementations of this class may further buffer data internally to reduce remote calls. Written + * data will only be visible after calling {@link #close()}. This class is serializable, to allow + * incremental writes. */ public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable { diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index 1c3077e41bb6..16937fda2b4b 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -60,14 +60,16 @@ public final class Bucket implements Serializable { static final Function FROM_PB_FUNCTION = new Function() { - @Override public Bucket apply(com.google.api.services.storage.model.Bucket pb) { + @Override + public Bucket apply(com.google.api.services.storage.model.Bucket pb) { return Bucket.fromPb(pb); } }; static final Function TO_PB_FUNCTION = new Function() { - @Override public com.google.api.services.storage.model.Bucket apply(Bucket bucket) { + @Override + public com.google.api.services.storage.model.Bucket apply(Bucket bucket) { return bucket.toPb(); } }; @@ -226,8 +228,7 @@ public static final class StorageClass implements Serializable { private final String value; public enum Option { - DURABLE_REDUCED_AVAILABILITY, - STANDARD; + DURABLE_REDUCED_AVAILABILITY, STANDARD; private final StorageClass storageClass; @@ -345,8 +346,7 @@ public final static class Builder { private ImmutableList acl; private ImmutableList defaultAcl; - private Builder() { - } + private Builder() {} public Builder name(String name) { this.name = checkNotNull(name); @@ -573,14 +573,16 @@ com.google.api.services.storage.model.Bucket toPb() { } if (acl != null) { bucketPb.setAcl(transform(acl, new Function() { - @Override public BucketAccessControl apply(Acl acl) { + @Override + public BucketAccessControl apply(Acl acl) { return acl.toBucketPb(); } })); } if (defaultAcl != null) { bucketPb.setDefaultObjectAcl(transform(defaultAcl, new Function() { - @Override public ObjectAccessControl apply(Acl acl) { + @Override + public ObjectAccessControl apply(Acl acl) { return acl.toObjectPb(); } })); @@ -601,7 +603,8 @@ com.google.api.services.storage.model.Bucket toPb() { if (deleteRules != null) { Lifecycle lifecycle = new Lifecycle(); lifecycle.setRule(transform(deleteRules, new Function() { - @Override public Rule apply(DeleteRule deleteRule) { + @Override + public Rule apply(DeleteRule deleteRule) { return deleteRule.toPb(); } })); @@ -626,7 +629,8 @@ static Bucket fromPb(com.google.api.services.storage.model.Bucket bucketPb) { } if (bucketPb.getAcl() != null) { builder.acl(transform(bucketPb.getAcl(), new Function() { - @Override public Acl apply(BucketAccessControl bucketAccessControl) { + @Override + public Acl apply(BucketAccessControl bucketAccessControl) { return Acl.fromPb(bucketAccessControl); } })); @@ -654,7 +658,8 @@ public Acl apply(ObjectAccessControl objectAccessControl) { if (bucketPb.getLifecycle() != null && bucketPb.getLifecycle().getRule() != null) { builder.deleteRules(transform(bucketPb.getLifecycle().getRule(), new Function() { - @Override public DeleteRule apply(Rule rule) { + @Override + public DeleteRule apply(Rule rule) { return DeleteRule.fromPb(rule); } })); diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java index 87d33348983d..ccbe2413042e 100644 --- a/src/main/java/com/google/gcloud/storage/Cors.java +++ b/src/main/java/com/google/gcloud/storage/Cors.java @@ -35,13 +35,15 @@ public final class Cors implements Serializable { private static final long serialVersionUID = -8637770919343335655L; static final Function FROM_PB_FUNCTION = new Function() { - @Override public Cors apply(Bucket.Cors pb) { + @Override + public Cors apply(Bucket.Cors pb) { return Cors.fromPb(pb); } }; static final Function TO_PB_FUNCTION = new Function() { - @Override public Bucket.Cors apply(Cors cors) { + @Override + public Bucket.Cors apply(Cors cors) { return cors.toPb(); } }; @@ -103,8 +105,7 @@ public static final class Builder { private ImmutableList origins; private ImmutableList responseHeaders; - private Builder() { - } + private Builder() {} public Builder maxAgeSeconds(Integer maxAgeSeconds) { this.maxAgeSeconds = maxAgeSeconds; @@ -178,12 +179,14 @@ Bucket.Cors toPb() { static Cors fromPb(Bucket.Cors cors) { Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds()); builder.methods(transform(cors.getMethod(), new Function() { - @Override public Method apply(String name) { + @Override + public Method apply(String name) { return Method.valueOf(name.toUpperCase()); } })); builder.origins(transform(cors.getOrigin(), new Function() { - @Override public Origin apply(String value) { + @Override + public Origin apply(String value) { return Origin.of(value); } })); diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java index 05257babca00..d571b176d9ab 100644 --- a/src/main/java/com/google/gcloud/storage/ListOptions.java +++ b/src/main/java/com/google/gcloud/storage/ListOptions.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; private final boolean includeOlderVersions; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; private boolean includeOlderVersions; private Builder() { } public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } public Builder includeOlderVersions(boolean include) { this.includeOlderVersions = include; return this; } public ListOptions build() { return new ListOptions(this); } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; includeOlderVersions = builder.includeOlderVersions; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } public boolean includeOlderVersions() { return includeOlderVersions; } public static ListOptions of() { return builder().build(); } public static Builder builder() { return new Builder(); } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; private final boolean includeOlderVersions; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; private boolean includeOlderVersions; private Builder() {} public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } public Builder includeOlderVersions(boolean include) { this.includeOlderVersions = include; return this; } public ListOptions build() { return new ListOptions(this); } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; includeOlderVersions = builder.includeOlderVersions; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } public boolean includeOlderVersions() { return includeOlderVersions; } public static ListOptions of() { return builder().build(); } public static Builder builder() { return new Builder(); } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/Option.java b/src/main/java/com/google/gcloud/storage/Option.java index cdbf00015b72..7e130603b79c 100644 --- a/src/main/java/com/google/gcloud/storage/Option.java +++ b/src/main/java/com/google/gcloud/storage/Option.java @@ -18,29 +18,32 @@ import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.base.MoreObjects; +import com.google.gcloud.spi.StorageRpc; + import java.io.Serializable; import java.util.Objects; /** * Base class for Storage operation option */ -public class Option implements Serializable { +class Option implements Serializable { private static final long serialVersionUID = -73199088766477208L; - private final String name; + private final StorageRpc.Option rpcOption; private final Object value; - Option(String name, Object value) { - this.name = checkNotNull(name); - this.value = value; + Option(StorageRpc.Option rpcOption, Object value) { + this.rpcOption = checkNotNull(rpcOption); + this.value = checkNotNull(value); } - public String name() { - return name; + StorageRpc.Option rpcOption() { + return rpcOption; } - public Object value() { + Object value() { return value; } @@ -50,12 +53,20 @@ public boolean equals(Object obj) { return false; } Option other = (Option) obj; - return Objects.equals(name, other.name) + return Objects.equals(rpcOption, other.rpcOption) && Objects.equals(value, other.value); } @Override public int hashCode() { - return Objects.hash(name, value); + return Objects.hash(rpcOption, value); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("name", rpcOption.value()) + .add("value", value) + .toString(); } } diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index ca3752e08b28..2fb92cd66743 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.gcloud.Service; +import com.google.gcloud.spi.StorageRpc; import java.io.Serializable; import java.util.Arrays; @@ -33,7 +34,8 @@ public interface StorageService extends Service { - // todo: provide way for construct signed URLs - https://cloud.google.com/storage/docs/access-control#Signed-URLs + // todo: provide way for construct signed URLs - + // https://cloud.google.com/storage/docs/access-control#Signed-URLs enum PredefinedAcl { AUTHENTICATED_READ("authenticatedRead"), @@ -60,20 +62,20 @@ class BucketTargetOption extends Option { private static final long serialVersionUID = -5880204616982900975L; - private BucketTargetOption(String name, Object value) { - super(name, value); + private BucketTargetOption(StorageRpc.Option rpcOption, Object value) { + super(rpcOption, value); } public static BucketTargetOption predefinedAcl(PredefinedAcl acl) { - return new BucketTargetOption("predefinedAcl", acl.entry()); + return new BucketTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry()); } public static BucketTargetOption predefinedDefaultObjectAcl(PredefinedAcl acl) { - return new BucketTargetOption("predefinedDefaultObjectAcl", acl.entry()); + return new BucketTargetOption(StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL, acl.entry()); } public static BucketTargetOption metagenerationMatch(boolean match) { - return new BucketTargetOption("ifMetagenerationMatch", match); + return new BucketTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match); } } @@ -81,12 +83,12 @@ class BucketSourceOption extends Option { private static final long serialVersionUID = 5185657617120212117L; - private BucketSourceOption(String name, Object value) { - super(name, value); + private BucketSourceOption(StorageRpc.Option rpcOption, Object value) { + super(rpcOption, value); } public static BucketSourceOption metagenerationMatch(boolean match) { - return new BucketSourceOption("ifMetagenerationMatch", match); + return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match); } } @@ -94,20 +96,20 @@ class BlobTargetOption extends Option { private static final long serialVersionUID = 214616862061934846L; - private BlobTargetOption(String name, Object value) { - super(name, value); + private BlobTargetOption(StorageRpc.Option rpcOption, Object value) { + super(rpcOption, value); } public static BlobTargetOption predefinedAcl(PredefinedAcl acl) { - return new BlobTargetOption("predefinedAcl", acl.entry()); + return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry()); } public static BlobTargetOption generationMath(boolean match) { - return new BlobTargetOption("ifGenerationMatch", match); + return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, match); } public static BlobTargetOption metagenerationMatch(boolean match) { - return new BlobTargetOption("ifMetagenerationMatch", match); + return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match); } } @@ -115,16 +117,16 @@ class BlobSourceOption extends Option { private static final long serialVersionUID = -3712768261070182991L; - private BlobSourceOption(String name, Object value) { - super(name, value); + private BlobSourceOption(StorageRpc.Option rpcOption, Object value) { + super(rpcOption, value); } public static BlobSourceOption generationMath(boolean match) { - return new BlobSourceOption("ifGenerationMatch", match); + return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH, match); } public static BlobSourceOption metagenerationMatch(boolean match) { - return new BlobSourceOption("ifMetagenerationMatch", match); + return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match); } } @@ -303,12 +305,12 @@ public static Builder builder() { /** * @throws StorageServiceException upon failure */ - Bucket create(Bucket bucket, BucketTargetOption... option); + Bucket create(Bucket bucket, BucketTargetOption... options); /** * @throws StorageServiceException upon failure */ - Blob create(Blob blob, byte[] content, BlobTargetOption... option); + Blob create(Blob blob, byte[] content, BlobTargetOption... options); /** * @throws StorageServiceException upon failure @@ -333,22 +335,22 @@ public static Builder builder() { /** * @throws StorageServiceException upon failure */ - Bucket update(Bucket bucket, BucketTargetOption... option); + Bucket update(Bucket bucket, BucketTargetOption... options); /** * @throws StorageServiceException upon failure */ - Blob update(Blob blob, BlobTargetOption... option); + Blob update(Blob blob, BlobTargetOption... options); /** * @throws StorageServiceException upon failure */ - void delete(Bucket bucket, BucketSourceOption... option); + void delete(Bucket bucket, BucketSourceOption... options); /** * @throws StorageServiceException upon failure */ - void delete(Blob blob, BlobSourceOption... option); + void delete(Blob blob, BlobSourceOption... options); /** * @throws StorageServiceException upon failure @@ -360,13 +362,19 @@ public static Builder builder() { */ Blob copy(CopyRequest copyRequest); + + /** + * @throws StorageServiceException upon failure + */ + byte[] load(Blob blob, BlobSourceOption... options); + /** * @throws StorageServiceException upon failure */ - BlobReadChannel readFrom(Blob blob, BlobSourceOption... option); + BlobReadChannel reader(Blob blob, BlobSourceOption... options); /** * @throws StorageServiceException upon failure */ - BlobWriteChannel writeTo(Blob blob, BlobTargetOption... option); + BlobWriteChannel writer(Blob blob, BlobTargetOption... options); } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceException.java b/src/main/java/com/google/gcloud/storage/StorageServiceException.java index 4a3438819250..900a762d46d2 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceException.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceException.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; /** * Storage service exception. * * @see Google Cloud Storage error codes */ public class StorageServiceException extends RuntimeException { private static final long serialVersionUID = -3748432005065428084L; private final int code; private final boolean retryable; public StorageServiceException(int code, String message, boolean retryable) { super(message); this.code = code; this.retryable = retryable; } /** * Returns the code associated with this exception. */ public int code() { return code; } public boolean retryable() { return retryable; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; /** * Storage service exception. * * @see Google Cloud * Storage error codes */ public class StorageServiceException extends RuntimeException { private static final long serialVersionUID = -3748432005065428084L; private final int code; private final boolean retryable; public StorageServiceException(int code, String message, boolean retryable) { super(message); this.code = code; this.retryable = retryable; } /** * Returns the code associated with this exception. */ public int code() { return code; } public boolean retryable() { return retryable; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java index 2c2053b72d6b..a801622082ef 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java @@ -21,10 +21,11 @@ public abstract class StorageServiceFactory { private static final StorageServiceFactory INSTANCE = new StorageServiceFactory() { - @Override public StorageService get(StorageServiceOptions options) { - return new StorageServiceImpl(options); - } - }; + @Override + public StorageService get(StorageServiceOptions options) { + return new StorageServiceImpl(options); + } + }; public static StorageServiceFactory instance() { return INSTANCE; diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 7d653a7fb25a..afaca940c9f0 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -20,6 +20,7 @@ import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.gcloud.BaseService; @@ -28,34 +29,34 @@ import com.google.gcloud.RetryParams; import com.google.gcloud.spi.StorageRpc; +import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.concurrent.Callable; final class StorageServiceImpl extends BaseService implements StorageService { - private static final Interceptor EXCEPTION_HANDLER_INTERCEPTOR = - new Interceptor() { + private static final Interceptor EXCEPTION_HANDLER_INTERCEPTOR = new Interceptor() { - private static final long serialVersionUID = -7758580330857881124L; + private static final long serialVersionUID = -7758580330857881124L; - @Override - public RetryResult afterEval(Exception exception, RetryResult retryResult) { - return null; - } + @Override + public RetryResult afterEval(Exception exception, RetryResult retryResult) { + return null; + } - @Override - public RetryResult beforeEval(Exception exception) { - if (exception instanceof StorageServiceException) { - boolean retriable = ((StorageServiceException) exception).retryable(); - return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT; - } - return null; - } - }; + @Override + public RetryResult beforeEval(Exception exception) { + if (exception instanceof StorageServiceException) { + boolean retriable = ((StorageServiceException) exception).retryable(); + return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT; + } + return null; + } + }; private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder() - .abortOn(RuntimeException.class) - .interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build(); + .abortOn(RuntimeException.class).interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build(); private final StorageRpc storageRpc; private final RetryParams retryParams; @@ -68,56 +69,68 @@ public RetryResult beforeEval(Exception exception) { } @Override - public Bucket create(Bucket bucket, final BucketTargetOption... options) { + public Bucket create(Bucket bucket, BucketTargetOption... options) { + Validator.checkBucketOptions(bucket, options); final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); + final Map optionsMap = optionMap(options); return Bucket.fromPb(runWithRetries( new Callable() { - @Override public com.google.api.services.storage.model.Bucket call() { - return storageRpc.create(bucketPb, options); + @Override + public com.google.api.services.storage.model.Bucket call() { + return storageRpc.create(bucketPb, optionsMap); } }, retryParams, EXCEPTION_HANDLER)); } @Override - public Blob create(Blob blob, final byte[] content, final BlobTargetOption... options) { + public Blob create(Blob blob, final byte[] content, BlobTargetOption... options) { + Validator.checkBlobOptions(blob, options); final StorageObject blobPb = blob.toPb(); + final Map optionsMap = optionMap(options); return Blob.fromPb(runWithRetries(new Callable() { - @Override public StorageObject call() { - return storageRpc.create(blobPb, content, options); - } - }, retryParams, EXCEPTION_HANDLER)); + @Override + public StorageObject call() { + return storageRpc.create(blobPb, content, optionsMap); + } + }, retryParams, EXCEPTION_HANDLER)); } @Override - public Bucket get(Bucket bucket, final BucketSourceOption... options) { + public Bucket get(Bucket bucket, BucketSourceOption... options) { + Validator.checkBucketOptions(bucket, options); final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); + final Map optionsMap = optionMap(options); return Bucket.fromPb(runWithRetries( new Callable() { - @Override public com.google.api.services.storage.model.Bucket call() { - return storageRpc.get(bucketPb, options); + @Override + public com.google.api.services.storage.model.Bucket call() { + return storageRpc.get(bucketPb, optionsMap); } }, retryParams, EXCEPTION_HANDLER)); } @Override - public Blob get(Blob blob, final BlobSourceOption... options) { + public Blob get(Blob blob, BlobSourceOption... options) { + Validator.checkBlobOptions(blob, options); final StorageObject storedObject = blob.toPb(); + final Map optionsMap = optionMap(options); return Blob.fromPb(runWithRetries(new Callable() { - @Override public StorageObject call() { - return storageRpc.get(storedObject, options); + @Override + public StorageObject call() { + return storageRpc.get(storedObject, optionsMap); } }, retryParams, EXCEPTION_HANDLER)); } @Override public Iterator list() { - return Iterators.transform(runWithRetries( - new Callable>() { - @Override public Iterator call() { + return Iterators.transform( + runWithRetries(new Callable>() { + @Override + public Iterator call() { return storageRpc.list(); } - }, retryParams, EXCEPTION_HANDLER), - Bucket.FROM_PB_FUNCTION); + }, retryParams, EXCEPTION_HANDLER), Bucket.FROM_PB_FUNCTION); } @Override @@ -126,48 +139,59 @@ public Iterator list(String bucket, ListOptions settings) { String delimiter = settings.recursive() ? options().pathDelimiter() : null; return Iterators.transform( storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(), - settings.includeOlderVersions(), settings.maxResults()), - Blob.FROM_PB_FUNCTION); + settings.includeOlderVersions(), settings.maxResults()), Blob.FROM_PB_FUNCTION); } @Override - public Bucket update(Bucket bucket, final BucketTargetOption... options) { + public Bucket update(Bucket bucket, BucketTargetOption... options) { + Validator.checkBucketOptions(bucket, options); final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); + final Map optionsMap = optionMap(options); return Bucket.fromPb(runWithRetries( new Callable() { - @Override public com.google.api.services.storage.model.Bucket call() { - return storageRpc.patch(bucketPb, options); + @Override + public com.google.api.services.storage.model.Bucket call() { + return storageRpc.patch(bucketPb, optionsMap); } }, retryParams, EXCEPTION_HANDLER)); } @Override - public Blob update(Blob blob, final BlobTargetOption... options) { + public Blob update(Blob blob, BlobTargetOption... options) { + Validator.checkBlobOptions(blob, options); final StorageObject storageObject = blob.toPb(); + final Map optionsMap = optionMap(options); return Blob.fromPb(runWithRetries(new Callable() { - @Override public StorageObject call() { - return storageRpc.patch(storageObject, options); + @Override + public StorageObject call() { + return storageRpc.patch(storageObject, optionsMap); } }, retryParams, EXCEPTION_HANDLER)); } @Override - public void delete(Bucket bucket, final BucketSourceOption... options) { + public void delete(Bucket bucket, BucketSourceOption... options) { + Validator.checkBucketOptions(bucket, options); final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); + final Map optionsMap = optionMap(options); runWithRetries(new Callable() { - @Override public Void call() { - storageRpc.delete(bucketPb, options); + @Override + public Void call() { + storageRpc.delete(bucketPb, optionsMap); return null; } }, retryParams, EXCEPTION_HANDLER); } @Override - public void delete(Blob blob, final BlobSourceOption... options) { + public void delete(Blob blob, BlobSourceOption... options) { + Validator.checkBlobOptions(blob, options); final StorageObject storageObject = blob.toPb(); + final Map optionsMap = optionMap(options); runWithRetries(new Callable() { - @Override public Void call() { - storageRpc.delete(storageObject, options); + @Override + public Void call() { + storageRpc.delete(storageObject, optionsMap); return null; } }, retryParams, EXCEPTION_HANDLER); @@ -179,39 +203,70 @@ public Blob compose(final ComposeRequest composeRequest) { Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size()); for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) { sources.add(Blob.builder(composeRequest.sourceBucket(), sourceBlob.blob) - .generation(sourceBlob.generation) - .build() - .toPb()); + .generation(sourceBlob.generation).build().toPb()); } final StorageObject target = composeRequest.target().toPb(); + final Map targetOptions = optionMap(composeRequest.targetOptions()); return Blob.fromPb(runWithRetries(new Callable() { - @Override public StorageObject call() { - return storageRpc.compose(sources, target, composeRequest.targetOptions()); + @Override + public StorageObject call() { + return storageRpc.compose(sources, target, targetOptions); } }, retryParams, EXCEPTION_HANDLER)); } @Override - public Blob copy(final CopyRequest copyRequest) { + public Blob copy(CopyRequest copyRequest) { final StorageObject source = copyRequest.source().toPb(); + final Map sourceOptions = optionMap(copyRequest.sourceOptions()); final StorageObject target = copyRequest.target().toPb(); + final Map targetOptions = optionMap(copyRequest.targetOptions()); return Blob.fromPb(runWithRetries(new Callable() { - @Override public StorageObject call() { - return storageRpc.copy(source, copyRequest.sourceOptions(), target, - copyRequest.targetOptions()); + @Override + public StorageObject call() { + return storageRpc.copy(source, sourceOptions, target, targetOptions); } }, retryParams, EXCEPTION_HANDLER)); } @Override - public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) { + public byte[] load(Blob blob, BlobSourceOption... options) { + Validator.checkBlobOptions(blob, options); + final StorageObject storageObject = blob.toPb(); + final Map optionsMap = optionMap(options); + return runWithRetries(new Callable() { + @Override + public byte[] call() { + return storageRpc.load(storageObject, optionsMap); + } + }, retryParams, EXCEPTION_HANDLER); + } + + @Override + public BlobReadChannel reader(Blob blob, BlobSourceOption... options) { + Validator.checkBlobOptions(blob, options); // todo: Use retry helper on retriable failures - return storageRpc.reader(blob.toPb(), options); + final Map optionsMap = optionMap(options); + return storageRpc.reader(blob.toPb(), optionsMap); } @Override - public BlobWriteChannel writeTo(Blob blob, BlobTargetOption... options) { + public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) { + Validator.checkBlobOptions(blob, options); // todo: Use retry helper on retriable failures - return storageRpc.writer(blob.toPb(), options); + final Map optionsMap = optionMap(options); + return storageRpc.writer(blob.toPb(), optionsMap); + } + + private static Map optionMap(Iterable options) { + ImmutableMap.Builder mapBuider = ImmutableMap.builder(); + for (Option option : options) { + mapBuider.put(option.rpcOption(), option.value()); + } + return mapBuider.build(); + } + + private static Map optionMap(Option... options) { + return optionMap(Arrays.asList(options)); } } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java index 10b54d79e413..c36d28c1293f 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java @@ -42,8 +42,7 @@ public static class Builder extends ServiceOptions.Builder { private String pathDelimiter; private StorageRpc storageRpc; - private Builder() { - } + private Builder() {} private Builder(StorageServiceOptions options) { super(options); diff --git a/src/main/java/com/google/gcloud/storage/Validator.java b/src/main/java/com/google/gcloud/storage/Validator.java index 0db8fd3be16e..5590d67528cc 100644 --- a/src/main/java/com/google/gcloud/storage/Validator.java +++ b/src/main/java/com/google/gcloud/storage/Validator.java @@ -18,21 +18,37 @@ import static com.google.common.base.Preconditions.checkArgument; +import java.util.Arrays; + /** * Utility to validate Storage type/values. */ public class Validator { + static void checkBlobOptions(Blob blob, Option... options) { + checkBlobOptions("requested", blob, Arrays.asList(options)); + } + static void checkBlobOptions(String name, Blob blob, Iterable options) { for (Option option : options) { - switch (option.name()) { - case "ifGenerationMatch": + switch (option.rpcOption()) { + case IF_GENERATION_MATCH: checkArgument(blob.generation() != 0, "%s blob is missing generation", name); break; - case "ifMetagenerationMatch": + case IF_METAGENERATION_MATCH: checkArgument(blob.metageneration() != 0, "%s blob is missing metageneration", name); break; } } } + + static void checkBucketOptions(Bucket bucket, Option... options) { + for (Option option : options) { + switch (option.rpcOption()) { + case IF_METAGENERATION_MATCH: + checkArgument(bucket.metageneration() != 0, "bucket is missing metageneration"); + break; + } + } + } } diff --git a/src/main/java/com/google/gcloud/storage/package-info.java b/src/main/java/com/google/gcloud/storage/package-info.java index 9aabec93c93f..0aa916666e8c 100644 --- a/src/main/java/com/google/gcloud/storage/package-info.java +++ b/src/main/java/com/google/gcloud/storage/package-info.java @@ -20,3 +20,4 @@ * @see Google Cloud Storageg */ package com.google.gcloud.storage; + diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java index 1d96e86d0e46..3844f9de36d7 100644 --- a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java +++ b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java @@ -129,7 +129,8 @@ public void testShouldTry() { assertTrue(handler.shouldRetry(new NullPointerException())); final AtomicReference before = new AtomicReference<>(RetryResult.ABORT); - @SuppressWarnings("serial") Interceptor interceptor = new Interceptor() { + @SuppressWarnings("serial") + Interceptor interceptor = new Interceptor() { @Override public RetryResult afterEval(Exception exception, RetryResult retryResult) { From c92f2ea8075db60d3ca7ab3041605b7ac7d0ecd1 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 23 Apr 2015 17:09:36 -0700 Subject: [PATCH 26/48] make AuthCredentials serializable --- .../com/google/gcloud/AuthCredentials.java | 79 ++++++++++++++----- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java index 3e6c3651c576..ff5d20add781 100644 --- a/src/main/java/com/google/gcloud/AuthCredentials.java +++ b/src/main/java/com/google/gcloud/AuthCredentials.java @@ -30,6 +30,8 @@ import com.google.auth.oauth2.GoogleCredentials; import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; import java.security.GeneralSecurityException; import java.security.PrivateKey; import java.util.Set; @@ -37,10 +39,14 @@ /** * Credentials for accessing Google Cloud services. */ -public abstract class AuthCredentials { +public abstract class AuthCredentials implements Serializable { + + private static final long serialVersionUID = 236297804453464604L; private static class AppEngineAuthCredentials extends AuthCredentials { + private static final long serialVersionUID = 7931300552744202954L; + @Override protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, Set scopes) { @@ -50,6 +56,7 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, private static class ServiceAccountAuthCredentials extends AuthCredentials { + private static final long serialVersionUID = 8007708734318445901L; private final String account; private final PrivateKey privateKey; @@ -78,6 +85,54 @@ protected HttpRequestInitializer httpRequestInitializer( } } + private static class ComputeEngineAuthCredentials extends AuthCredentials { + + private static final long serialVersionUID = -5217355402127260144L; + + private transient ComputeCredential computeCredential; + + ComputeEngineAuthCredentials() throws IOException, GeneralSecurityException { + computeCredential = getComputeCredential(); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + try { + computeCredential = getComputeCredential(); + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + } + + @Override + protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, + Set scopes) { + return computeCredential; + } + } + + private static class ApplicationDefaultAuthCredentials extends AuthCredentials { + + private static final long serialVersionUID = -8306873864136099893L; + + private transient GoogleCredentials googleCredentials; + + ApplicationDefaultAuthCredentials() throws IOException { + googleCredentials = GoogleCredentials.getApplicationDefault(); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + googleCredentials = GoogleCredentials.getApplicationDefault(); + } + + @Override + protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, + Set scopes) { + return new HttpCredentialsAdapter(googleCredentials); + } + } + protected abstract HttpRequestInitializer httpRequestInitializer(HttpTransport transport, Set scopes); @@ -85,16 +140,9 @@ public static AuthCredentials createForAppEngine() { return new AppEngineAuthCredentials(); } - public static AuthCredentials createForComputeEngine() throws IOException, - GeneralSecurityException { - final ComputeCredential cred = getComputeCredential(); - return new AuthCredentials() { - @Override - protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, - Set scopes) { - return cred; - } - }; + public static AuthCredentials createForComputeEngine() + throws IOException, GeneralSecurityException { + return new ComputeEngineAuthCredentials(); } /** @@ -111,14 +159,7 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, * @throws IOException if the credentials cannot be created in the current environment. */ public static AuthCredentials createApplicationDefaults() throws IOException { - final GoogleCredentials credentials = GoogleCredentials.getApplicationDefault(); - return new AuthCredentials() { - @Override - protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, - Set scopes) { - return new HttpCredentialsAdapter(credentials); - } - }; + return new ApplicationDefaultAuthCredentials(); } public static AuthCredentials createFor(String account, PrivateKey privateKey) { diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 41d4359c73d4..867913c2f286 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.Map; public class DefaultStorageRpc implements StorageRpc { private final StorageServiceOptions options; private final Storage storage; public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: set projection to full // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); // todo: based on code consider if error is retryable translated = new StorageServiceException(details.getCode(), details.getMessage(), false); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets().insert(this.options.project(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects().insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets().list(options.project()).execute().getItems().iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets().get(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects().get(object.getBucket(), object.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets().patch(bucket.getName(), bucket).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest try { return storage.objects().compose(target.getBucket(), target.getName(), null) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName()); if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) { getRequest.setIfGenerationMatch(from.getGeneration()); } else { getRequest.setIfGenerationNotMatch(from.getGeneration()); } } if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) { getRequest.setIfMetagenerationMatch(from.getMetageneration()); } else { getRequest.setIfMetagenerationNotMatch(from.getMetageneration()); } } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets() .list(options.project()) .setProjection(DEFAULT_PROJECTION) .execute() .getItems() .iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest // todo: missing setProjection try { return storage.objects() .compose(target.getBucket(), target.getName(), null) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName()); if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) { getRequest.setIfGenerationMatch(from.getGeneration()); } else { getRequest.setIfGenerationNotMatch(from.getGeneration()); } } if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) { getRequest.setIfMetagenerationMatch(from.getMetageneration()); } else { getRequest.setIfMetagenerationNotMatch(from.getMetageneration()); } } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file From d504256207b9248462c425612a8baef2b496ca3b Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 23 Apr 2015 17:27:16 -0700 Subject: [PATCH 27/48] work in progress --- .../com/google/gcloud/AuthCredentials.java | 2 +- .../com/google/gcloud/ServiceOptions.java | 68 +++++++++++-------- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java index ff5d20add781..e6c783565d8f 100644 --- a/src/main/java/com/google/gcloud/AuthCredentials.java +++ b/src/main/java/com/google/gcloud/AuthCredentials.java @@ -114,7 +114,7 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, private static class ApplicationDefaultAuthCredentials extends AuthCredentials { private static final long serialVersionUID = -8306873864136099893L; - + private transient GoogleCredentials googleCredentials; ApplicationDefaultAuthCredentials() throws IOException { diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 36cdf64aa7f8..896d5c1572f5 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -28,24 +28,54 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.io.Serializable; import java.lang.reflect.Method; import java.net.URL; import java.net.URLConnection; import java.util.Set; -public abstract class ServiceOptions { +public abstract class ServiceOptions implements Serializable { private static final String DEFAULT_HOST = "https://www.googleapis.com"; + private static final long serialVersionUID = 1203687993961393350L; private final String host; - private final HttpTransport httpTransport; + private final HttpTransportFactory httpTransportFactory; private final AuthCredentials authCredentials; private final RetryParams retryParams; + public interface HttpTransportFactory extends Serializable { + HttpTransport create(); + } + + private static class DefaultHttpTransportFactory implements HttpTransportFactory { + + private static final long serialVersionUID = 2684907202489506911L; + + @Override + public HttpTransport create() { + // Consider App Engine + if (appEngineAppId() != null) { + try { + return new UrlFetchTransport(); + } catch (Exception ignore) { + // Maybe not on App Engine + } + } + // Consider Compute + try { + return AuthCredentials.getComputeCredential().getTransport(); + } catch (Exception e) { + // Maybe not on GCE + } + return new NetHttpTransport(); + } + } + protected abstract static class Builder> { private String host; - private HttpTransport httpTransport; + private HttpTransportFactory httpTransportFactory; private AuthCredentials authCredentials; private RetryParams retryParams; @@ -53,7 +83,7 @@ protected Builder() {} protected Builder(ServiceOptions options) { host = options.host; - httpTransport = options.httpTransport; + httpTransportFactory = options.httpTransportFactory; authCredentials = options.authCredentials; retryParams = options.retryParams; } @@ -70,8 +100,8 @@ public B host(String host) { return self(); } - public B httpTransport(HttpTransport httpTransport) { - this.httpTransport = httpTransport; + public B httpTransportFactory(HttpTransportFactory httpTransportFactory) { + this.httpTransportFactory = httpTransportFactory; return self(); } @@ -88,29 +118,12 @@ public B retryParams(RetryParams retryParams) { protected ServiceOptions(Builder builder) { host = firstNonNull(builder.host, DEFAULT_HOST); - httpTransport = firstNonNull(builder.httpTransport, defaultHttpTransport()); + httpTransportFactory = + firstNonNull(builder.httpTransportFactory, new DefaultHttpTransportFactory()); authCredentials = firstNonNull(builder.authCredentials, defaultAuthCredentials()); retryParams = builder.retryParams; } - private static HttpTransport defaultHttpTransport() { - // Consider App Engine - if (appEngineAppId() != null) { - try { - return new UrlFetchTransport(); - } catch (Exception ignore) { - // Maybe not on App Engine - } - } - // Consider Compute - try { - return AuthCredentials.getComputeCredential().getTransport(); - } catch (Exception e) { - // Maybe not on GCE - } - return new NetHttpTransport(); - } - private static AuthCredentials defaultAuthCredentials() { // Consider App Engine. This will not be needed once issue #21 is fixed. if (appEngineAppId() != null) { @@ -179,8 +192,8 @@ public String host() { return host; } - public HttpTransport httpTransport() { - return httpTransport; + public HttpTransportFactory httpTransportFactory() { + return httpTransportFactory; } public AuthCredentials authCredentials() { @@ -192,6 +205,7 @@ public RetryParams retryParams() { } public HttpRequestInitializer httpRequestInitializer() { + HttpTransport httpTransport = httpTransportFactory.create(); return authCredentials().httpRequestInitializer(httpTransport, scopes()); } diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 867913c2f286..5451375e64f8 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransport(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets() .list(options.project()) .setProjection(DEFAULT_PROJECTION) .execute() .getItems() .iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest // todo: missing setProjection try { return storage.objects() .compose(target.getBucket(), target.getName(), null) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName()); if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) { getRequest.setIfGenerationMatch(from.getGeneration()); } else { getRequest.setIfGenerationNotMatch(from.getGeneration()); } } if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) { getRequest.setIfMetagenerationMatch(from.getMetageneration()); } else { getRequest.setIfMetagenerationNotMatch(from.getMetageneration()); } } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets() .list(options.project()) .setProjection(DEFAULT_PROJECTION) .execute() .getItems() .iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest // todo: missing setProjection try { return storage.objects() .compose(target.getBucket(), target.getName(), null) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName()); if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) { getRequest.setIfGenerationMatch(from.getGeneration()); } else { getRequest.setIfGenerationNotMatch(from.getGeneration()); } } if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) { getRequest.setIfMetagenerationMatch(from.getMetageneration()); } else { getRequest.setIfMetagenerationNotMatch(from.getMetageneration()); } } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file From c1488178dccfcef9749f9d9ae95eed7109e39f40 Mon Sep 17 00:00:00 2001 From: ozarov Date: Thu, 23 Apr 2015 22:27:12 -0700 Subject: [PATCH 28/48] work in progress --- .../com/google/gcloud/ServiceOptions.java | 23 +++++++++++--- .../datastore/DatastoreServiceOptions.java | 30 +++++++------------ .../google/gcloud/spi/ServiceRpcFactory.java | 2 +- .../gcloud/storage/StorageServiceOptions.java | 21 ++++++------- .../DatastoreServiceOptionsTest.java | 10 ++++++- 5 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 896d5c1572f5..76419d596958 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -24,6 +24,7 @@ import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.gcloud.spi.ServiceRpcFactory; import java.io.BufferedReader; import java.io.IOException; @@ -34,7 +35,7 @@ import java.net.URLConnection; import java.util.Set; -public abstract class ServiceOptions implements Serializable { +public abstract class ServiceOptions> implements Serializable { private static final String DEFAULT_HOST = "https://www.googleapis.com"; private static final long serialVersionUID = 1203687993961393350L; @@ -43,6 +44,7 @@ public abstract class ServiceOptions implements Serializable { private final HttpTransportFactory httpTransportFactory; private final AuthCredentials authCredentials; private final RetryParams retryParams; + private final ServiceRpcFactory serviceRpcFactory; public interface HttpTransportFactory extends Serializable { HttpTransport create(); @@ -72,12 +74,14 @@ public HttpTransport create() { } } - protected abstract static class Builder> { + protected abstract static class Builder, + B extends Builder> { private String host; private HttpTransportFactory httpTransportFactory; private AuthCredentials authCredentials; private RetryParams retryParams; + private ServiceRpcFactory serviceRpcFactory; protected Builder() {} @@ -86,6 +90,7 @@ protected Builder(ServiceOptions options) { httpTransportFactory = options.httpTransportFactory; authCredentials = options.authCredentials; retryParams = options.retryParams; + serviceRpcFactory = options.serviceRpcFactory; } protected abstract ServiceOptions build(); @@ -114,14 +119,20 @@ public B retryParams(RetryParams retryParams) { this.retryParams = retryParams; return self(); } + + public B serviceRpcFactory(ServiceRpcFactory serviceRpcFactory) { + this.serviceRpcFactory = serviceRpcFactory; + return self(); + } } - protected ServiceOptions(Builder builder) { + protected ServiceOptions(Builder builder) { host = firstNonNull(builder.host, DEFAULT_HOST); httpTransportFactory = firstNonNull(builder.httpTransportFactory, new DefaultHttpTransportFactory()); authCredentials = firstNonNull(builder.authCredentials, defaultAuthCredentials()); retryParams = builder.retryParams; + serviceRpcFactory = builder.serviceRpcFactory; } private static AuthCredentials defaultAuthCredentials() { @@ -204,10 +215,14 @@ public RetryParams retryParams() { return retryParams; } + public ServiceRpcFactory serviceRpcFactory() { + return serviceRpcFactory; + } + public HttpRequestInitializer httpRequestInitializer() { HttpTransport httpTransport = httpTransportFactory.create(); return authCredentials().httpRequestInitializer(httpTransport, scopes()); } - public abstract Builder toBuilder(); + public abstract Builder toBuilder(); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index bc7838df4a6b..de369d33f214 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -34,25 +34,25 @@ import java.util.Iterator; import java.util.Set; -public class DatastoreServiceOptions extends ServiceOptions { +public class DatastoreServiceOptions extends ServiceOptions { + private static final long serialVersionUID = -8636602944160689193L; private static final String DATASET_ENV_NAME = "DATASTORE_DATASET"; private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore"; private static final String USERINFO_SCOPE = "https://www.googleapis.com/auth/userinfo.email"; private static final Set SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE); + private final String dataset; private final String namespace; private final boolean force; - private final DatastoreRpc datastoreRpc; private final boolean normalizeDataset; - private final boolean defaultDatastoreRpc; - public static class Builder extends ServiceOptions.Builder { + public static class Builder extends + ServiceOptions.Builder { private String dataset; private String namespace; private boolean force; - private DatastoreRpc datastoreRpc; private boolean normalizeDataset = true; private Builder() { @@ -63,7 +63,6 @@ private Builder(DatastoreServiceOptions options) { dataset = options.dataset; force = options.force; namespace = options.namespace; - datastoreRpc = options.datastoreRpc; normalizeDataset = options.normalizeDataset; } @@ -73,11 +72,6 @@ public DatastoreServiceOptions build() { return normalizeDataset ? options.normalize() : options; } - public Builder datastoreRpc(DatastoreRpc datastoreRpc) { - this.datastoreRpc = datastoreRpc; - return this; - } - public Builder dataset(String dataset) { this.dataset = validateDataset(dataset); return this; @@ -105,8 +99,6 @@ private DatastoreServiceOptions(Builder builder) { namespace = builder.namespace != null ? builder.namespace : defaultNamespace(); force = builder.force; dataset = firstNonNull(builder.dataset, defaultDataset()); - datastoreRpc = firstNonNull(builder.datastoreRpc, ServiceRpcProvider.datastore(this)); - defaultDatastoreRpc = builder.datastoreRpc == null; } private DatastoreServiceOptions normalize() { @@ -123,7 +115,7 @@ private DatastoreServiceOptions normalize() { .build(); requestPb.addKey(key); try { - LookupResponse responsePb = datastoreRpc.lookup(requestPb.build()); + LookupResponse responsePb = datastoreRpc().lookup(requestPb.build()); if (responsePb.getDeferredCount() > 0) { key = responsePb.getDeferred(0); } else { @@ -132,9 +124,6 @@ private DatastoreServiceOptions normalize() { key = combinedIter.next().getEntity().getKey(); } builder.dataset(key.getPartitionId().getDatasetId()); - if (defaultDatastoreRpc && !dataset.equals(builder.dataset)) { - builder.datastoreRpc(ServiceRpcProvider.datastore(builder.build())); - } return new DatastoreServiceOptions(builder); } catch (DatastoreRpcException e) { throw DatastoreServiceException.translateAndThrow(e); @@ -185,8 +174,11 @@ public Builder toBuilder() { return new Builder(this); } - public DatastoreRpc datastoreRpc() { - return datastoreRpc; + DatastoreRpc datastoreRpc() { + if (serviceRpcFactory() != null) { + return serviceRpcFactory().create(this); + } + return ServiceRpcProvider.datastore(this); } public static Builder builder() { diff --git a/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java b/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java index 42c8da0f20f4..a16d11217bb1 100644 --- a/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java +++ b/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.gcloud.ServiceOptions; public interface ServiceRpcFactory { S create(O options); } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.gcloud.ServiceOptions; import java.io.Serializable; public interface ServiceRpcFactory extends Serializable { S create(O options); } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java index c36d28c1293f..8894cadad7d3 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java @@ -25,22 +25,22 @@ import java.util.Set; -public class StorageServiceOptions extends ServiceOptions { +public class StorageServiceOptions extends ServiceOptions { + private static final long serialVersionUID = -7804860602287801084L; private static final String GCS_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control"; private static final Set SCOPES = ImmutableSet.of(GCS_SCOPE); private static final String DEFAULT_PATH_DELIMITER = "/"; private static final String PROJECT_ENV_NAME = "default_project_id"; - private final StorageRpc storageRpc; private final String project; private final String pathDelimiter; - public static class Builder extends ServiceOptions.Builder { + public static class Builder extends + ServiceOptions.Builder { private String project; private String pathDelimiter; - private StorageRpc storageRpc; private Builder() {} @@ -58,11 +58,6 @@ public Builder pathDelimiter(String pathDelimiter) { return this; } - public Builder storageRpc(StorageRpc storageRpc) { - this.storageRpc = storageRpc; - return this; - } - @Override public StorageServiceOptions build() { return new StorageServiceOptions(this); @@ -74,7 +69,6 @@ private StorageServiceOptions(Builder builder) { pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER); project = builder.project != null ? builder.project : defaultProject(); Preconditions.checkArgument(project != null, "Missing required project id"); - storageRpc = MoreObjects.firstNonNull(builder.storageRpc, ServiceRpcProvider.storage(this)); } @Override @@ -82,8 +76,11 @@ protected Set scopes() { return SCOPES; } - public StorageRpc storageRpc() { - return storageRpc; + StorageRpc storageRpc() { + if (serviceRpcFactory() != null) { + return serviceRpcFactory().create(this); + } + return ServiceRpcProvider.storage(this); } public String project() { diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java index c847d3504580..7274f1ca9e91 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import com.google.gcloud.spi.DatastoreRpc; +import com.google.gcloud.spi.DatastoreRpcFactory; import org.easymock.EasyMock; import org.junit.Before; @@ -33,17 +34,23 @@ public class DatastoreServiceOptionsTest { private static final String DATASET = "dataset"; + private DatastoreRpcFactory datastoreRpcFactory; private DatastoreRpc datastoreRpc; private DatastoreServiceOptions.Builder options; @Before public void setUp() throws IOException, InterruptedException { + datastoreRpcFactory = EasyMock.createMock(DatastoreRpcFactory.class); datastoreRpc = EasyMock.createMock(DatastoreRpc.class); options = DatastoreServiceOptions.builder() .normalizeDataset(false) - .datastoreRpc(datastoreRpc) + .serviceRpcFactory(datastoreRpcFactory) .dataset(DATASET) .host("http://localhost:" + LocalGcdHelper.PORT); + EasyMock.expect(datastoreRpcFactory.create(EasyMock.anyObject(DatastoreServiceOptions.class))) + .andReturn(datastoreRpc) + .anyTimes(); + EasyMock.replay(datastoreRpcFactory, datastoreRpc); } @Test @@ -70,6 +77,7 @@ public void testForce() throws Exception { @Test public void testDatastore() throws Exception { + assertSame(datastoreRpcFactory, options.build().serviceRpcFactory()); assertSame(datastoreRpc, options.build().datastoreRpc()); } From 09a890f99c36819fcaeec3dd120ad25114e09931 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 24 Apr 2015 16:13:45 -0700 Subject: [PATCH 29/48] work in progress --- .../com/google/gcloud/AuthCredentials.java | 29 +++++++- .../com/google/gcloud/ServiceOptions.java | 23 ++++-- .../datastore/DatastoreServiceOptions.java | 18 +++++ .../com/google/gcloud/datastore/Value.java | 2 +- .../gcloud/storage/StorageServiceOptions.java | 16 +++++ .../gcloud/datastore/SerializationTest.java | 26 ++++++- .../gcloud/storage/SerializationTest.java | 71 +++++++++++++++++++ 7 files changed, 176 insertions(+), 9 deletions(-) create mode 100644 src/test/java/com/google/gcloud/storage/SerializationTest.java diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java index e6c783565d8f..839da54e62cf 100644 --- a/src/main/java/com/google/gcloud/AuthCredentials.java +++ b/src/main/java/com/google/gcloud/AuthCredentials.java @@ -31,9 +31,11 @@ import java.io.IOException; import java.io.ObjectInputStream; +import java.io.ObjectStreamException; import java.io.Serializable; import java.security.GeneralSecurityException; import java.security.PrivateKey; +import java.util.Objects; import java.util.Set; /** @@ -47,11 +49,17 @@ private static class AppEngineAuthCredentials extends AuthCredentials { private static final long serialVersionUID = 7931300552744202954L; + private static final AuthCredentials INSTANCE = new AppEngineAuthCredentials(); + @Override protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, Set scopes) { return new AppIdentityCredential(scopes); } + + private Object readResolve() throws ObjectStreamException { + return INSTANCE; + } } private static class ServiceAccountAuthCredentials extends AuthCredentials { @@ -60,6 +68,8 @@ private static class ServiceAccountAuthCredentials extends AuthCredentials { private final String account; private final PrivateKey privateKey; + private static final AuthCredentials NO_CREDENTIALS = new ServiceAccountAuthCredentials(); + ServiceAccountAuthCredentials(String account, PrivateKey privateKey) { this.account = checkNotNull(account); this.privateKey = checkNotNull(privateKey); @@ -83,6 +93,21 @@ protected HttpRequestInitializer httpRequestInitializer( } return builder.build(); } + + @Override + public int hashCode() { + return Objects.hash(account, privateKey); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ServiceAccountAuthCredentials)) { + return false; + } + ServiceAccountAuthCredentials other = (ServiceAccountAuthCredentials) obj; + return Objects.equals(account, other.account) + && Objects.equals(privateKey, other.privateKey); + } } private static class ComputeEngineAuthCredentials extends AuthCredentials { @@ -137,7 +162,7 @@ protected abstract HttpRequestInitializer httpRequestInitializer(HttpTransport t Set scopes); public static AuthCredentials createForAppEngine() { - return new AppEngineAuthCredentials(); + return AppEngineAuthCredentials.INSTANCE; } public static AuthCredentials createForComputeEngine() @@ -167,7 +192,7 @@ public static AuthCredentials createFor(String account, PrivateKey privateKey) { } public static AuthCredentials noCredentials() { - return new ServiceAccountAuthCredentials(); + return ServiceAccountAuthCredentials.NO_CREDENTIALS; } static ComputeCredential getComputeCredential() throws IOException, GeneralSecurityException { diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 76419d596958..ea3d37f80b9e 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -33,6 +33,7 @@ import java.lang.reflect.Method; import java.net.URL; import java.net.URLConnection; +import java.util.Objects; import java.util.Set; public abstract class ServiceOptions> implements Serializable { @@ -50,9 +51,9 @@ public interface HttpTransportFactory extends Serializable { HttpTransport create(); } - private static class DefaultHttpTransportFactory implements HttpTransportFactory { + private enum DefaultHttpTransportFactory implements HttpTransportFactory { - private static final long serialVersionUID = 2684907202489506911L; + INSTANCE; @Override public HttpTransport create() { @@ -85,7 +86,7 @@ protected abstract static class Builder, protected Builder() {} - protected Builder(ServiceOptions options) { + protected Builder(ServiceOptions options) { host = options.host; httpTransportFactory = options.httpTransportFactory; authCredentials = options.authCredentials; @@ -129,7 +130,7 @@ public B serviceRpcFactory(ServiceRpcFactory serviceRpcFactory) { protected ServiceOptions(Builder builder) { host = firstNonNull(builder.host, DEFAULT_HOST); httpTransportFactory = - firstNonNull(builder.httpTransportFactory, new DefaultHttpTransportFactory()); + firstNonNull(builder.httpTransportFactory, DefaultHttpTransportFactory.INSTANCE); authCredentials = firstNonNull(builder.authCredentials, defaultAuthCredentials()); retryParams = builder.retryParams; serviceRpcFactory = builder.serviceRpcFactory; @@ -224,5 +225,19 @@ public HttpRequestInitializer httpRequestInitializer() { return authCredentials().httpRequestInitializer(httpTransport, scopes()); } + @Override + public int hashCode() { + return Objects.hash(host, httpTransportFactory, authCredentials, retryParams, + serviceRpcFactory); + } + + protected boolean isEquals(ServiceOptions other) { + return Objects.equals(host, other.host) + && Objects.equals(httpTransportFactory, other.httpTransportFactory) + && Objects.equals(authCredentials, other.authCredentials) + && Objects.equals(retryParams, other.retryParams) + && Objects.equals(serviceRpcFactory, other.serviceRpcFactory); + } + public abstract Builder toBuilder(); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index de369d33f214..8d5f062fd548 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -32,6 +32,7 @@ import java.lang.reflect.Method; import java.util.Iterator; +import java.util.Objects; import java.util.Set; public class DatastoreServiceOptions extends ServiceOptions { @@ -174,6 +175,23 @@ public Builder toBuilder() { return new Builder(this); } + @Override + public int hashCode() { + return super.hashCode() ^ Objects.hash(dataset, namespace, force, normalizeDataset); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof DatastoreServiceOptions)) { + return false; + } + DatastoreServiceOptions other = (DatastoreServiceOptions) obj; + return isEquals(other) && Objects.equals(dataset, other.dataset) + && Objects.equals(namespace, other.namespace) + && Objects.equals(force, other.force) + && Objects.equals(normalizeDataset, other.normalizeDataset); + } + DatastoreRpc datastoreRpc() { if (serviceRpcFactory() != null) { return serviceRpcFactory().create(this); diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java index 60d1468cb90c..c5fc63a960b1 100644 --- a/src/main/java/com/google/gcloud/datastore/Value.java +++ b/src/main/java/com/google/gcloud/datastore/Value.java @@ -85,6 +85,7 @@ public final DatastoreV1.Value toProto(P value) { protected abstract void setValue(P from, DatastoreV1.Value.Builder to); } + @SuppressWarnings("deprecation") abstract static class BaseBuilder, B extends BaseBuilder> implements ValueBuilder { @@ -102,7 +103,6 @@ public ValueType getValueType() { return valueType; } - @SuppressWarnings("deprecation") @Override public B mergeFrom(P other) { indexed = other.indexed(); diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java index 8894cadad7d3..a08edeee7819 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java @@ -23,6 +23,7 @@ import com.google.gcloud.spi.ServiceRpcProvider; import com.google.gcloud.spi.StorageRpc; +import java.util.Objects; import java.util.Set; public class StorageServiceOptions extends ServiceOptions { @@ -96,6 +97,21 @@ public Builder toBuilder() { return new Builder(this); } + @Override + public int hashCode() { + return super.hashCode() ^ Objects.hash(project, pathDelimiter); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof StorageServiceOptions)) { + return false; + } + StorageServiceOptions other = (StorageServiceOptions) obj; + return isEquals(other) && Objects.equals(project, other.project) + && Objects.equals(pathDelimiter, other.pathDelimiter); + } + private static String defaultProject() { String projectId = System.getProperty(PROJECT_ENV_NAME, System.getenv(PROJECT_ENV_NAME)); if (projectId == null) { diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 870aee8add27..ed0da1e9ef13 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -23,6 +23,8 @@ import com.google.api.services.datastore.DatastoreV1; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; +import com.google.gcloud.AuthCredentials; +import com.google.gcloud.RetryParams; import com.google.gcloud.datastore.StructuredQuery.CompositeFilter; import com.google.gcloud.datastore.StructuredQuery.OrderBy; import com.google.gcloud.datastore.StructuredQuery.Projection; @@ -47,7 +49,7 @@ public class SerializationTest { private static final DateTime DATE_TIME1 = DateTime.now(); private static final Blob BLOB1 = Blob.copyFrom(UTF_8.encode("hello world")); private static final Cursor CURSOR1 = Cursor.copyFrom(new byte[] {1,2}); - private static final Cursor CURSOR2 = Cursor.copyFrom(new byte[] {10}); + private static final Cursor CURSOR2 = Cursor.copyFrom(new byte[]{10}); private static final Query GQL1 = Query.gqlQueryBuilder("select * from kind1 where name = @name and age > @1") .setBinding("name", "name1") @@ -128,6 +130,26 @@ public class SerializationTest { .put(ValueType.RAW_VALUE, RAW_VALUE) .build(); + @Test + public void testServiceOptions() throws Exception { + DatastoreServiceOptions options = DatastoreServiceOptions.builder() + .authCredentials(AuthCredentials.createForAppEngine()) + .normalizeDataset(false) + .dataset("ds1") + .build(); + DatastoreServiceOptions serializedCopy = serializeAndDeserialize(options); + assertEquals(options, serializedCopy); + + options = options.toBuilder() + .namespace("ns1") + .retryParams(RetryParams.getDefaultInstance()) + .authCredentials(AuthCredentials.noCredentials()) + .force(true) + .build(); + serializedCopy = serializeAndDeserialize(options); + assertEquals(options, serializedCopy); + } + @Test public void testValues() throws Exception { for (ValueType valueType : ValueType.values()) { @@ -157,7 +179,7 @@ public void testTypes() throws Exception { } @SuppressWarnings("unchecked") - private T serializeAndDeserialize(T obj) + private T serializeAndDeserialize(T obj) throws IOException, ClassNotFoundException { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { diff --git a/src/test/java/com/google/gcloud/storage/SerializationTest.java b/src/test/java/com/google/gcloud/storage/SerializationTest.java new file mode 100644 index 000000000000..dc340c01480a --- /dev/null +++ b/src/test/java/com/google/gcloud/storage/SerializationTest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import static org.junit.Assert.assertEquals; + +import com.google.gcloud.AuthCredentials; +import com.google.gcloud.RetryParams; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +public class SerializationTest { + + + @Test + public void testServiceOptions() throws Exception { + StorageServiceOptions options = StorageServiceOptions.builder() + .project("p1") + .authCredentials(AuthCredentials.createForAppEngine()) + .build(); + StorageServiceOptions serializedCopy = serializeAndDeserialize(options); + assertEquals(options, serializedCopy); + + options = options.toBuilder() + .project("p2") + .retryParams(RetryParams.getDefaultInstance()) + .authCredentials(AuthCredentials.noCredentials()) + .pathDelimiter(":") + .build(); + serializedCopy = serializeAndDeserialize(options); + assertEquals(options, serializedCopy); + } + + @Test + public void testTypes() throws Exception { + // todo: implement + } + + @SuppressWarnings("unchecked") + private T serializeAndDeserialize(T obj) + throws IOException, ClassNotFoundException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(obj); + } + try (ObjectInputStream input = + new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (T) input.readObject(); + } + } +} From a4076b7da82887b93344833c25836230a57ca066 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 24 Apr 2015 17:44:06 -0700 Subject: [PATCH 30/48] work in progress --- .../gcloud/examples/StorageExample.java | 2 +- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../gcloud/storage/BlobIterOptions.java | 1 + .../com/google/gcloud/storage/BlobList.java | 42 +++++++++++++++++++ .../gcloud/storage/BlobListOptions.java | 37 ++++++++++++++++ .../google/gcloud/storage/ListOptions.java | 1 - .../google/gcloud/storage/StorageService.java | 7 +++- .../gcloud/storage/StorageServiceImpl.java | 2 +- 9 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/google/gcloud/storage/BlobIterOptions.java create mode 100644 src/main/java/com/google/gcloud/storage/BlobList.java create mode 100644 src/main/java/com/google/gcloud/storage/BlobListOptions.java delete mode 100644 src/main/java/com/google/gcloud/storage/ListOptions.java diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index f65c8cdd02c2..071c7fad2de7 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -149,7 +149,7 @@ public void run(StorageService storage, String bucket) { System.out.println(buckets.next()); } } else { - Iterator blobs = storage.list(bucket, ListOptions.of()); + Iterator blobs = storage.list(bucket, BlobIterOptions.of()); while (blobs.hasNext()) { System.out.println(blobs.next()); } diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 5451375e64f8..e3fd45e528e0 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { try { return storage.buckets() .list(options.project()) .setProjection(DEFAULT_PROJECTION) .execute() .getItems() .iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, int limit) { // todo: implement return null; } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest // todo: missing setProjection try { return storage.objects() .compose(target.getBucket(), target.getName(), null) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName()); if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) { getRequest.setIfGenerationMatch(from.getGeneration()); } else { getRequest.setIfGenerationNotMatch(from.getGeneration()); } } if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) { getRequest.setIfMetagenerationMatch(from.getMetageneration()); } else { getRequest.setIfMetagenerationNotMatch(from.getMetageneration()); } } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { // todo: paging try { return storage.buckets() .list(options.project()) .setProjection(DEFAULT_PROJECTION) .execute() .getItems() .iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, long limit) { // todo: implement try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(includeOlderVersions) .setDelimiter(delimiter) .setPrefix(prefix) .setMaxResults(limit) .execute(); } catch (IOException ex) { throw translate(ex); } return null; } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest // todo: missing setProjection try { return storage.objects() .compose(target.getBucket(), target.getName(), null) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName()); if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) { getRequest.setIfGenerationMatch(from.getGeneration()); } else { getRequest.setIfGenerationNotMatch(from.getGeneration()); } } if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) { getRequest.setIfMetagenerationMatch(from.getMetageneration()); } else { getRequest.setIfMetagenerationNotMatch(from.getMetageneration()); } } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 17646392686f..af90731ac08d 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.Iterator; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_GENERATION_MATCH("ifGenerationMatch"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Iterator list() throws StorageServiceException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, int limit) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; void delete(Bucket bucket, Map options) throws StorageServiceException; void delete(StorageObject object, Map options) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.Iterator; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_GENERATION_MATCH("ifGenerationMatch"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Iterator list() throws StorageServiceException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, long limit) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; void delete(Bucket bucket, Map options) throws StorageServiceException; void delete(StorageObject object, Map options) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/BlobIterOptions.java b/src/main/java/com/google/gcloud/storage/BlobIterOptions.java new file mode 100644 index 000000000000..9ad1eeda85cb --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/BlobIterOptions.java @@ -0,0 +1 @@ +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class BlobIterOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final Integer maxResults; private final boolean includeOlderVersions; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; private boolean includeOlderVersions; Builder() {} public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } public Builder includeOlderVersions(boolean include) { this.includeOlderVersions = include; return this; } public BlobIterOptions build() { return new BlobIterOptions(this); } } BlobIterOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; maxResults = builder.maxResults; includeOlderVersions = builder.includeOlderVersions; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public Integer maxResults() { return maxResults; } public boolean includeOlderVersions() { return includeOlderVersions; } public static BlobIterOptions of() { return builder().build(); } public static Builder builder() { return new Builder(); } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/BlobList.java b/src/main/java/com/google/gcloud/storage/BlobList.java new file mode 100644 index 000000000000..c11c671f96e8 --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/BlobList.java @@ -0,0 +1,42 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import java.io.Serializable; +import java.util.List; + +/** + * Created by ozarov on 4/24/15. + */ +public class BlobList implements Serializable{ + + private final List blobs; + private final String cursor; + + BlobList(List blobs, String cursor) { + this.blobs = blobs; + this.cursor = cursor; + } + + public List result() { + return blobs; + } + + public String cursor() { + return cursor; + } +} diff --git a/src/main/java/com/google/gcloud/storage/BlobListOptions.java b/src/main/java/com/google/gcloud/storage/BlobListOptions.java new file mode 100644 index 000000000000..236f68f485cf --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/BlobListOptions.java @@ -0,0 +1,37 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +/** + * Created by ozarov on 4/24/15. + */ +public class BlobListOptions extends BlobIterOptions { + + private final String cursor; + + public Builder extends BlobIterOptions.Builder { + + } + + private BlobListOptions(Builder builder) { + super(builder); + } + + public String cursor() { + return cursor; + } +} diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java deleted file mode 100644 index d571b176d9ab..000000000000 --- a/src/main/java/com/google/gcloud/storage/ListOptions.java +++ /dev/null @@ -1 +0,0 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class ListOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final String cursor; private final Integer maxResults; private final boolean includeOlderVersions; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; private boolean includeOlderVersions; private Builder() {} public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } public Builder includeOlderVersions(boolean include) { this.includeOlderVersions = include; return this; } public ListOptions build() { return new ListOptions(this); } } private ListOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; cursor = builder.cursor; maxResults = builder.maxResults; includeOlderVersions = builder.includeOlderVersions; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public String cursor() { return cursor; } public Integer maxResults() { return maxResults; } public boolean includeOlderVersions() { return includeOlderVersions; } public static ListOptions of() { return builder().build(); } public static Builder builder() { return new Builder(); } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 2fb92cd66743..6d81dc5b8539 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -325,12 +325,15 @@ public static Builder builder() { /** * @throws StorageServiceException upon failure */ - Iterator list(); + Iterator buckets(); /** * @throws StorageServiceException upon failure */ - Iterator list(String bucket, ListOptions settings); + Iterator blobs(String bucket, BlobIterOptions settings); + + + BlobList blobs(String bucket, BlobListOptions settings); /** * @throws StorageServiceException upon failure diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index afaca940c9f0..2d6bc68dafa9 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -134,7 +134,7 @@ public Iterator call() { } @Override - public Iterator list(String bucket, ListOptions settings) { + public Iterator list(String bucket, BlobIterOptions settings) { // todo implement paging (with retries) with if limit is not given or > X String delimiter = settings.recursive() ? options().pathDelimiter() : null; return Iterators.transform( From 9cb77fa923468d1666cd00306c3acf2ea740a133 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 28 Apr 2015 17:37:36 -0700 Subject: [PATCH 31/48] work in progress --- pom.xml | 2 +- .../gcloud/examples/StorageExample.java | 118 +++++++++----- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../java/com/google/gcloud/storage/Acl.java | 5 + .../gcloud/storage/BlobIterOptions.java | 1 - .../gcloud/storage/BlobListOptions.java | 37 ----- .../com/google/gcloud/storage/Bucket.java | 19 ++- .../{BlobList.java => ListResult.java} | 23 +-- .../google/gcloud/storage/StorageService.java | 123 +++++++++----- .../gcloud/storage/StorageServiceImpl.java | 152 ++++++++++++------ .../com/google/gcloud/storage/Validator.java | 54 ------- 12 files changed, 302 insertions(+), 236 deletions(-) delete mode 100644 src/main/java/com/google/gcloud/storage/BlobIterOptions.java delete mode 100644 src/main/java/com/google/gcloud/storage/BlobListOptions.java rename src/main/java/com/google/gcloud/storage/{BlobList.java => ListResult.java} (63%) delete mode 100644 src/main/java/com/google/gcloud/storage/Validator.java diff --git a/pom.xml b/pom.xml index 06b070ecf874..c09da7963c42 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ com.google.apis google-api-services-storage - v1-rev23-1.19.0 + v1-rev33-1.20.0 compile diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 071c7fad2de7..b06a99217b99 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -16,7 +16,14 @@ package com.google.gcloud.examples; -import com.google.gcloud.storage.*; +import com.google.gcloud.spi.StorageRpc.Tuple; +import com.google.gcloud.storage.Blob; +import com.google.gcloud.storage.Bucket; +import com.google.gcloud.storage.StorageService; +import com.google.gcloud.storage.StorageService.CopyRequest; +import com.google.gcloud.storage.StorageService.ComposeRequest; +import com.google.gcloud.storage.StorageServiceFactory; +import com.google.gcloud.storage.StorageServiceOptions; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -24,7 +31,6 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; /** @@ -38,7 +44,9 @@ *
    1. compile using maven - {@code mvn compile}
      2. *
    2. run using maven - * {@code mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" - * -Dexec.args="project_id list []|info [ []]|get |upload []|delete "} + * -Dexec.args="project_id list []| info [ []]| get | + * upload []| delete | + * cp | compose + "} *
      3. *
    */ @@ -57,30 +65,6 @@ protected String params() { } } - private static class Tuple { - - private final X x; - private final Y y; - - private Tuple(X x, Y y) { - this.x = x; - this.y = y; - } - - static Tuple of(X x, Y y) { - return new Tuple<>(x, y); - } - - X first() { - return x; - } - - Y second() { - return y; - } - } - - private static abstract class BlobAction extends StorageAction { @Override @@ -144,14 +128,12 @@ String parse(String... args) { @Override public void run(StorageService storage, String bucket) { if (bucket == null) { - Iterator buckets = storage.list(); - while (buckets.hasNext()) { - System.out.println(buckets.next()); + for (Bucket b : storage.list()) { + System.out.println(b); } } else { - Iterator blobs = storage.list(bucket, BlobIterOptions.of()); - while (blobs.hasNext()) { - System.out.println(blobs.next()); + for (Blob b : storage.list(bucket)) { + System.out.println(b); } } } @@ -165,12 +147,12 @@ public String params() { private static class UploadAction extends StorageAction> { @Override public void run(StorageService storage, Tuple tuple) throws Exception { - if (Files.size(tuple.first()) > 1_000_000) { + if (Files.size(tuple.x()) > 1_000_000) { // todo: upload via streaming API throw new IllegalArgumentException("file is too big"); } else { - byte[] bytes = Files.readAllBytes(tuple.first()); - System.out.println(storage.create(tuple.second(), bytes)); + byte[] bytes = Files.readAllBytes(tuple.x()); + System.out.println(storage.create(tuple.y(), bytes)); } } @@ -196,6 +178,7 @@ public void run(StorageService storage, Blob blob) { blob = storage.get(blob); if (blob == null) { System.out.println("No such object"); + return; } if (blob.size() < 1_000_000) { System.out.println(new String(storage.load(blob), StandardCharsets.UTF_8)); @@ -206,27 +189,73 @@ public void run(StorageService storage, Blob blob) { } } + private static class CopyAction extends StorageAction { + @Override + public void run(StorageService storage, CopyRequest request) { + System.out.println(storage.copy(request)); + } + + @Override + CopyRequest parse(String... args) { + if (args.length != 4) { + throw new IllegalArgumentException(); + } + return CopyRequest.of(Blob.of(args[0], args[1]), Blob.of(args[2], args[3])); + } + + @Override + public String params() { + return " "; + } + } + + private static class ComposeAction extends StorageAction { + @Override + public void run(StorageService storage, ComposeRequest request) { + System.out.println(storage.compose(request)); + } + + @Override + ComposeRequest parse(String... args) { + if (args.length < 3) { + throw new IllegalArgumentException(); + } + ComposeRequest.Builder request = ComposeRequest.builder(); + request.target(Blob.of(args[0], args[args.length - 1])); + for (int i = 1; i < args.length - 1; i++) { + request.addSource(args[i]); + } + return request.build(); + } + + @Override + public String params() { + return " + "; + } + } + static { ACTIONS.put("info", new InfoAction()); ACTIONS.put("delete", new DeleteAction()); ACTIONS.put("list", new ListAction()); ACTIONS.put("upload", new UploadAction()); ACTIONS.put("get", new GetAction()); + ACTIONS.put("cp", new CopyAction()); + ACTIONS.put("compose", new ComposeAction()); } public static void printUsage() { - StringBuilder actionAndParams = new StringBuilder(); + StringBuilder actionAndParams = new StringBuilder(""); for (Map.Entry entry : ACTIONS.entrySet()) { - actionAndParams.append(entry.getKey()); + actionAndParams.append("\n\t").append(entry.getKey()); String param = entry.getValue().params(); if (param != null && !param.isEmpty()) { actionAndParams.append(' ').append(param); } - actionAndParams.append('|'); } - actionAndParams.setLength(actionAndParams.length() - 1); - System.out.printf("Usage: %s [%s]%n", StorageExample.class.getSimpleName(), actionAndParams); + System.out.printf("Usage: %s *%s%n", + StorageExample.class.getSimpleName(), actionAndParams); } @SuppressWarnings("unchecked") @@ -236,7 +265,6 @@ public static void main(String... args) throws Exception { printUsage(); return; } - String projectId = args[0]; StorageAction action = ACTIONS.get(args[1]); if (action == null) { System.out.println("Unrecognized action '" + args[1] + "'"); @@ -249,10 +277,14 @@ public static void main(String... args) throws Exception { Object request; try { request = action.parse(args); - } catch (Exception ex) { + } catch (IllegalArgumentException ex) { System.out.println("Invalid input for action '" + args[1] + "'"); System.out.println("Expected: " + action.params()); return; + } catch (Exception ex) { + System.out.println("Failed to parse request."); + ex.printStackTrace(); + return; } action.run(storage, request); } diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index e3fd45e528e0..1d999d23d5ef 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list() { // todo: paging try { return storage.buckets() .list(options.project()) .setProjection(DEFAULT_PROJECTION) .execute() .getItems() .iterator(); } catch (IOException ex) { throw translate(ex); } } @Override public Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean includeOlderVersions, long limit) { // todo: implement try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(includeOlderVersions) .setDelimiter(delimiter) .setPrefix(prefix) .setMaxResults(limit) .execute(); } catch (IOException ex) { throw translate(ex); } return null; } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets().delete(bucket.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects().delete(blob.getBucket(), blob.getName()).execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { // todo: implement null -> ComposeRequest // todo: missing setProjection try { return storage.objects() .compose(target.getBucket(), target.getName(), null) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target) .setProjection(DEFAULT_PROJECTION) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName()); if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) { getRequest.setIfGenerationMatch(from.getGeneration()); } else { getRequest.setIfGenerationNotMatch(from.getGeneration()); } } if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) { if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) { getRequest.setIfMetagenerationMatch(from.getMetageneration()); } else { getRequest.setIfMetagenerationNotMatch(from.getMetageneration()); } } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { request.setDestination(target); } List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index af90731ac08d..1140fc7469f8 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.Iterator; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_GENERATION_MATCH("ifGenerationMatch"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Iterator list() throws StorageServiceException; Iterator list(String bucket, String prefix, String delimiter, String cursor, boolean versions, long limit) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; void delete(Bucket bucket, Map options) throws StorageServiceException; void delete(StorageObject object, Map options) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; void delete(Bucket bucket, Map options) throws StorageServiceException; void delete(StorageObject object, Map options) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index 3058386d7201..10673b5aeae0 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -35,6 +35,7 @@ public enum Role { public static abstract class Entity implements Serializable { private static final long serialVersionUID = -2707407252771255840L; + private final Type type; private final String value; @@ -152,6 +153,8 @@ public static User ofAllAuthenticatedUsers() { public static class Project extends Entity { + private static final long serialVersionUID = 7933776866530023027L; + private final ProjectRole pRole; private final String projectId; @@ -176,6 +179,8 @@ public String projectId() { public static class RawEntity extends Entity { + private static final long serialVersionUID = 3966205614223053950L; + RawEntity(String entity) { super(Type.UNKNOWN, entity); } diff --git a/src/main/java/com/google/gcloud/storage/BlobIterOptions.java b/src/main/java/com/google/gcloud/storage/BlobIterOptions.java deleted file mode 100644 index 9ad1eeda85cb..000000000000 --- a/src/main/java/com/google/gcloud/storage/BlobIterOptions.java +++ /dev/null @@ -1 +0,0 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.storage; import java.io.Serializable; public class BlobIterOptions implements Serializable { private static final long serialVersionUID = -7260643619742432140L; private final boolean recursive; private final String prefix; private final Integer maxResults; private final boolean includeOlderVersions; public static class Builder { private boolean recursive; private String prefix; private String cursor; private int maxResults; private boolean includeOlderVersions; Builder() {} public Builder recursive(boolean recursive) { this.recursive = recursive; return this; } public Builder prefix(String prefix) { this.prefix = prefix; return this; } public Builder cursor(String cursor) { this.cursor = cursor; return this; } public Builder maxResults(Integer maxResults) { this.maxResults = maxResults; return this; } public Builder includeOlderVersions(boolean include) { this.includeOlderVersions = include; return this; } public BlobIterOptions build() { return new BlobIterOptions(this); } } BlobIterOptions(Builder builder) { recursive = builder.recursive; prefix = builder.prefix; maxResults = builder.maxResults; includeOlderVersions = builder.includeOlderVersions; } public boolean recursive() { return recursive; } public String prefix() { return prefix; } public Integer maxResults() { return maxResults; } public boolean includeOlderVersions() { return includeOlderVersions; } public static BlobIterOptions of() { return builder().build(); } public static Builder builder() { return new Builder(); } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/BlobListOptions.java b/src/main/java/com/google/gcloud/storage/BlobListOptions.java deleted file mode 100644 index 236f68f485cf..000000000000 --- a/src/main/java/com/google/gcloud/storage/BlobListOptions.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.storage; - -/** - * Created by ozarov on 4/24/15. - */ -public class BlobListOptions extends BlobIterOptions { - - private final String cursor; - - public Builder extends BlobIterOptions.Builder { - - } - - private BlobListOptions(Builder builder) { - super(builder); - } - - public String cursor() { - return cursor; - } -} diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index 16937fda2b4b..981dc7f788c0 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -19,6 +19,7 @@ import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Lists.transform; +import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.Bucket.Lifecycle; import com.google.api.services.storage.model.Bucket.Lifecycle.Rule; @@ -33,6 +34,9 @@ import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.Acl.Entity; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.List; @@ -148,7 +152,9 @@ void populateCondition(Rule.Condition condition) { static class RawDeleteRule extends DeleteRule { - private final Rule rule; + private static final long serialVersionUID = -7166938278642301933L; + + private transient Rule rule; RawDeleteRule(Rule rule) { super(Type.UNKNOWN); @@ -159,6 +165,17 @@ void populateCondition(Rule.Condition condition) { throw new UnsupportedOperationException(); } + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + out.writeUTF(rule.toString()); + } + + private void readObject(ObjectInputStream in) throws IOException, + ClassNotFoundException { + in.defaultReadObject(); + rule = new JacksonFactory().fromString(in.readUTF(), Rule.class); + } + Rule toPb() { return rule; } diff --git a/src/main/java/com/google/gcloud/storage/BlobList.java b/src/main/java/com/google/gcloud/storage/ListResult.java similarity index 63% rename from src/main/java/com/google/gcloud/storage/BlobList.java rename to src/main/java/com/google/gcloud/storage/ListResult.java index c11c671f96e8..180e5f7a4155 100644 --- a/src/main/java/com/google/gcloud/storage/BlobList.java +++ b/src/main/java/com/google/gcloud/storage/ListResult.java @@ -17,26 +17,29 @@ package com.google.gcloud.storage; import java.io.Serializable; -import java.util.List; +import java.util.Iterator; /** - * Created by ozarov on 4/24/15. + * Google Cloud storage list result. */ -public class BlobList implements Serializable{ +public class ListResult implements Iterable, Serializable { + + private static final long serialVersionUID = -6937287874908527950L; - private final List blobs; private final String cursor; + private final Iterable results; - BlobList(List blobs, String cursor) { - this.blobs = blobs; + ListResult(String cursor, Iterable results) { this.cursor = cursor; + this.results = results; } - public List result() { - return blobs; + public String nextPageCursor() { + return cursor; } - public String cursor() { - return cursor; + @Override + public Iterator iterator() { + return results.iterator(); } } diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 6d81dc5b8539..714343a96c5d 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -17,7 +17,6 @@ package com.google.gcloud.storage; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.gcloud.storage.Validator.checkBlobOptions; import com.google.common.collect.ImmutableList; import com.google.gcloud.Service; @@ -26,7 +25,6 @@ import java.io.Serializable; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -130,43 +128,90 @@ public static BlobSourceOption metagenerationMatch(boolean match) { } } + class BucketListOption extends Option { + + private static final long serialVersionUID = 8754017079673290353L; + + private BucketListOption(StorageRpc.Option option, Object value) { + super(option, value); + } + + public static BucketListOption maxResults(long maxResults) { + return new BucketListOption(StorageRpc.Option.MAX_RESULTS, maxResults); + } + + public static BucketListOption startPageToken(String pageToken) { + return new BucketListOption(StorageRpc.Option.PAGE_TOKEN, pageToken); + } + + public static BucketListOption prefix(String prefix) { + return new BucketListOption(StorageRpc.Option.PREFIX, prefix); + } + } + + class BlobListOption extends Option { + + private static final long serialVersionUID = 9083383524788661294L; + + private BlobListOption(StorageRpc.Option option, Object value) { + super(option, value); + } + + public static BlobListOption maxResults(long maxResults) { + return new BlobListOption(StorageRpc.Option.MAX_RESULTS, maxResults); + } + + public static BlobListOption startPageToken(String pageToken) { + return new BlobListOption(StorageRpc.Option.PAGE_TOKEN, pageToken); + } + + public static BlobListOption prefix(String prefix) { + return new BlobListOption(StorageRpc.Option.PREFIX, prefix); + } + + public static BlobListOption recursive(boolean recursive) { + return new BlobListOption(StorageRpc.Option.DELIMITER, recursive); + } + } + class ComposeRequest implements Serializable { private static final long serialVersionUID = -7385681353748590911L; - private final String sourceBucket; private final List sourceBlobs; private final Blob target; private final List targetOptions; - static class SourceBlob implements Serializable { + public static class SourceBlob implements Serializable { private static final long serialVersionUID = 4094962795951990439L; - final String blob; + final String name; final Long generation; - SourceBlob(String blob) { - this(blob, null); + SourceBlob(String name) { + this(name, null); } - SourceBlob(String blob, Long generation) { - this.blob = blob; + SourceBlob(String name, Long generation) { + this.name = name; this.generation = generation; } + + public String name() { + return name; + } + + public Long generation() { + return generation; + } } public static class Builder { - private String bucket; - private List sourceBlobs = new LinkedList<>(); + private final List sourceBlobs = new LinkedList<>(); private Blob target; - private Set targetOptions = new LinkedHashSet<>(); - - public Builder sourceBucket(String bucket) { - this.bucket = bucket; - return this; - } + private final Set targetOptions = new LinkedHashSet<>(); public Builder addSource(Iterable blobs) { for (String blob : blobs) { @@ -195,36 +240,33 @@ public Builder targetOptions(BlobTargetOption... options) { } public ComposeRequest build() { - checkNotNull(bucket); checkNotNull(target); - checkBlobOptions("Target", target, targetOptions); return new ComposeRequest(this); } } private ComposeRequest(Builder builder) { - sourceBucket = builder.bucket; sourceBlobs = ImmutableList.copyOf(builder.sourceBlobs); target = builder.target; targetOptions = ImmutableList.copyOf(builder.targetOptions); } - String sourceBucket() { - return sourceBucket; - } - - List sourceBlobs() { + public List sourceBlobs() { return sourceBlobs; } - Blob target() { + public Blob target() { return target; } - List targetOptions() { + public List targetOptions() { return targetOptions; } + public static ComposeRequest of(Iterable sources, Blob target) { + return builder().target(target).addSource(sources).build(); + } + public static Builder builder() { return new Builder(); } @@ -242,12 +284,13 @@ class CopyRequest implements Serializable { public static class Builder { private Blob source; - private Set sourceOptions; + private final Set sourceOptions = new LinkedHashSet<>(); private Blob target; - private Set targetOptions; + private final Set targetOptions = new LinkedHashSet<>(); - public void source(Blob source) { + public Builder source(Blob source) { this.source = source; + return this; } public Builder sourceOptions(BlobSourceOption... options) { @@ -268,8 +311,6 @@ public Builder targetOptions(BlobTargetOption... options) { public CopyRequest build() { checkNotNull(source); checkNotNull(target); - checkBlobOptions("Source", source, sourceOptions); - checkBlobOptions("Target", target, targetOptions); return new CopyRequest(this); } } @@ -281,7 +322,7 @@ private CopyRequest(Builder builder) { targetOptions = ImmutableList.copyOf(builder.targetOptions); } - Blob source() { + public Blob source() { return source; } @@ -289,14 +330,18 @@ public List sourceOptions() { return sourceOptions; } - Blob target() { + public Blob target() { return target; } - List targetOptions() { + public List targetOptions() { return targetOptions; } + public static CopyRequest of(Blob source, Blob target) { + return builder().source(source).target(target).build(); + } + public static Builder builder() { return new Builder(); } @@ -325,15 +370,13 @@ public static Builder builder() { /** * @throws StorageServiceException upon failure */ - Iterator buckets(); + ListResult list(BucketListOption... options); /** + * Lists blobs for a bucket. * @throws StorageServiceException upon failure */ - Iterator blobs(String bucket, BlobIterOptions settings); - - - BlobList blobs(String bucket, BlobListOptions settings); + ListResult list(String bucket, BlobListOption... options); /** * @throws StorageServiceException upon failure diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 2d6bc68dafa9..5288c064e626 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -16,21 +16,24 @@ package com.google.gcloud.storage; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.gcloud.RetryHelper.runWithRetries; import com.google.api.services.storage.model.StorageObject; +import com.google.common.base.Function; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterators; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.gcloud.BaseService; import com.google.gcloud.ExceptionHandler; import com.google.gcloud.ExceptionHandler.Interceptor; import com.google.gcloud.RetryParams; import com.google.gcloud.spi.StorageRpc; +import com.google.gcloud.spi.StorageRpc.Tuple; import java.util.Arrays; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -70,9 +73,8 @@ public RetryResult beforeEval(Exception exception) { @Override public Bucket create(Bucket bucket, BucketTargetOption... options) { - Validator.checkBucketOptions(bucket, options); final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(bucket, options); return Bucket.fromPb(runWithRetries( new Callable() { @Override @@ -84,9 +86,8 @@ public com.google.api.services.storage.model.Bucket call() { @Override public Blob create(Blob blob, final byte[] content, BlobTargetOption... options) { - Validator.checkBlobOptions(blob, options); final StorageObject blobPb = blob.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(blob, options); return Blob.fromPb(runWithRetries(new Callable() { @Override public StorageObject call() { @@ -97,9 +98,8 @@ public StorageObject call() { @Override public Bucket get(Bucket bucket, BucketSourceOption... options) { - Validator.checkBucketOptions(bucket, options); final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(bucket, options); return Bucket.fromPb(runWithRetries( new Callable() { @Override @@ -111,9 +111,8 @@ public com.google.api.services.storage.model.Bucket call() { @Override public Blob get(Blob blob, BlobSourceOption... options) { - Validator.checkBlobOptions(blob, options); final StorageObject storedObject = blob.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(blob, options); return Blob.fromPb(runWithRetries(new Callable() { @Override public StorageObject call() { @@ -123,30 +122,47 @@ public StorageObject call() { } @Override - public Iterator list() { - return Iterators.transform( - runWithRetries(new Callable>() { + public ListResult list(BucketListOption... options) { + final Map optionsMap = optionMap(null, null, options); + Tuple> result = runWithRetries( + new Callable>>() { @Override - public Iterator call() { - return storageRpc.list(); + public Tuple> call() { + return storageRpc.list(optionsMap); } - }, retryParams, EXCEPTION_HANDLER), Bucket.FROM_PB_FUNCTION); + }, retryParams, EXCEPTION_HANDLER); + return new ListResult<>(result.x(), Iterables.transform(result.y(), + new Function() { + @Override + public Bucket apply(com.google.api.services.storage.model.Bucket bucketPb) { + return Bucket.fromPb(bucketPb); + } + })); } @Override - public Iterator list(String bucket, BlobIterOptions settings) { - // todo implement paging (with retries) with if limit is not given or > X - String delimiter = settings.recursive() ? options().pathDelimiter() : null; - return Iterators.transform( - storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(), - settings.includeOlderVersions(), settings.maxResults()), Blob.FROM_PB_FUNCTION); + public ListResult list(final String bucket, BlobListOption... options) { + final Map optionsMap = optionMap(null, null, options); + Tuple> result = runWithRetries( + new Callable>>() { + @Override + public Tuple> call() { + return storageRpc.list(bucket, optionsMap); + } + }, retryParams, EXCEPTION_HANDLER); + return new ListResult<>(result.x(), Iterables.transform(result.y(), + new Function() { + @Override + public Blob apply(StorageObject storageObject) { + return Blob.fromPb(storageObject); + } + })); } @Override public Bucket update(Bucket bucket, BucketTargetOption... options) { - Validator.checkBucketOptions(bucket, options); final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(bucket, options); return Bucket.fromPb(runWithRetries( new Callable() { @Override @@ -158,9 +174,8 @@ public com.google.api.services.storage.model.Bucket call() { @Override public Blob update(Blob blob, BlobTargetOption... options) { - Validator.checkBlobOptions(blob, options); final StorageObject storageObject = blob.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(blob, options); return Blob.fromPb(runWithRetries(new Callable() { @Override public StorageObject call() { @@ -171,9 +186,8 @@ public StorageObject call() { @Override public void delete(Bucket bucket, BucketSourceOption... options) { - Validator.checkBucketOptions(bucket, options); final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(bucket, options); runWithRetries(new Callable() { @Override public Void call() { @@ -185,9 +199,8 @@ public Void call() { @Override public void delete(Blob blob, BlobSourceOption... options) { - Validator.checkBlobOptions(blob, options); final StorageObject storageObject = blob.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(blob, options); runWithRetries(new Callable() { @Override public Void call() { @@ -202,11 +215,12 @@ public Blob compose(final ComposeRequest composeRequest) { final List sources = Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size()); for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) { - sources.add(Blob.builder(composeRequest.sourceBucket(), sourceBlob.blob) - .generation(sourceBlob.generation).build().toPb()); + sources.add(Blob.builder(composeRequest.target().bucket(), sourceBlob.name()) + .generation(sourceBlob.generation()).build().toPb()); } final StorageObject target = composeRequest.target().toPb(); - final Map targetOptions = optionMap(composeRequest.targetOptions()); + final Map targetOptions = optionMap(composeRequest.target().generation(), + composeRequest.target().metageneration(), composeRequest.targetOptions()); return Blob.fromPb(runWithRetries(new Callable() { @Override public StorageObject call() { @@ -218,9 +232,12 @@ public StorageObject call() { @Override public Blob copy(CopyRequest copyRequest) { final StorageObject source = copyRequest.source().toPb(); - final Map sourceOptions = optionMap(copyRequest.sourceOptions()); + copyRequest.sourceOptions(); + final Map sourceOptions = optionMap(copyRequest.source().generation(), + copyRequest.source().metageneration(), copyRequest.sourceOptions(), true); final StorageObject target = copyRequest.target().toPb(); - final Map targetOptions = optionMap(copyRequest.targetOptions()); + final Map targetOptions = optionMap(copyRequest.target().generation(), + copyRequest.target().metageneration(), copyRequest.targetOptions()); return Blob.fromPb(runWithRetries(new Callable() { @Override public StorageObject call() { @@ -231,9 +248,8 @@ public StorageObject call() { @Override public byte[] load(Blob blob, BlobSourceOption... options) { - Validator.checkBlobOptions(blob, options); final StorageObject storageObject = blob.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(blob, options); return runWithRetries(new Callable() { @Override public byte[] call() { @@ -244,29 +260,71 @@ public byte[] call() { @Override public BlobReadChannel reader(Blob blob, BlobSourceOption... options) { - Validator.checkBlobOptions(blob, options); // todo: Use retry helper on retriable failures - final Map optionsMap = optionMap(options); + // todo: consider changing lower level api to handle segments + final Map optionsMap = optionMap(blob, options); return storageRpc.reader(blob.toPb(), optionsMap); } @Override public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) { - Validator.checkBlobOptions(blob, options); // todo: Use retry helper on retriable failures - final Map optionsMap = optionMap(options); + // todo: consider changing lower level api to handle segments + final Map optionsMap = optionMap(blob, options); return storageRpc.writer(blob.toPb(), optionsMap); } - private static Map optionMap(Iterable options) { - ImmutableMap.Builder mapBuider = ImmutableMap.builder(); + private Map optionMap(Long generation, Long metaGeneration, + Iterable options) { + return optionMap(generation, metaGeneration, options, false); + } + + private Map optionMap(Long generation, Long metaGeneration, + Iterable options, boolean useAsSource) { + Map temp = Maps.newEnumMap(StorageRpc.Option.class); for (Option option : options) { - mapBuider.put(option.rpcOption(), option.value()); + Object prev = temp.put(option.rpcOption(), option.value()); + checkArgument(prev == null, "Duplicate option %s", option); + } + Boolean value = (Boolean) temp.remove(StorageRpc.Option.DELIMITER); + if (Boolean.TRUE.equals(value)) { + temp.put(StorageRpc.Option.DELIMITER, options().pathDelimiter()); } - return mapBuider.build(); + value = (Boolean) temp.remove(StorageRpc.Option.IF_GENERATION_MATCH); + if (value != null) { + checkArgument(generation != null, "missing generation value"); + if (value) { + temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_GENERATION_MATCH + : StorageRpc.Option.IF_GENERATION_MATCH, generation); + } else { + temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH + : StorageRpc.Option.IF_GENERATION_NOT_MATCH, generation); + } + } + value = (Boolean) temp.remove(StorageRpc.Option.IF_METAGENERATION_MATCH); + if (value != null) { + checkArgument(metaGeneration != null, "missing metaGeneration value"); + if (value) { + temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH + : StorageRpc.Option.IF_METAGENERATION_MATCH, metaGeneration); + } else { + temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH + : StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metaGeneration); + } + } + return ImmutableMap.copyOf(temp); + } + + private Map optionMap(Long generation, Long metaGeneration, + Option... options) { + return optionMap(generation, metaGeneration, Arrays.asList(options)); + } + + private Map optionMap(Bucket bucket, Option... options) { + return optionMap(null, bucket.metageneration(), options); } - private static Map optionMap(Option... options) { - return optionMap(Arrays.asList(options)); + private Map optionMap(Blob blob, Option... options) { + return optionMap(blob.generation(), blob.metageneration(), options); } } diff --git a/src/main/java/com/google/gcloud/storage/Validator.java b/src/main/java/com/google/gcloud/storage/Validator.java deleted file mode 100644 index 5590d67528cc..000000000000 --- a/src/main/java/com/google/gcloud/storage/Validator.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.storage; - -import static com.google.common.base.Preconditions.checkArgument; - -import java.util.Arrays; - -/** - * Utility to validate Storage type/values. - */ -public class Validator { - - static void checkBlobOptions(Blob blob, Option... options) { - checkBlobOptions("requested", blob, Arrays.asList(options)); - } - - static void checkBlobOptions(String name, Blob blob, Iterable options) { - for (Option option : options) { - switch (option.rpcOption()) { - case IF_GENERATION_MATCH: - checkArgument(blob.generation() != 0, "%s blob is missing generation", name); - break; - case IF_METAGENERATION_MATCH: - checkArgument(blob.metageneration() != 0, "%s blob is missing metageneration", name); - break; - } - } - } - - static void checkBucketOptions(Bucket bucket, Option... options) { - for (Option option : options) { - switch (option.rpcOption()) { - case IF_METAGENERATION_MATCH: - checkArgument(bucket.metageneration() != 0, "bucket is missing metageneration"); - break; - } - } - } -} From acff3c160062c1fdae1739e9da09df66965fa165 Mon Sep 17 00:00:00 2001 From: ozarov Date: Tue, 28 Apr 2015 20:26:27 -0700 Subject: [PATCH 32/48] add default content type to compose --- src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 1d999d23d5ef..35ee9ec22a1a 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { request.setDestination(target); } List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { target.setContentType("binary/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file From ad396b77829e5a473d42953fd2aeee21d8c1be2c Mon Sep 17 00:00:00 2001 From: ozarov Date: Wed, 29 Apr 2015 17:49:13 -0700 Subject: [PATCH 33/48] work in progress --- src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 35ee9ec22a1a..0c2ef339a092 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { target.setContentType("binary/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file From 6df1164c1cf1a872045e0c0aac5dc3a86db25837 Mon Sep 17 00:00:00 2001 From: ozarov Date: Fri, 1 May 2015 14:43:18 -0700 Subject: [PATCH 34/48] adding batch --- .../gcloud/examples/StorageExample.java | 39 +++++++++++++++++-- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../google/gcloud/storage/BatchRequest.java | 34 ++++++++++++++++ .../google/gcloud/storage/BatchResponse.java | 23 +++++++++++ .../google/gcloud/storage/StorageService.java | 8 +++- .../gcloud/storage/StorageServiceImpl.java | 32 ++++++++++----- 7 files changed, 122 insertions(+), 18 deletions(-) create mode 100644 src/main/java/com/google/gcloud/storage/BatchRequest.java create mode 100644 src/main/java/com/google/gcloud/storage/BatchResponse.java diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index b06a99217b99..b6c0a74d0ab3 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -17,6 +17,7 @@ package com.google.gcloud.examples; import com.google.gcloud.spi.StorageRpc.Tuple; +import com.google.gcloud.storage.BatchRequest; import com.google.gcloud.storage.Blob; import com.google.gcloud.storage.Bucket; import com.google.gcloud.storage.StorageService; @@ -45,7 +46,7 @@ *
  • run using maven - * {@code mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" * -Dexec.args="project_id list []| info [ []]| get | - * upload []| delete | + * upload []| delete +| * cp | compose + "} *
    • * @@ -81,6 +82,26 @@ public String params() { } } + private static abstract class BlobsAction extends StorageAction { + + @Override + Blob[] parse(String... args) { + if (args.length < 2) { + throw new IllegalArgumentException(); + } + Blob[] blobs = new Blob[args.length - 1]; + for (int i = 1; i < args.length; i++) { + blobs[i - 1] = Blob.of(args[0], args[i]); + } + return blobs; + } + + @Override + public String params() { + return " +"; + } + } + private static class InfoAction extends BlobAction { @Override public void run(StorageService storage, Blob blob) { @@ -105,10 +126,20 @@ public String params() { } } - private static class DeleteAction extends BlobAction { + private static class DeleteAction extends BlobsAction { @Override - public void run(StorageService storage, Blob blob) { - storage.delete(blob); + public void run(StorageService storage, Blob... blobs) { + if (blobs.length == 1) { + System.out.println("Calling delete"); + System.out.println(storage.delete(blobs[0])); + } else { + BatchRequest batch = new BatchRequest(); + for (Blob blob : blobs) { + batch.delete(blob); + } + System.out.println("Calling batch.delete"); + storage.apply(batch); + } } } diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 0c2ef339a092..119d4ad1c698 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails(); boolean retryable = RETRYABLE_CODES.contains(details.getCode()) || "InternalError".equals(details.getMessage()); translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public void delete(StorageObject blob, Map options) { try { storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.*; import com.google.api.client.googleapis.batch.BatchRequest; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public void batch(Iterable>> toDelete) throws StorageServiceException { BatchRequest batch = storage.batch(); final Map failures = Maps.newConcurrentMap(); try { for (final Tuple> tuple : toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void storageObject, HttpHeaders responseHeaders) { // do nothing } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { failures.put(tuple.x(), translate(e)); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 1140fc7469f8..c41ac53c6e41 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; void delete(Bucket bucket, Map options) throws StorageServiceException; void delete(StorageObject object, Map options) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; void batch(Iterable>> toDelete) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java new file mode 100644 index 000000000000..0f9e56f8a866 --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import com.google.gcloud.storage.StorageService.BlobSourceOption; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Created by ozarov on 4/30/15. + */ +public class BatchRequest { + + Map toDelete = new LinkedHashMap<>(); + + public void delete(Blob blob, BlobSourceOption... options) { + toDelete.put(blob, options); + } +} diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java new file mode 100644 index 000000000000..6fe0a14fd31e --- /dev/null +++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java @@ -0,0 +1,23 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +/** + * Created by ozarov on 4/30/15. + */ +public class BatchResponse { +} diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 714343a96c5d..5933f6bdfeef 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -391,12 +391,13 @@ public static Builder builder() { /** * @throws StorageServiceException upon failure */ - void delete(Bucket bucket, BucketSourceOption... options); + boolean delete(Bucket bucket, BucketSourceOption... options); /** * @throws StorageServiceException upon failure */ - void delete(Blob blob, BlobSourceOption... options); + boolean delete(Blob blob, BlobSourceOption... options); + /** * @throws StorageServiceException upon failure @@ -414,6 +415,9 @@ public static Builder builder() { */ byte[] load(Blob blob, BlobSourceOption... options); + + BatchResponse apply(BatchRequest batchRequest); + /** * @throws StorageServiceException upon failure */ diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 5288c064e626..9dc4aa558441 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -185,27 +185,25 @@ public StorageObject call() { } @Override - public void delete(Bucket bucket, BucketSourceOption... options) { + public boolean delete(Bucket bucket, BucketSourceOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb(); final Map optionsMap = optionMap(bucket, options); - runWithRetries(new Callable() { + return runWithRetries(new Callable() { @Override - public Void call() { - storageRpc.delete(bucketPb, optionsMap); - return null; + public Boolean call() { + return storageRpc.delete(bucketPb, optionsMap); } }, retryParams, EXCEPTION_HANDLER); } @Override - public void delete(Blob blob, BlobSourceOption... options) { + public boolean delete(Blob blob, BlobSourceOption... options) { final StorageObject storageObject = blob.toPb(); final Map optionsMap = optionMap(blob, options); - runWithRetries(new Callable() { + return runWithRetries(new Callable() { @Override - public Void call() { - storageRpc.delete(storageObject, optionsMap); - return null; + public Boolean call() { + return storageRpc.delete(storageObject, optionsMap); } }, retryParams, EXCEPTION_HANDLER); } @@ -258,6 +256,20 @@ public byte[] call() { }, retryParams, EXCEPTION_HANDLER); } + @Override + public BatchResponse apply(BatchRequest batchRequest) { + BatchResponse response = new BatchResponse(); + List>> request = + Lists.newArrayListWithCapacity(batchRequest.toDelete.size()); + for (Map.Entry entry : batchRequest.toDelete.entrySet()) { + Blob blob = entry.getKey(); + Map optionsMap = optionMap(blob, entry.getValue()); + request.add(Tuple.>of(blob.toPb(), optionsMap)); + } + storageRpc.batch(request); + return response; + } + @Override public BlobReadChannel reader(Blob blob, BlobSourceOption... options) { // todo: Use retry helper on retriable failures From a4ee293c3d60e1e21bf18621d45895f26918b300 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 1 May 2015 17:39:00 -0700 Subject: [PATCH 35/48] work in progress --- .../gcloud/examples/StorageExample.java | 4 +- .../google/gcloud/storage/BatchRequest.java | 59 +++++++++++-- .../google/gcloud/storage/BatchResponse.java | 83 ++++++++++++++++++- .../google/gcloud/storage/StorageService.java | 2 - .../gcloud/storage/StorageServiceImpl.java | 13 ++- 5 files changed, 146 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index b6c0a74d0ab3..ee37b7e937d4 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -133,12 +133,12 @@ public void run(StorageService storage, Blob... blobs) { System.out.println("Calling delete"); System.out.println(storage.delete(blobs[0])); } else { - BatchRequest batch = new BatchRequest(); + BatchRequest.Builder batch = BatchRequest.builder(); for (Blob blob : blobs) { batch.delete(blob); } System.out.println("Calling batch.delete"); - storage.apply(batch); + storage.apply(batch.build()); } } } diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java index 0f9e56f8a866..0056f354ac2b 100644 --- a/src/main/java/com/google/gcloud/storage/BatchRequest.java +++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java @@ -16,19 +16,68 @@ package com.google.gcloud.storage; +import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageService.BlobSourceOption; +import com.google.gcloud.storage.StorageService.BlobTargetOption; +import java.io.Serializable; import java.util.LinkedHashMap; import java.util.Map; /** - * Created by ozarov on 4/30/15. + * Google storage batch request. */ -public class BatchRequest { +public class BatchRequest implements Serializable { - Map toDelete = new LinkedHashMap<>(); + private static final long serialVersionUID = -1527992265939800345L; - public void delete(Blob blob, BlobSourceOption... options) { - toDelete.put(blob, options); + private Map toDelete; + private Map toUpdate; + private Map toGet; + + public static class Builder { + + private Map toDelete = new LinkedHashMap<>(); + private Map toUpdate = new LinkedHashMap<>(); + private Map toGet = new LinkedHashMap<>(); + + private Builder() {} + + public void delete(Blob blob, BlobSourceOption... options) { + toDelete.put(blob, options); + } + + public void update(Blob blob, BlobTargetOption... options) { + toUpdate.put(blob, options); + } + + public void get(Blob blob, BlobSourceOption... options) { + toGet.put(blob, options); + } + + public BatchRequest build() { + return new BatchRequest(this); + } + } + + private BatchRequest(Builder builder) { + toDelete = ImmutableMap.copyOf(builder.toDelete); + toUpdate = ImmutableMap.copyOf(builder.toUpdate); + } + + public Map toDelete() { + return toDelete; + } + + public Map toUpdate() { + return toUpdate; + } + + public Map toGet() { + return toGet; + } + + public static Builder builder() { + return new Builder(); } } diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java index 6fe0a14fd31e..9c835b54fbee 100644 --- a/src/main/java/com/google/gcloud/storage/BatchResponse.java +++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java @@ -16,8 +16,87 @@ package com.google.gcloud.storage; +import com.google.common.collect.ImmutableList; + +import java.io.Serializable; +import java.util.List; + /** - * Created by ozarov on 4/30/15. + * Google Storage batch response. */ -public class BatchResponse { +public class BatchResponse implements Serializable { + + private List> deleteResult; + private List> updateResult; + private List> getResult; + + public static class Result implements Serializable { + + private static final long serialVersionUID = -1946539570170529094L; + + private final T value; + private final StorageServiceException exception; + + Result(T value) { + this.value = value; + this.exception = null; + } + + Result(StorageServiceException exception) { + this.exception = exception; + this.value = null; + } + + + /** + * Returns the result. + * + * @throws StorageServiceException if failed + */ + public T result() throws StorageServiceException { + return value; + } + + /** + * Returns the failure or {@code null} if was successful. + */ + public StorageServiceException failure() { + return exception; + } + + /** + * Returns {@code true} if failed, {@code false} otherwise. + */ + public boolean failed() { + return exception != null; + } + } + + BatchResponse(List> deleteResult, List> updateResult, + List> getResult) { + this.deleteResult = ImmutableList.copyOf(deleteResult); + this.updateResult = ImmutableList.copyOf(updateResult); + this.getResult = ImmutableList.copyOf(getResult); + } + + /** + * Returns the results for the delete operations using the request order. + */ + public List> deletes() { + return deleteResult; + } + + /** + * Returns the results for the update operations using the request order. + */ + public List> updates() { + return updateResult; + } + + /** + * Returns the results for the get operations using the request order. + */ + public List> gets() { + return getResult; + } } diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 5933f6bdfeef..71026fda6078 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -398,7 +398,6 @@ public static Builder builder() { */ boolean delete(Blob blob, BlobSourceOption... options); - /** * @throws StorageServiceException upon failure */ @@ -409,7 +408,6 @@ public static Builder builder() { */ Blob copy(CopyRequest copyRequest); - /** * @throws StorageServiceException upon failure */ diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 9dc4aa558441..478df47949c9 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -34,6 +34,7 @@ import com.google.gcloud.spi.StorageRpc.Tuple; import java.util.Arrays; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -258,16 +259,20 @@ public byte[] call() { @Override public BatchResponse apply(BatchRequest batchRequest) { - BatchResponse response = new BatchResponse(); + List> deletes = new LinkedList<>(); + List> updates = new LinkedList<>(); + List> gets = new LinkedList<>(); List>> request = - Lists.newArrayListWithCapacity(batchRequest.toDelete.size()); - for (Map.Entry entry : batchRequest.toDelete.entrySet()) { + Lists.newArrayListWithCapacity(batchRequest.toDelete().size()); + for (Map.Entry entry : batchRequest.toDelete().entrySet()) { Blob blob = entry.getKey(); Map optionsMap = optionMap(blob, entry.getValue()); request.add(Tuple.>of(blob.toPb(), optionsMap)); } + // todo: populate deletes, updates and gets (the latter 2 needs to be sent to RPC) + // todo: change example to use multi-update and multi-get storageRpc.batch(request); - return response; + return new BatchResponse(deletes, updates, gets); } @Override From 2ed5ce181bcad063237092663b7ca849274d8ccd Mon Sep 17 00:00:00 2001 From: ozarov Date: Sat, 2 May 2015 12:51:08 -0700 Subject: [PATCH 36/48] work in progress --- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../gcloud/storage/StorageServiceImpl.java | 37 ++++++++++++++++--- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 119d4ad1c698..b31964d40c55 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.*; import com.google.api.client.googleapis.batch.BatchRequest; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public void batch(Iterable>> toDelete) throws StorageServiceException { BatchRequest batch = storage.batch(); final Map failures = Maps.newConcurrentMap(); try { for (final Tuple> tuple : toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void storageObject, HttpHeaders responseHeaders) { // do nothing } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { failures.put(tuple.x(), translate(e)); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.*; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index c41ac53c6e41..4b7600135f6c 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; void batch(Iterable>> toDelete) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 478df47949c9..50c7bb039c7c 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -259,19 +259,44 @@ public byte[] call() { @Override public BatchResponse apply(BatchRequest batchRequest) { - List> deletes = new LinkedList<>(); - List> updates = new LinkedList<>(); - List> gets = new LinkedList<>(); - List>> request = + List>> toDelete = Lists.newArrayListWithCapacity(batchRequest.toDelete().size()); for (Map.Entry entry : batchRequest.toDelete().entrySet()) { Blob blob = entry.getKey(); Map optionsMap = optionMap(blob, entry.getValue()); - request.add(Tuple.>of(blob.toPb(), optionsMap)); + toDelete.add(Tuple.>of(blob.toPb(), optionsMap)); + } + List>> toUpdate = + Lists.newArrayListWithCapacity(batchRequest.toUpdate().size()); + for (Map.Entry entry : batchRequest.toUpdate().entrySet()) { + Blob blob = entry.getKey(); + Map optionsMap = optionMap(blob, entry.getValue()); + toUpdate.add(Tuple.>of(blob.toPb(), optionsMap)); + } + List>> toGet = + Lists.newArrayListWithCapacity(batchRequest.toGet().size()); + for (Map.Entry entry : batchRequest.toGet().entrySet()) { + Blob blob = entry.getKey(); + Map optionsMap = optionMap(blob, entry.getValue()); + toGet.add(Tuple.>of(blob.toPb(), optionsMap)); } // todo: populate deletes, updates and gets (the latter 2 needs to be sent to RPC) // todo: change example to use multi-update and multi-get - storageRpc.batch(request); + StorageRpc.BatchResponse response = + storageRpc.batch(new StorageRpc.BatchRequest(toDelete, toUpdate, toGet)); + List> deletes = new LinkedList<>(); + for (Blob blob : batchRequest.toDelete().keySet()) { + Tuple result = response.deletes.get(blob); + if (result.x() != null) { + deletes.add(new BatchResponse.Result<>(false)); + } else { + deletes.add(new BatchResponse.Result(result.y())); + } + } + + List> updates = new LinkedList<>(); + List> gets = new LinkedList<>(); + return new BatchResponse(deletes, updates, gets); } From 4c2f40e88a03b2d44aae3a5b1030459deb5189ea Mon Sep 17 00:00:00 2001 From: ozarov Date: Sun, 3 May 2015 17:17:04 -0700 Subject: [PATCH 37/48] complete batch --- .../gcloud/storage/StorageServiceImpl.java | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 50c7bb039c7c..45e5430c1505 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -21,6 +21,7 @@ import com.google.api.services.storage.model.StorageObject; import com.google.common.base.Function; +import com.google.common.base.Functions; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; @@ -33,8 +34,8 @@ import com.google.gcloud.spi.StorageRpc; import com.google.gcloud.spi.StorageRpc.Tuple; +import java.io.Serializable; import java.util.Arrays; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -264,7 +265,8 @@ public BatchResponse apply(BatchRequest batchRequest) { for (Map.Entry entry : batchRequest.toDelete().entrySet()) { Blob blob = entry.getKey(); Map optionsMap = optionMap(blob, entry.getValue()); - toDelete.add(Tuple.>of(blob.toPb(), optionsMap)); + StorageObject storageObject = blob.toPb(); + toDelete.add(Tuple.>of(storageObject, optionsMap)); } List>> toUpdate = Lists.newArrayListWithCapacity(batchRequest.toUpdate().size()); @@ -280,24 +282,30 @@ public BatchResponse apply(BatchRequest batchRequest) { Map optionsMap = optionMap(blob, entry.getValue()); toGet.add(Tuple.>of(blob.toPb(), optionsMap)); } - // todo: populate deletes, updates and gets (the latter 2 needs to be sent to RPC) - // todo: change example to use multi-update and multi-get StorageRpc.BatchResponse response = storageRpc.batch(new StorageRpc.BatchRequest(toDelete, toUpdate, toGet)); - List> deletes = new LinkedList<>(); - for (Blob blob : batchRequest.toDelete().keySet()) { - Tuple result = response.deletes.get(blob); + List> deletes = transformBatchResult( + toDelete, response.deletes, Functions.identity()); + List> updates = transformBatchResult( + toUpdate, response.updates, Blob.FROM_PB_FUNCTION); + List> gets = transformBatchResult( + toGet, response.gets, Blob.FROM_PB_FUNCTION); + return new BatchResponse(deletes, updates, gets); + } + + private List> transformBatchResult( + Iterable>> request, + Map> results, Function transform) { + List> response = Lists.newArrayListWithCapacity(results.size()); + for (Tuple tuple : request) { + Tuple result = results.get(tuple.x()); if (result.x() != null) { - deletes.add(new BatchResponse.Result<>(false)); + response.add(new BatchResponse.Result<>(transform.apply(result.x()))); } else { - deletes.add(new BatchResponse.Result(result.y())); + response.add(new BatchResponse.Result(result.y())); } } - - List> updates = new LinkedList<>(); - List> gets = new LinkedList<>(); - - return new BatchResponse(deletes, updates, gets); + return response; } @Override From 59410aba329e3b319f4a66cae5bf64a642de6d9f Mon Sep 17 00:00:00 2001 From: ozarov Date: Sun, 3 May 2015 17:50:12 -0700 Subject: [PATCH 38/48] add multi info to example --- .../gcloud/examples/StorageExample.java | 33 ++++++++++++------- .../google/gcloud/storage/BatchRequest.java | 7 ++-- .../google/gcloud/storage/BatchResponse.java | 6 ++++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index ee37b7e937d4..00d16b19f713 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -18,11 +18,12 @@ import com.google.gcloud.spi.StorageRpc.Tuple; import com.google.gcloud.storage.BatchRequest; +import com.google.gcloud.storage.BatchResponse; import com.google.gcloud.storage.Blob; import com.google.gcloud.storage.Bucket; import com.google.gcloud.storage.StorageService; -import com.google.gcloud.storage.StorageService.CopyRequest; import com.google.gcloud.storage.StorageService.ComposeRequest; +import com.google.gcloud.storage.StorageService.CopyRequest; import com.google.gcloud.storage.StorageServiceFactory; import com.google.gcloud.storage.StorageServiceOptions; @@ -102,27 +103,36 @@ public String params() { } } - private static class InfoAction extends BlobAction { + private static class InfoAction extends BlobsAction { @Override - public void run(StorageService storage, Blob blob) { - if (blob.name().isEmpty()) { - System.out.println(storage.get(Bucket.of(blob.bucket()))); + public void run(StorageService storage, Blob... blobs) { + if (blobs.length == 1) { + if (blobs[0].name().isEmpty()) { + System.out.println(storage.get(Bucket.of(blobs[0].bucket()))); + } else { + System.out.println(storage.get(blobs[0])); + } } else { - System.out.println(storage.get(blob)); + BatchRequest.Builder batch = BatchRequest.builder(); + for (Blob blob : blobs) { + batch.get(blob); + } + BatchResponse response = storage.apply(batch.build()); + System.out.println(response.gets()); } } @Override - Blob parse(String... args) { + Blob[] parse(String... args) { if (args.length < 2) { - return Blob.of(args[0], ""); + return new Blob[] {Blob.of(args[0], "")}; } return super.parse(args); } @Override public String params() { - return " []"; + return " [+]"; } } @@ -130,15 +140,14 @@ private static class DeleteAction extends BlobsAction { @Override public void run(StorageService storage, Blob... blobs) { if (blobs.length == 1) { - System.out.println("Calling delete"); System.out.println(storage.delete(blobs[0])); } else { BatchRequest.Builder batch = BatchRequest.builder(); for (Blob blob : blobs) { batch.delete(blob); } - System.out.println("Calling batch.delete"); - storage.apply(batch.build()); + BatchResponse response = storage.apply(batch.build()); + System.out.println(response.deletes()); } } } diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java index 0056f354ac2b..4f27abd89547 100644 --- a/src/main/java/com/google/gcloud/storage/BatchRequest.java +++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java @@ -31,9 +31,9 @@ public class BatchRequest implements Serializable { private static final long serialVersionUID = -1527992265939800345L; - private Map toDelete; - private Map toUpdate; - private Map toGet; + private final Map toDelete; + private final Map toUpdate; + private final Map toGet; public static class Builder { @@ -63,6 +63,7 @@ public BatchRequest build() { private BatchRequest(Builder builder) { toDelete = ImmutableMap.copyOf(builder.toDelete); toUpdate = ImmutableMap.copyOf(builder.toUpdate); + toGet = ImmutableMap.copyOf(builder.toGet); } public Map toDelete() { diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java index 9c835b54fbee..bef428c37a74 100644 --- a/src/main/java/com/google/gcloud/storage/BatchResponse.java +++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java @@ -16,6 +16,7 @@ package com.google.gcloud.storage; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import java.io.Serializable; @@ -70,6 +71,11 @@ public StorageServiceException failure() { public boolean failed() { return exception != null; } + + @Override + public String toString() { + return MoreObjects.firstNonNull(value, exception).toString(); + } } BatchResponse(List> deleteResult, List> updateResult, From f0920d0552c9ac83640856187cf9bd25c8669ceb Mon Sep 17 00:00:00 2001 From: ozarov Date: Sun, 3 May 2015 18:01:43 -0700 Subject: [PATCH 39/48] work in progress --- src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index b31964d40c55..4bf7b8ec647d 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.*; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.*; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement (consider passing a range from, to) and return ByteBuffer try { storage.objects().get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); //.getMediaHttpDownloader().setContentRange() } catch (IOException ex) { throw translate(ex); } return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement (consider passing headers, resumable-write, from // and get byte buffer return null; } } \ No newline at end of file From 64da673f705f6c04c9ce8cfd7cf55a34637d48f2 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 4 May 2015 12:31:40 -0700 Subject: [PATCH 40/48] find project-id from env --- .../com/google/gcloud/ServiceOptions.java | 39 +++++++++++++++---- .../gcloud/examples/StorageExample.java | 19 ++++++--- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index ea3d37f80b9e..082caa7eb6ee 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -27,14 +27,19 @@ import com.google.gcloud.spi.ServiceRpcFactory; import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; import java.lang.reflect.Method; +import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLConnection; import java.util.Objects; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public abstract class ServiceOptions> implements Serializable { @@ -168,16 +173,36 @@ protected static String appEngineAppId() { protected static String googleCloudProjectId() { try { URL url = new URL("http://metadata/computeMetadata/v1/project/project-id"); - URLConnection connection = url.openConnection(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestProperty("X-Google-Metadata-Request", "True"); - try (BufferedReader reader = - new BufferedReader(new InputStreamReader(connection.getInputStream(), UTF_8))) { - return reader.readLine(); + InputStream input = connection.getInputStream(); + if (connection.getResponseCode() == 200) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) { + return reader.readLine(); + } } } catch (IOException ignore) { - // return null if can't determine - return null; + // ignore + } + String configDir = System.getenv("CLOUDSDK_CONFIG"); + if (configDir == null) { + configDir = new File(System.getProperty("user.home"), "/.config/gcloud/").getPath(); + } + try (BufferedReader reader = + new BufferedReader(new FileReader(new File(configDir, "properties")))) { + String line; + Pattern pattern = Pattern.compile("^\\s*project\\s*=\\s*(.*?)\\s*$"); + while((line = reader.readLine()) != null) { + Matcher matcher = pattern.matcher(line); + if (matcher.matches()) { + return matcher.group(1); + } + } + } catch (IOException ex) { + // ignore } + // return null if can't determine + return null; } protected static String getAppEngineProjectId() { diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 00d16b19f713..c93d7d0885a4 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -46,7 +46,7 @@ *
  • compile using maven - {@code mvn compile}
    • *
  • run using maven - * {@code mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" - * -Dexec.args="project_id list []| info [ []]| get | + * -Dexec.args="[] list []| info [ []]| get | * upload []| delete +| * cp | compose + "} *
    • @@ -300,20 +300,27 @@ public static void printUsage() { @SuppressWarnings("unchecked") public static void main(String... args) throws Exception { - if (args.length < 2) { + if (args.length < 1) { System.out.println("Missing required project id and action"); printUsage(); return; } - StorageAction action = ACTIONS.get(args[1]); + StorageServiceOptions.Builder optionsBuilder = StorageServiceOptions.builder(); + StorageAction action; + if (args.length >= 2 && !ACTIONS.containsKey(args[0])) { + optionsBuilder.project(args[0]); + action = ACTIONS.get(args[1]); + args = Arrays.copyOfRange(args, 2, args.length); + } else { + action = ACTIONS.get(args[0]); + args = Arrays.copyOfRange(args, 1, args.length); + } if (action == null) { System.out.println("Unrecognized action '" + args[1] + "'"); printUsage(); return; } - StorageServiceOptions options = StorageServiceOptions.builder().project(args[0]).build(); - StorageService storage = StorageServiceFactory.instance().get(options); - args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length) : new String[] {}; + StorageService storage = StorageServiceFactory.instance().get(optionsBuilder.build()); Object request; try { request = action.parse(args); From e3222e5b6d51531aab8a20f532b6e0c9ddd2df32 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 4 May 2015 13:30:14 -0700 Subject: [PATCH 41/48] update ini hanlding --- .../java/com/google/gcloud/ServiceOptions.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 082caa7eb6ee..0b03e7951e86 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -191,11 +191,22 @@ protected static String googleCloudProjectId() { try (BufferedReader reader = new BufferedReader(new FileReader(new File(configDir, "properties")))) { String line; - Pattern pattern = Pattern.compile("^\\s*project\\s*=\\s*(.*?)\\s*$"); + String section = null; + Pattern projectPattern = Pattern.compile("^project\\s*=\\s*(.*)$"); + Pattern sectionPattern = Pattern.compile("^\\[(.*)\\]$"); while((line = reader.readLine()) != null) { - Matcher matcher = pattern.matcher(line); + if (line.isEmpty() || line.startsWith(";")) { + continue; + } + line = line.trim(); + Matcher matcher = sectionPattern.matcher(line); if (matcher.matches()) { - return matcher.group(1); + section = matcher.group(1); + } else if (section == null || section.equals("core")) { + matcher = projectPattern.matcher(line); + if (matcher.matches()) { + return matcher.group(1); + } } } } catch (IOException ex) { From 9b751aee2b873b2cabc84ee5e080d6c6241c524a Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 4 May 2015 16:51:49 -0700 Subject: [PATCH 42/48] implement Serializable reader --- .../gcloud/examples/StorageExample.java | 24 ++-- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../gcloud/storage/BlobReadChannel.java | 3 +- .../gcloud/storage/StorageServiceImpl.java | 111 +++++++++++++++++- .../gcloud/storage/StorageServiceOptions.java | 1 + 6 files changed, 129 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index c93d7d0885a4..5fa003c5672c 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -16,10 +16,13 @@ package com.google.gcloud.examples; +import static java.nio.charset.StandardCharsets.UTF_8; + import com.google.gcloud.spi.StorageRpc.Tuple; import com.google.gcloud.storage.BatchRequest; import com.google.gcloud.storage.BatchResponse; import com.google.gcloud.storage.Blob; +import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.Bucket; import com.google.gcloud.storage.StorageService; import com.google.gcloud.storage.StorageService.ComposeRequest; @@ -27,7 +30,8 @@ import com.google.gcloud.storage.StorageServiceFactory; import com.google.gcloud.storage.StorageServiceOptions; -import java.nio.charset.StandardCharsets; +import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -214,17 +218,23 @@ public String params() { private static class GetAction extends BlobAction { @Override - public void run(StorageService storage, Blob blob) { + public void run(StorageService storage, Blob blob) throws IOException { blob = storage.get(blob); if (blob == null) { System.out.println("No such object"); return; } - if (blob.size() < 1_000_000) { - System.out.println(new String(storage.load(blob), StandardCharsets.UTF_8)); + if (blob.size() < 1024) { + System.out.println(new String(storage.load(blob), UTF_8)); } else { - // todo: download via streaming API - throw new IllegalArgumentException("file is too big"); + try (BlobReadChannel reader = storage.reader(blob)) { + ByteBuffer bytes = ByteBuffer.allocate(64 * 1024); + while (reader.read(bytes) > 0) { + bytes.flip(); + System.out.print(UTF_8.decode(bytes)); + bytes.clear(); + } + } } } } @@ -294,7 +304,7 @@ public static void printUsage() { actionAndParams.append(' ').append(param); } } - System.out.printf("Usage: %s *%s%n", + System.out.printf("Usage: %s [] operation *%s%n", StorageExample.class.getSimpleName(), actionAndParams); } diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 4bf7b8ec647d..0f45f06830b7 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.*; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException { // todo: implement (consider passing a range from, to) and return ByteBuffer try { storage.objects().get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); //.getMediaHttpDownloader().setContentRange() } catch (IOException ex) { throw translate(ex); } return null; } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement (consider passing headers, resumable-write, from // and get byte buffer return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); downloader.setContentRange(position, position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement (consider passing headers, resumable-write, from // and get byte buffer return null; } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 4b7600135f6c..14adcb6b502e 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.BlobReadChannel; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; BlobReadChannel reader(StorageObject from, Map options) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java index 32b68b3e97f6..89f3420a2a28 100644 --- a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java +++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java @@ -17,6 +17,7 @@ package com.google.gcloud.storage; import java.io.Closeable; +import java.io.IOException; import java.io.Serializable; import java.nio.channels.ReadableByteChannel; @@ -37,5 +38,5 @@ public interface BlobReadChannel extends ReadableByteChannel, Serializable, Clos @Override void close(); - void seek(int position); + void seek(int position) throws IOException; } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 45e5430c1505..88c3a8df126b 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -34,7 +34,11 @@ import com.google.gcloud.spi.StorageRpc; import com.google.gcloud.spi.StorageRpc.Tuple; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.Serializable; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -308,12 +312,111 @@ private List> transformBatch return response; } + private static class BlobReadChannelImpl implements BlobReadChannel { + + private final StorageServiceOptions serviceOptions; + private final Blob blob; + private final Map requestOptions; + private int position; + private boolean isOpen; + private boolean endOfStream; + + private transient StorageRpc storageRpc; + private transient RetryParams retryParams; + private transient StorageObject storageObject; + private transient int bufferPos; + private transient byte[] buffer; + + BlobReadChannelImpl(StorageServiceOptions serviceOptions, Blob blob, + Map requestOptions) { + this.serviceOptions = serviceOptions; + this.blob = blob; + this.requestOptions = requestOptions; + isOpen = true; + initTransients(); + } + + private void writeObject(ObjectOutputStream out) throws IOException { + if (buffer != null) { + position += bufferPos; + buffer = null; + bufferPos = 0; + endOfStream = false; + } + out.defaultWriteObject(); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + initTransients(); + } + + private void initTransients() { + storageRpc = serviceOptions.storageRpc(); + retryParams = MoreObjects.firstNonNull(serviceOptions.retryParams(), RetryParams.noRetries()); + storageObject = blob.toPb(); + } + + @Override + public boolean isOpen() { + return isOpen; + } + + @Override + public void close() { + if (isOpen) { + buffer = null; + isOpen = false; + } + } + + private void validateOpen() throws IOException { + if (!isOpen) { + throw new IOException("stream is closed"); + } + } + + @Override + public void seek(int position) throws IOException { + validateOpen(); + throw new UnsupportedOperationException("not supported yet"); + // todo: implement + } + + @Override + public int read(ByteBuffer byteBuffer) throws IOException { + validateOpen(); + if (buffer == null) { + if (endOfStream) { + return -1; + } + final int toRead = Math.max(byteBuffer.remaining(), 256 * 1024); + buffer = runWithRetries(new Callable() { + @Override + public byte[] call() { + return storageRpc.read(storageObject, requestOptions, position, toRead); + } + }, retryParams, EXCEPTION_HANDLER); + if (toRead > buffer.length) { + endOfStream = true; + } + } + int toWrite = Math.min(buffer.length - bufferPos, byteBuffer.remaining()); + byteBuffer.put(buffer, bufferPos, toWrite); + bufferPos += toWrite; + if (bufferPos >= buffer.length) { + position += buffer.length; + buffer = null; + bufferPos = 0; + } + return toWrite; + } + } + @Override public BlobReadChannel reader(Blob blob, BlobSourceOption... options) { - // todo: Use retry helper on retriable failures - // todo: consider changing lower level api to handle segments - final Map optionsMap = optionMap(blob, options); - return storageRpc.reader(blob.toPb(), optionsMap); + Map optionsMap = optionMap(blob, options); + return new BlobReadChannelImpl(options(), blob, optionsMap); } @Override diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java index a08edeee7819..ed16dd3863a6 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java @@ -70,6 +70,7 @@ private StorageServiceOptions(Builder builder) { pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER); project = builder.project != null ? builder.project : defaultProject(); Preconditions.checkArgument(project != null, "Missing required project id"); + // todo: consider providing read-timeout } @Override From 2d279f42ad248e4a160a34ce9a1128bf9010523b Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 4 May 2015 17:35:16 -0700 Subject: [PATCH 43/48] work in progress --- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../gcloud/storage/StorageServiceImpl.java | 44 ++++++++++++++++--- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 0f45f06830b7..4bf69d1dba70 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); downloader.setContentRange(position, position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException { // todo: implement (consider passing headers, resumable-write, from // and get byte buffer return null; } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); downloader.setContentRange(position, position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public void append(byte[] bytes, StorageObject to, String uploadId, int position, boolean last) throws StorageServiceException { return; } @Override public String write(byte[] bytes, StorageObject to, Map options) throws StorageServiceException { String type = MoreObjects.firstNonNull(to.getContentType(), "application/octet-stream"); try { Insert req = storage.objects().insert(to.getBucket(), to, new ByteArrayContent(type, bytes)); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)); req.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)); req.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)); req.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); System.out.println("KOKO.request: " + req.getRequestMethod()); System.out.println("KOKO.request.autherization: " + req.getRequestHeaders().getAuthorization()); System.out.println("KOKO.request.authToken: " + req.getOauthToken()); return null; } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 14adcb6b502e..ddfdd311e386 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException; BlobWriteChannel writer(StorageObject to, Map options) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException; String write(byte[] bytes, StorageObject to, Map options) throws StorageServiceException; void append(byte[] bytes, StorageObject to, String uploadId, int position, boolean last) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 88c3a8df126b..56e4293dd9ad 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -75,6 +75,10 @@ public RetryResult beforeEval(Exception exception) { storageRpc = options.storageRpc(); retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries()); // todo: replace nulls with Value.asNull (per toPb) + // todo: configure timeouts - https://developers.google.com/api-client-library/java/google-api-java-client/errors + // todo: provide rewrite - https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite + // todo: provide signed urls - https://cloud.google.com/storage/docs/access-control#Signed-URLs + // todo: check if we need to expose https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert vs using bucket update/patch } @Override @@ -379,8 +383,10 @@ private void validateOpen() throws IOException { @Override public void seek(int position) throws IOException { validateOpen(); - throw new UnsupportedOperationException("not supported yet"); - // todo: implement + this.position = position; + buffer = null; + bufferPos = 0; + endOfStream = false; } @Override @@ -419,12 +425,40 @@ public BlobReadChannel reader(Blob blob, BlobSourceOption... options) { return new BlobReadChannelImpl(options(), blob, optionsMap); } + private static class BlobWriterChannelImpl implements BlobWriteChannel { + + private final StorageServiceOptions options; + private final Blob blob; + private final Map optionsMap; + + public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob, + Map optionsMap) { + this.options = options; + this.blob = blob; + this.optionsMap = optionsMap; + } + + @Override + public int write(ByteBuffer byteBuffer) throws IOException { + // todo: Use retry helper on retriable failures + return 0; + } + + @Override + public boolean isOpen() { + return false; + } + + @Override + public void close() throws IOException { + + } + } + @Override public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) { - // todo: Use retry helper on retriable failures - // todo: consider changing lower level api to handle segments final Map optionsMap = optionMap(blob, options); - return storageRpc.writer(blob.toPb(), optionsMap); + return new BlobWriterChannelImpl(options(), blob, optionsMap); } private Map optionMap(Long generation, Long metaGeneration, From 4260e8ee8b5b1d6f59fae63148fa115df7f11b16 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 5 May 2015 15:37:59 -0700 Subject: [PATCH 44/48] work in progress --- .../gcloud/datastore/DatastoreServiceOptions.java | 4 ++++ .../com/google/gcloud/examples/StorageExample.java | 10 ++++++++++ .../com/google/gcloud/spi/DefaultStorageRpc.java | 2 +- src/main/java/com/google/gcloud/spi/StorageRpc.java | 2 +- .../google/gcloud/storage/StorageServiceImpl.java | 12 +++++++++++- .../google/gcloud/storage/StorageServiceOptions.java | 4 ++++ 6 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index 8d5f062fd548..b1b1ee31638c 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -199,6 +199,10 @@ DatastoreRpc datastoreRpc() { return ServiceRpcProvider.datastore(this); } + public static DatastoreServiceOptions defaultIntance() { + return builder().build(); + } + public static Builder builder() { return new Builder(); } diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 5fa003c5672c..0f9650a63bcc 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -23,6 +23,7 @@ import com.google.gcloud.storage.BatchResponse; import com.google.gcloud.storage.Blob; import com.google.gcloud.storage.BlobReadChannel; +import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Bucket; import com.google.gcloud.storage.StorageService; import com.google.gcloud.storage.StorageService.ComposeRequest; @@ -311,6 +312,15 @@ public static void printUsage() { @SuppressWarnings("unchecked") public static void main(String... args) throws Exception { if (args.length < 1) { + + System.out.println("KOKO.start -------------------------------"); + StorageService storage = + StorageServiceFactory.instance().get(StorageServiceOptions.defaultInstnace()); + BlobWriteChannel writer = storage + .writer(Blob.of("ozarov-javamrsample.appspot.com", "bla")); + writer.write(ByteBuffer.wrap("hello world".getBytes())); + writer.close(); + System.out.println("KOKO.end -------------------------------"); System.out.println("Missing required project id and action"); printUsage(); return; diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 4bf69d1dba70..938ab24c1ce8 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() != null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); downloader.setContentRange(position, position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public void append(byte[] bytes, StorageObject to, String uploadId, int position, boolean last) throws StorageServiceException { return; } @Override public String write(byte[] bytes, StorageObject to, Map options) throws StorageServiceException { String type = MoreObjects.firstNonNull(to.getContentType(), "application/octet-stream"); try { Insert req = storage.objects().insert(to.getBucket(), to, new ByteArrayContent(type, bytes)); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)); req.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)); req.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)); req.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); System.out.println("KOKO.request: " + req.getRequestMethod()); System.out.println("KOKO.request.autherization: " + req.getRequestHeaders().getAuthorization()); System.out.println("KOKO.request.authToken: " + req.getOauthToken()); return null; } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); downloader.setContentRange(position, position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public void write(byte[] bytes, StorageObject to, String uploadId, int position, boolean last) throws StorageServiceException { return; } @Override public String open(StorageObject object, Map options) throws StorageServiceException { String type = MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"); try { Insert req = storage.objects().insert(object.getBucket(), object); try { GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object value = option.get(options); if (value != null) { url.set(option.name(), value.toString()); } } HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, null); httpRequest.getHeaders().setContentType(type); httpRequest.getHeaders().setContentLength(0L); System.out.println("koko.url: " + httpRequest.getUrl()); System.out.println("koko.contentType:" + httpRequest.getHeaders().getContentType()); System.out.println("koko.contentLength:" + httpRequest.getHeaders().getContentLength()); httpRequest.setLoggingEnabled(true); httpRequest.setCurlLoggingEnabled(true); HttpResponse response = httpRequest.execute(); System.out.println("koko.response: " + response.toString()); System.out.println("koko.response.code: " + response.getStatusCode()); System.out.println("koko.response.message: " + response.getStatusMessage()); System.out.println("koko.response.content: " + response.getContentCharset()); } catch (Exception ex) { ex.printStackTrace(); } return null; } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index ddfdd311e386..5bf96a6ea824 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException; String write(byte[] bytes, StorageObject to, Map options) throws StorageServiceException; void append(byte[] bytes, StorageObject to, String uploadId, int position, boolean last) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException; String open(StorageObject object, Map options) throws StorageServiceException; void write(byte[] bytes, StorageObject object, String uploadId, int position, boolean last) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 56e4293dd9ad..bd2ef247d103 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -430,18 +430,28 @@ private static class BlobWriterChannelImpl implements BlobWriteChannel { private final StorageServiceOptions options; private final Blob blob; private final Map optionsMap; + private final String uploadId; + private int position; public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob, Map optionsMap) { this.options = options; this.blob = blob; this.optionsMap = optionsMap; + System.out.println("Koko. BlobWriterChannelImpl.init"); + uploadId = options.storageRpc().open(blob.toPb(), optionsMap); } @Override public int write(ByteBuffer byteBuffer) throws IOException { + int size = Math.min(byteBuffer.remaining(), 1024 * 1024); + byte[] bytes = new byte[size]; + byteBuffer.get(bytes); // todo: Use retry helper on retriable failures - return 0; + System.out.println("Koko. BlobWriterChannelImpl.write"); + options.storageRpc().write(bytes, blob.toPb(), uploadId, position, false); + position += size; + return size; } @Override diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java index ed16dd3863a6..24dea9c91788 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java @@ -121,6 +121,10 @@ private static String defaultProject() { return projectId != null ? projectId : googleCloudProjectId(); } + public static StorageServiceOptions defaultInstnace() { + return builder().build(); + } + public static Builder builder() { return new Builder(); } From e830ad33e3bbee940416e007e8c43b940581eb8a Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 5 May 2015 17:41:39 -0700 Subject: [PATCH 45/48] work in progress --- src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +- src/main/java/com/google/gcloud/spi/StorageRpc.java | 2 +- src/main/java/com/google/gcloud/storage/StorageServiceImpl.java | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 938ab24c1ce8..71b94384bbb2 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); downloader.setContentRange(position, position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public void write(byte[] bytes, StorageObject to, String uploadId, int position, boolean last) throws StorageServiceException { return; } @Override public String open(StorageObject object, Map options) throws StorageServiceException { String type = MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"); try { Insert req = storage.objects().insert(object.getBucket(), object); try { GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object value = option.get(options); if (value != null) { url.set(option.name(), value.toString()); } } HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, null); httpRequest.getHeaders().setContentType(type); httpRequest.getHeaders().setContentLength(0L); System.out.println("koko.url: " + httpRequest.getUrl()); System.out.println("koko.contentType:" + httpRequest.getHeaders().getContentType()); System.out.println("koko.contentLength:" + httpRequest.getHeaders().getContentLength()); httpRequest.setLoggingEnabled(true); httpRequest.setCurlLoggingEnabled(true); HttpResponse response = httpRequest.execute(); System.out.println("koko.response: " + response.toString()); System.out.println("koko.response.code: " + response.getStatusCode()); System.out.println("koko.response.message: " + response.getStatusMessage()); System.out.println("koko.response.content: " + response.getContentCharset()); } catch (Exception ex) { ex.printStackTrace(); } return null; } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.EmptyContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); // todo: Fix int casting downloader.setContentRange(position, (int) position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public void write(byte[] bytes, StorageObject to, String uploadId, long position, boolean last) throws StorageServiceException { try { GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new ByteArrayContent(null, bytes)); long limit = position + bytes.length; String range = position + "-" + (bytes.length - 1); httpRequest.getHeaders().setContentRange("bytes " + range + (last ? "/" + limit : "/*")); throw new IOException("bla"); } catch (IOException ex) { throw translate(ex); } } @Override public String open(StorageObject object, Map options) throws StorageServiceException { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object value = option.get(options); if (value != null) { url.set(option.name(), value.toString()); } } HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new EmptyContent()); httpRequest.getHeaders().set("X-Upload-Content-Type", MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index 5bf96a6ea824..e3e28c2dc56f 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, int position, int bytes) throws StorageServiceException; String open(StorageObject object, Map options) throws StorageServiceException; void write(byte[] bytes, StorageObject object, String uploadId, int position, boolean last) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException; String open(StorageObject object, Map options) throws StorageServiceException; void write(byte[] bytes, StorageObject object, String uploadId, long position, boolean last) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index bd2ef247d103..e84f9c2ff447 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -440,6 +440,7 @@ public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob, this.optionsMap = optionsMap; System.out.println("Koko. BlobWriterChannelImpl.init"); uploadId = options.storageRpc().open(blob.toPb(), optionsMap); + System.out.println("Koko. BlobWriterChannelImpl.init.uploadId: " + uploadId); } @Override From a96435119c1d687a4b6e30b51745787cdddee60f Mon Sep 17 00:00:00 2001 From: ozarov Date: Tue, 5 May 2015 20:35:15 -0700 Subject: [PATCH 46/48] work in progress --- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../gcloud/storage/StorageServiceImpl.java | 54 ++++++++++++++++--- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 71b94384bbb2..502979953221 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.EmptyContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); // todo: Fix int casting downloader.setContentRange(position, (int) position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public void write(byte[] bytes, StorageObject to, String uploadId, long position, boolean last) throws StorageServiceException { try { GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new ByteArrayContent(null, bytes)); long limit = position + bytes.length; String range = position + "-" + (bytes.length - 1); httpRequest.getHeaders().setContentRange("bytes " + range + (last ? "/" + limit : "/*")); throw new IOException("bla"); } catch (IOException ex) { throw translate(ex); } } @Override public String open(StorageObject object, Map options) throws StorageServiceException { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object value = option.get(options); if (value != null) { url.set(option.name(), value.toString()); } } HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new EmptyContent()); httpRequest.getHeaders().set("X-Upload-Content-Type", MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.EmptyContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937) downloader.setContentRange(position, (int) position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public void write(byte[] bytes, StorageObject to, String uploadId, long position, boolean last) throws StorageServiceException { try { GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new ByteArrayContent(null, bytes)); long limit = position + bytes.length; String range = position + "-" + (bytes.length - 1); httpRequest.getHeaders().setContentRange("bytes " + range + (last ? "/" + limit : "/*")); HttpResponse response = httpRequest.execute(); int code = response.getStatusCode(); if (!last && code != 308 || last && (code != 200 || code != 201)) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } } catch (IOException ex) { throw translate(ex); } } @Override public String open(StorageObject object, Map options) throws StorageServiceException { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object value = option.get(options); if (value != null) { url.set(option.name(), value.toString()); } } HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new EmptyContent()); httpRequest.getHeaders().set("X-Upload-Content-Type", MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index e84f9c2ff447..bfe1d78f8e03 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -427,25 +427,59 @@ public BlobReadChannel reader(Blob blob, BlobSourceOption... options) { private static class BlobWriterChannelImpl implements BlobWriteChannel { + private static final int CHUNK_SIZE = 256 * 1024; + private static final int MIN_BUFFER_SIZE = CHUNK_SIZE * 4; + private final StorageServiceOptions options; private final Blob blob; - private final Map optionsMap; private final String uploadId; private int position; + private byte[] buffer = new byte[MIN_BUFFER_SIZE]; + private int bufferLimit; + private boolean isOpen; + + private transient StorageRpc storageRpc; + private transient RetryParams retryParams; + private transient StorageObject storageObject; public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob, Map optionsMap) { this.options = options; this.blob = blob; - this.optionsMap = optionsMap; - System.out.println("Koko. BlobWriterChannelImpl.init"); - uploadId = options.storageRpc().open(blob.toPb(), optionsMap); - System.out.println("Koko. BlobWriterChannelImpl.init.uploadId: " + uploadId); + initTransients(); + uploadId = options.storageRpc().open(storageObject, optionsMap); + } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + initTransients(); + } + + private void initTransients() { + storageRpc = options.storageRpc(); + retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries()); + storageObject = blob.toPb(); + } + + private void validateOpen() throws IOException { + if (!isOpen) { + throw new IOException("stream is closed"); + } } @Override public int write(ByteBuffer byteBuffer) throws IOException { - int size = Math.min(byteBuffer.remaining(), 1024 * 1024); + validateOpen(); + if (byteBuffer.remaining() < (buffer.length - bufferLimit)) { + byteBuffer.get(buffer, bufferLimit, byteBuffer.remaining()); + // write buffer if full + } + int toWrite = buffer.length + byteBuffer.remaining() / + int size = Math.min(b, 1024 * 1024); byte[] bytes = new byte[size]; byteBuffer.get(bytes); // todo: Use retry helper on retriable failures @@ -457,12 +491,16 @@ public int write(ByteBuffer byteBuffer) throws IOException { @Override public boolean isOpen() { - return false; + return isOpen; } @Override public void close() throws IOException { - + if (isOpen) { + storageRpc.write(buffer, storageObject, uploadId, position, true); + position += buffer.length; + isOpen = false; + } } } From 6d4ac35f8e780fa4a7c2d1b443ff2fd9603885f7 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 6 May 2015 16:28:51 -0700 Subject: [PATCH 47/48] work in progress --- .../com/google/gcloud/ServiceOptions.java | 15 +++- .../datastore/DatastoreServiceOptions.java | 2 +- .../gcloud/examples/StorageExample.java | 23 +++--- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../com/google/gcloud/spi/StorageRpc.java | 2 +- .../gcloud/storage/StorageServiceImpl.java | 73 ++++++++++++++----- .../gcloud/storage/StorageServiceOptions.java | 2 +- 7 files changed, 83 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 0b03e7951e86..391631a695c0 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -36,6 +36,7 @@ import java.lang.reflect.Method; import java.net.HttpURLConnection; import java.net.URL; +import java.util.Locale; import java.util.Objects; import java.util.Set; import java.util.regex.Matcher; @@ -184,9 +185,13 @@ protected static String googleCloudProjectId() { } catch (IOException ignore) { // ignore } - String configDir = System.getenv("CLOUDSDK_CONFIG"); - if (configDir == null) { - configDir = new File(System.getProperty("user.home"), "/.config/gcloud/").getPath(); + File configDir; + if (System.getenv().containsKey("CLOUDSDK_CONFIG")) { + configDir = new File(System.getenv("CLOUDSDK_CONFIG")); + } else if (isWindows() && System.getenv().containsKey("APPDATA")) { + configDir = new File(System.getenv("APPDATA"), "gcloud"); + } else { + configDir = new File(System.getProperty("user.home"), ".config/gcloud"); } try (BufferedReader reader = new BufferedReader(new FileReader(new File(configDir, "properties")))) { @@ -216,6 +221,10 @@ protected static String googleCloudProjectId() { return null; } + private static boolean isWindows() { + return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).indexOf("windows") > -1; + } + protected static String getAppEngineProjectId() { // TODO(ozarov): An alternative to reflection would be to depend on AE api jar: // http://mvnrepository.com/artifact/com.google.appengine/appengine-api-1.0-sdk/1.2.0 diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index b1b1ee31638c..9378549db3d2 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -199,7 +199,7 @@ DatastoreRpc datastoreRpc() { return ServiceRpcProvider.datastore(this); } - public static DatastoreServiceOptions defaultIntance() { + public static DatastoreServiceOptions defaultInstance() { return builder().build(); } diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 0f9650a63bcc..aa38527ec8fc 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -32,6 +32,7 @@ import com.google.gcloud.storage.StorageServiceOptions; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; @@ -192,9 +193,16 @@ public String params() { private static class UploadAction extends StorageAction> { @Override public void run(StorageService storage, Tuple tuple) throws Exception { - if (Files.size(tuple.x()) > 1_000_000) { - // todo: upload via streaming API - throw new IllegalArgumentException("file is too big"); + if (Files.size(tuple.x()) > 1024) { + try (BlobWriteChannel writer = storage.writer(tuple.y())) { + byte[] buffer = new byte[1024]; + try (InputStream input = Files.newInputStream(tuple.x())) { + int limit; + while ((limit = input.read(buffer)) >= 0) { + writer.write(ByteBuffer.wrap(buffer, 0, limit)); + } + } + } } else { byte[] bytes = Files.readAllBytes(tuple.x()); System.out.println(storage.create(tuple.y(), bytes)); @@ -312,15 +320,6 @@ public static void printUsage() { @SuppressWarnings("unchecked") public static void main(String... args) throws Exception { if (args.length < 1) { - - System.out.println("KOKO.start -------------------------------"); - StorageService storage = - StorageServiceFactory.instance().get(StorageServiceOptions.defaultInstnace()); - BlobWriteChannel writer = storage - .writer(Blob.of("ozarov-javamrsample.appspot.com", "bla")); - writer.write(ByteBuffer.wrap("hello world".getBytes())); - writer.close(); - System.out.println("KOKO.end -------------------------------"); System.out.println("Missing required project id and action"); printUsage(); return; diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 502979953221..c165cf5504ed 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.EmptyContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937) downloader.setContentRange(position, (int) position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); // todo: optimize and eliminate unnecessary copy } catch (IOException ex) { throw translate(ex); } } @Override public void write(byte[] bytes, StorageObject to, String uploadId, long position, boolean last) throws StorageServiceException { try { GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new ByteArrayContent(null, bytes)); long limit = position + bytes.length; String range = position + "-" + (bytes.length - 1); httpRequest.getHeaders().setContentRange("bytes " + range + (last ? "/" + limit : "/*")); HttpResponse response = httpRequest.execute(); int code = response.getStatusCode(); if (!last && code != 308 || last && (code != 200 || code != 201)) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } } catch (IOException ex) { throw translate(ex); } } @Override public String open(StorageObject object, Map options) throws StorageServiceException { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object value = option.get(options); if (value != null) { url.set(option.name(), value.toString()); } } HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new EmptyContent()); httpRequest.getHeaders().set("X-Upload-Content-Type", MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.EmptyContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937) downloader.setContentRange(position, (int) position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public void write(byte[] bytes, int offset, int length, StorageObject to, String uploadId, long position, boolean last) throws StorageServiceException { try { GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new ByteArrayContent(null, bytes, offset, length)); long limit = position + length; StringBuilder range = new StringBuilder("bytes "); range.append(position).append('-').append(limit - 1).append('/'); if (last) { range.append('*'); } else { range.append(limit); } httpRequest.getHeaders().setContentRange(range.toString()); HttpResponse response = httpRequest.execute(); int code = response.getStatusCode(); if (!last && code != 308 || last && (code != 200 || code != 201)) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } } catch (IOException ex) { throw translate(ex); } } @Override public String open(StorageObject object, Map options) throws StorageServiceException { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object value = option.get(options); if (value != null) { url.set(option.name(), value.toString()); } } HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new EmptyContent()); httpRequest.getHeaders().set("X-Upload-Content-Type", MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index e3e28c2dc56f..be6bb96cb345 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException; String open(StorageObject object, Map options) throws StorageServiceException; void write(byte[] bytes, StorageObject object, String uploadId, long position, boolean last) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException; String open(StorageObject object, Map options) throws StorageServiceException; void write(byte[] bytes, int offset, int length, StorageObject object, String uploadId, long position, boolean last) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index bfe1d78f8e03..aea7b0c96016 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.gcloud.RetryHelper.runWithRetries; +import static java.util.concurrent.Executors.callable; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.Function; @@ -428,14 +429,14 @@ public BlobReadChannel reader(Blob blob, BlobSourceOption... options) { private static class BlobWriterChannelImpl implements BlobWriteChannel { private static final int CHUNK_SIZE = 256 * 1024; - private static final int MIN_BUFFER_SIZE = CHUNK_SIZE * 4; + private static final int COMPACT_THRESHOLD = (int) Math.round(CHUNK_SIZE * 0.8); private final StorageServiceOptions options; private final Blob blob; private final String uploadId; private int position; - private byte[] buffer = new byte[MIN_BUFFER_SIZE]; - private int bufferLimit; + private byte[] buffer = new byte[CHUNK_SIZE]; + private int limit; private boolean isOpen; private transient StorageRpc storageRpc; @@ -451,12 +452,45 @@ public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob, } private void writeObject(ObjectOutputStream out) throws IOException { + if (!isOpen) { + out.defaultWriteObject(); + return; + } + flush(); + byte[] temp = buffer; + if (limit < COMPACT_THRESHOLD) { + buffer = Arrays.copyOf(buffer, limit); + } out.defaultWriteObject(); + buffer = temp; + } + + private void flush() { + if (limit >= CHUNK_SIZE) { + final int length = limit - limit % CHUNK_SIZE; + runWithRetries(callable(new Runnable() { + @Override + public void run() { + System.out.println("Going to flush-> " + length + " bytes"); + storageRpc.write(buffer, 0, length, storageObject, uploadId, position, false); + } + })); + position += length; + limit -= length; + byte[] temp = new byte[CHUNK_SIZE]; + System.arraycopy(buffer, length, temp, 0, limit); + buffer = temp; + } } private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); - initTransients(); + if (isOpen) { + if (buffer.length < CHUNK_SIZE) { + buffer = Arrays.copyOf(buffer, CHUNK_SIZE); + } + initTransients(); + } } private void initTransients() { @@ -473,20 +507,18 @@ private void validateOpen() throws IOException { @Override public int write(ByteBuffer byteBuffer) throws IOException { + System.out.println("Going to write-> " + byteBuffer.remaining() + " bytes"); validateOpen(); - if (byteBuffer.remaining() < (buffer.length - bufferLimit)) { - byteBuffer.get(buffer, bufferLimit, byteBuffer.remaining()); - // write buffer if full + int toWrite = byteBuffer.remaining(); + if (buffer.length - limit >= toWrite) { + byteBuffer.get(buffer, limit, toWrite); + } else { + buffer = Arrays.copyOf(buffer, buffer.length + toWrite); + byteBuffer.get(buffer, limit, toWrite); } - int toWrite = buffer.length + byteBuffer.remaining() / - int size = Math.min(b, 1024 * 1024); - byte[] bytes = new byte[size]; - byteBuffer.get(bytes); - // todo: Use retry helper on retriable failures - System.out.println("Koko. BlobWriterChannelImpl.write"); - options.storageRpc().write(bytes, blob.toPb(), uploadId, position, false); - position += size; - return size; + limit += toWrite; + flush(); + return toWrite; } @Override @@ -497,9 +529,16 @@ public boolean isOpen() { @Override public void close() throws IOException { if (isOpen) { - storageRpc.write(buffer, storageObject, uploadId, position, true); + runWithRetries(callable(new Runnable() { + @Override + public void run() { + System.out.println("Going to close-> " + limit + " bytes"); + storageRpc.write(buffer, 0, limit, storageObject, uploadId, position, true); + } + })); position += buffer.length; isOpen = false; + buffer = null; } } } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java index 24dea9c91788..23ebcb0915fd 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java @@ -121,7 +121,7 @@ private static String defaultProject() { return projectId != null ? projectId : googleCloudProjectId(); } - public static StorageServiceOptions defaultInstnace() { + public static StorageServiceOptions defaultInstance() { return builder().build(); } From d2c63445456b15e5263a7fa574da5b84a677c024 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 6 May 2015 20:01:49 -0700 Subject: [PATCH 48/48] complete output writer --- .../com/google/gcloud/examples/StorageExample.java | 8 +++++++- .../com/google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../java/com/google/gcloud/spi/StorageRpc.java | 2 +- .../google/gcloud/storage/StorageServiceImpl.java | 14 ++++++-------- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index aa38527ec8fc..68bfbda59482 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -40,6 +40,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Random; /** * An example of using the Google Cloud Storage. @@ -194,12 +195,17 @@ private static class UploadAction extends StorageAction> { @Override public void run(StorageService storage, Tuple tuple) throws Exception { if (Files.size(tuple.x()) > 1024) { + Random rnd = new Random(); try (BlobWriteChannel writer = storage.writer(tuple.y())) { byte[] buffer = new byte[1024]; try (InputStream input = Files.newInputStream(tuple.x())) { int limit; while ((limit = input.read(buffer)) >= 0) { - writer.write(ByteBuffer.wrap(buffer, 0, limit)); + try { + writer.write(ByteBuffer.wrap(buffer, 0, limit)); + } catch (Exception ex) { + ex.printStackTrace(); + } } } } diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index c165cf5504ed..ab49d0b8c125 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.EmptyContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937) downloader.setContentRange(position, (int) position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public void write(byte[] bytes, int offset, int length, StorageObject to, String uploadId, long position, boolean last) throws StorageServiceException { try { GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new ByteArrayContent(null, bytes, offset, length)); long limit = position + length; StringBuilder range = new StringBuilder("bytes "); range.append(position).append('-').append(limit - 1).append('/'); if (last) { range.append('*'); } else { range.append(limit); } httpRequest.getHeaders().setContentRange(range.toString()); HttpResponse response = httpRequest.execute(); int code = response.getStatusCode(); if (!last && code != 308 || last && (code != 200 || code != 201)) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } } catch (IOException ex) { throw translate(ex); } } @Override public String open(StorageObject object, Map options) throws StorageServiceException { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object value = option.get(options); if (value != null) { url.set(option.name(), value.toString()); } } HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new EmptyContent()); httpRequest.getHeaders().set("X-Upload-Content-Type", MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.EmptyContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() // TOdo: consider options } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.project(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.project()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937) downloader.setContentRange(position, (int) position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest, long destOffset, int length, boolean last) throws StorageServiceException { try { GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new ByteArrayContent(null, toWrite, toWriteOffset, length)); long limit = destOffset + length; StringBuilder range = new StringBuilder("bytes "); range.append(destOffset).append('-').append(limit - 1).append('/'); if (last) { range.append(limit); } else { range.append('*'); } httpRequest.getHeaders().setContentRange(range.toString()); int code; String message; IOException exception = null; try { HttpResponse response = httpRequest.execute(); code = response.getStatusCode(); message = response.getStatusMessage(); } catch (HttpResponseException ex) { exception = ex; code = ex.getStatusCode(); message = ex.getStatusMessage(); } if (!last && code != 308 || last && !(code == 200 || code == 201)) { if (exception != null) { throw exception; } GoogleJsonError error = new GoogleJsonError(); error.setCode(code); error.setMessage(message); throw translate(error); } } catch (IOException ex) { throw translate(ex); } } @Override public String open(StorageObject object, Map options) throws StorageServiceException { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object value = option.get(options); if (value != null) { url.set(option.name(), value.toString()); } } HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new EmptyContent()); httpRequest.getHeaders().set("X-Upload-Content-Type", MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java index be6bb96cb345..ab1e9affbbce 100644 --- a/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException; String open(StorageObject object, Map options) throws StorageServiceException; void write(byte[] bytes, int offset, int length, StorageObject object, String uploadId, long position, boolean last) throws StorageServiceException; } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.gcloud.spi; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.StorageServiceException; import java.util.List; import java.util.Map; public interface StorageRpc { enum Option { PREDEFINED_ACL("predefinedAcl"), PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"), IF_METAGENERATION_MATCH("ifMetagenerationMatch"), IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"), IF_GENERATION_NOT_MATCH("ifGenerationMatch"), IF_GENERATION_MATCH("ifGenerationNotMatch"), IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"), IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"), IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"), IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"), PREFIX("prefix"), MAX_RESULTS("maxResults"), PAGE_TOKEN("pageToken"), DELIMITER("delimiter"), VERSIONS("versions"); private final String value; Option(String value) { this.value = value; } public String value() { return value; } @SuppressWarnings("unchecked") T get(Map options) { return (T) options.get(this); } String getString(Map options) { return get(options); } Long getLong(Map options) { return get(options); } Boolean getBoolean(Map options) { return get(options); } } class Tuple { private final X x; private final Y y; private Tuple(X x, Y y) { this.x = x; this.y = y; } public static Tuple of(X x, Y y) { return new Tuple<>(x, y); } public X x() { return x; } public Y y() { return y; } } class BatchRequest { public final List>> toDelete; public final List>> toUpdate; public final List>> toGet; public BatchRequest(Iterable>> toDelete, Iterable>> toUpdate, Iterable>> toGet) { this.toDelete = ImmutableList.copyOf(toDelete); this.toUpdate = ImmutableList.copyOf(toUpdate); this.toGet = ImmutableList.copyOf(toGet); } } class BatchResponse { public final Map> deletes; public final Map> updates; public final Map> gets; public BatchResponse(Map> deletes, Map> updates, Map> gets) { this.deletes = ImmutableMap.copyOf(deletes); this.updates = ImmutableMap.copyOf(updates); this.gets = ImmutableMap.copyOf(gets); } } Bucket create(Bucket bucket, Map options) throws StorageServiceException; StorageObject create(StorageObject object, byte[] content, Map options) throws StorageServiceException; Tuple> list(Map options) throws StorageServiceException; Tuple> list(String bucket, Map options) throws StorageServiceException; Bucket get(Bucket bucket, Map options) throws StorageServiceException; StorageObject get(StorageObject object, Map options) throws StorageServiceException; Bucket patch(Bucket bucket, Map options) throws StorageServiceException; StorageObject patch(StorageObject storageObject, Map options) throws StorageServiceException; boolean delete(Bucket bucket, Map options) throws StorageServiceException; boolean delete(StorageObject object, Map options) throws StorageServiceException; BatchResponse batch(BatchRequest request) throws StorageServiceException; StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException; StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException; byte[] load(StorageObject storageObject, Map options) throws StorageServiceException; byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException; String open(StorageObject object, Map options) throws StorageServiceException; void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest, long destOffset, int length, boolean last) throws StorageServiceException; } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index aea7b0c96016..fa911e431997 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -437,7 +437,7 @@ private static class BlobWriterChannelImpl implements BlobWriteChannel { private int position; private byte[] buffer = new byte[CHUNK_SIZE]; private int limit; - private boolean isOpen; + private boolean isOpen = true; private transient StorageRpc storageRpc; private transient RetryParams retryParams; @@ -471,8 +471,7 @@ private void flush() { runWithRetries(callable(new Runnable() { @Override public void run() { - System.out.println("Going to flush-> " + length + " bytes"); - storageRpc.write(buffer, 0, length, storageObject, uploadId, position, false); + storageRpc.write(uploadId, buffer, 0, storageObject, position, length, false); } })); position += length; @@ -507,13 +506,13 @@ private void validateOpen() throws IOException { @Override public int write(ByteBuffer byteBuffer) throws IOException { - System.out.println("Going to write-> " + byteBuffer.remaining() + " bytes"); validateOpen(); int toWrite = byteBuffer.remaining(); - if (buffer.length - limit >= toWrite) { + int spaceInBuffer = buffer.length - limit; + if (spaceInBuffer >= toWrite) { byteBuffer.get(buffer, limit, toWrite); } else { - buffer = Arrays.copyOf(buffer, buffer.length + toWrite); + buffer = Arrays.copyOf(buffer, buffer.length + toWrite - spaceInBuffer); byteBuffer.get(buffer, limit, toWrite); } limit += toWrite; @@ -532,8 +531,7 @@ public void close() throws IOException { runWithRetries(callable(new Runnable() { @Override public void run() { - System.out.println("Going to close-> " + limit + " bytes"); - storageRpc.write(buffer, 0, limit, storageObject, uploadId, position, true); + storageRpc.write(uploadId, buffer, 0, storageObject, position, limit, true); } })); position += buffer.length;