From a2ed9477f84942374be19523c69b1919a9b71f37 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Martin Date: Tue, 6 Nov 2018 16:51:32 -0800 Subject: [PATCH] Add a feature to list the buckets Also, update the README to reflect recent update to pseudoDirs. --- .../google-cloud-nio/README.md | 6 ++-- .../contrib/nio/CloudStorageFileSystem.java | 25 +++++++++++++++ .../nio/CloudStorageFileSystemProvider.java | 31 ++++++++++++++++++- .../storage/contrib/nio/it/ITGcsNio.java | 19 +++++++++++- 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/README.md b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/README.md index 79cdcb7910e8..7c88fd8f7dea 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/README.md +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/README.md @@ -138,7 +138,6 @@ Limitations This library is usable, but not yet complete. The following features are not yet implemented: - * Listing all the buckets * Resuming upload or download * Generations * File attributes @@ -153,8 +152,9 @@ subset via a familiar interface. **NOTE:** Cloud Storage uses a flat namespace and therefore doesn't support real directories. So this library supports what's known as "pseudo-directories". Any path that includes a trailing slash, will be considered a directory. It will -always be assumed to exist, without performing any I/O. This allows you to do -path manipulation in the same manner as you would with the normal UNIX file +always be assumed to exist, without performing any I/O. Paths without the trailing +slash will result in an I/O operation to check a file is present in that "directory". +This allows you to do path manipulation in the same manner as you would with the normal UNIX file system implementation. You can disable this feature with `CloudStorageConfiguration.usePseudoDirectories()`. diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java index 2b613d94c5b2..f3e9bd2d8038 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java @@ -19,6 +19,10 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; @@ -85,6 +89,27 @@ static CloudStorageConfiguration getDefaultCloudStorageConfiguration() { return userSpecifiedDefault; } + /** + * Lists the project's buckets. Pass "null" to use the default project. + * + *

Example of listing buckets, specifying the page size and a name prefix. + * 

 {@code
+   * String prefix = "bucket_";
+   * Page buckets = CloudStorageFileSystem.listBuckets("my-project", BucketListOption.prefix(prefix));
+   * Iterator bucketIterator = buckets.iterateAll();
+   * while (bucketIterator.hasNext()) {
+   *   Bucket bucket = bucketIterator.next();
+   *   // do something with the bucket
+   * }
+   * }
+ * + * @throws StorageException upon failure + */ + public static Page listBuckets(@Nullable String project, Storage.BucketListOption... options) { + CloudStorageFileSystemProvider provider = new CloudStorageFileSystemProvider(null, StorageOptions.newBuilder().setProjectId(project).build()); + return provider.listBuckets(options); + } + /** * Returns Google Cloud Storage {@link FileSystem} object for {@code bucket}. * diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java index a4e56c0c875d..701a0d854ab2 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java @@ -26,6 +26,7 @@ import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.CopyWriter; import com.google.cloud.storage.Storage; import com.google.cloud.storage.Storage.BlobGetOption; @@ -34,7 +35,6 @@ import com.google.cloud.storage.StorageOptions; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; -import com.google.common.base.Strings; import com.google.common.collect.AbstractIterator; import com.google.common.net.UrlEscapers; import com.google.common.primitives.Ints; @@ -915,6 +915,35 @@ public CloudStorageFileSystemProvider withNoUserProject() { return new CloudStorageFileSystemProvider("", this.storageOptions); } + /** + * Returns the project that is assigned to this provider. + */ + public String getProject() { + initStorage(); + return storage.getOptions().getProjectId(); + } + + /** + * Lists the project's buckets. But use the one in CloudStorageFileSystem. + * + *

Example of listing buckets, specifying the page size and a name prefix. + * 

 {@code
+   * String prefix = "bucket_";
+   * Page buckets = provider.listBuckets(BucketListOption.prefix(prefix));
+   * Iterator bucketIterator = buckets.iterateAll();
+   * while (bucketIterator.hasNext()) {
+   *   Bucket bucket = bucketIterator.next();
+   *   // do something with the bucket
+   * }
+   * }
+ * + * @throws StorageException upon failure + */ + Page listBuckets(Storage.BucketListOption... options) { + initStorage(); + return storage.list(options); + } + private IOException asIoException(StorageException oops) { // RPC API can only throw StorageException, but CloudStorageFileSystemProvider // can only throw IOException. Square peg, round hole. diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java index 08e75c1137fe..ffae384005b3 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java @@ -16,11 +16,13 @@ package com.google.cloud.storage.contrib.nio.it; +import static com.google.common.collect.ImmutableList.copyOf; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.client.http.HttpResponseException; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage.BlobTargetOption; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration; @@ -298,6 +300,21 @@ private void assertIsRequesterPaysException(String message, StorageException sex // End of tests related to the "requester pays" feature + @Test + public void testListBuckets() throws IOException { + boolean bucketFound = false; + boolean rpBucketFound = false; + for (Bucket b : CloudStorageFileSystem.listBuckets(project).iterateAll()) { + bucketFound |= BUCKET.equals(b.getName()); + rpBucketFound |= REQUESTER_PAYS_BUCKET.equals(b.getName()); + } + assertWithMessage("listBucket should have found the test bucket") + .that(bucketFound).isTrue(); + assertWithMessage("listBucket should have found the test requester-pays bucket") + .that(rpBucketFound).isTrue(); + } + + @Test public void testFileExists() throws IOException { CloudStorageFileSystem testBucket = getTestBucket(); @@ -736,7 +753,7 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx } public ImmutableList getPaths() { - return ImmutableList.copyOf(paths); + return copyOf(paths); } }