From 828f93fe0133982ac0cc797ca78e4a1c02b1619a Mon Sep 17 00:00:00 2001 From: Vishesh Garg Date: Mon, 11 Mar 2024 14:12:00 +0530 Subject: [PATCH 1/3] Fix GCS field read for retrieving version --- .../java/org/apache/druid/storage/google/GoogleStorage.java | 4 ++-- .../storage/google/GoogleTimestampVersionedDataFinder.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java index 15cca5e08d5e..15d75afef8b9 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java @@ -200,11 +200,11 @@ public long size(final String bucket, final String path) throws IOException public String version(final String bucket, final String path) throws IOException { - Blob blob = storage.get().get(bucket, path, Storage.BlobGetOption.fields(Storage.BlobField.GENERATION)); + Blob blob = storage.get().get(bucket, path, Storage.BlobGetOption.fields(Storage.BlobField.ETAG)); if (blob == null) { throw new IOE("Failed to fetch google cloud storage object from bucket [%s] and path [%s].", bucket, path); } - return blob.getGeneratedId(); + return blob.getEtag(); } /*** diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java index b93128cc2fa4..b0cae7d5d980 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java @@ -76,7 +76,7 @@ public URI getLatestVersion(URI descriptorBase, @Nullable Pattern pattern) return latest; } catch (IOException e) { - throw new RuntimeException(); + throw new RuntimeException(e); } } } From 99eca10e440eb843b76f6e1ce821de50ddceee51 Mon Sep 17 00:00:00 2001 From: Vishesh Garg Date: Mon, 11 Mar 2024 14:15:17 +0530 Subject: [PATCH 2/3] update test --- .../org/apache/druid/storage/google/GoogleStorageTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java index f227d593ce64..c28ed1cbe2b6 100644 --- a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java +++ b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java @@ -200,18 +200,18 @@ public void testSize() throws IOException @Test public void testVersion() throws IOException { - final String version = "7"; + final String etag = "abcd"; EasyMock.expect(mockStorage.get( EasyMock.eq(BUCKET), EasyMock.eq(PATH), EasyMock.anyObject(Storage.BlobGetOption.class) )).andReturn(blob); - EasyMock.expect(blob.getGeneratedId()).andReturn(version); + EasyMock.expect(blob.getEtag()).andReturn(etag); EasyMock.replay(mockStorage, blob); - assertEquals(version, googleStorage.version(BUCKET, PATH)); + assertEquals(etag, googleStorage.version(BUCKET, PATH)); } @Test From dcdfeda40bc3ee3ebdb1c727f60bd319fdfbbfb8 Mon Sep 17 00:00:00 2001 From: Vishesh Garg Date: Mon, 11 Mar 2024 14:43:11 +0530 Subject: [PATCH 3/3] Add javadoc for version --- .../org/apache/druid/storage/google/GoogleStorage.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java index 15d75afef8b9..c322af54021f 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java @@ -198,6 +198,15 @@ public long size(final String bucket, final String path) throws IOException return blob.getSize(); } + /** + * Return the etag for an object. This is a value that changes whenever the object's data or metadata changes and is + * typically but not always the MD5 hash of the object. Ref: + * ETags + * @param bucket + * @param path + * @return + * @throws IOException + */ public String version(final String bucket, final String path) throws IOException { Blob blob = storage.get().get(bucket, path, Storage.BlobGetOption.fields(Storage.BlobField.ETAG));