diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java index 91d290b17856..301f848339f4 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java @@ -138,7 +138,7 @@ public GoogleStorageObjectMetadata getMetadata( blob.getName(), blob.getSize(), blob.getUpdateTimeOffsetDateTime() - .toEpochSecond() + .toEpochSecond() * 1000 ); } @@ -234,7 +234,7 @@ public GoogleStorageObjectPage list( blob.getName(), blob.getSize(), blob.getUpdateTimeOffsetDateTime() - .toEpochSecond() + .toEpochSecond() * 1000 )) .collect(Collectors.toList()); diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java index 87feb774a5d8..ce4b4ca3064b 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java @@ -26,19 +26,24 @@ public class GoogleStorageObjectMetadata final String bucket; final String name; final Long size; - Long lastUpdateTime; + Long lastUpdateTimeMillis; - public GoogleStorageObjectMetadata(final String bucket, final String name, final Long size, final Long lastUpdateTime) + public GoogleStorageObjectMetadata( + final String bucket, + final String name, + final Long size, + final Long lastUpdateTimeMillis + ) { this.bucket = bucket; this.name = name; this.size = size; - this.lastUpdateTime = lastUpdateTime; + this.lastUpdateTimeMillis = lastUpdateTimeMillis; } - public void setLastUpdateTime(Long lastUpdateTime) + public void setLastUpdateTimeMillis(Long lastUpdateTimeMillis) { - this.lastUpdateTime = lastUpdateTime; + this.lastUpdateTimeMillis = lastUpdateTimeMillis; } @@ -57,9 +62,9 @@ public Long getSize() return size; } - public Long getLastUpdateTime() + public Long getLastUpdateTimeMillis() { - return lastUpdateTime; + return lastUpdateTimeMillis; } @Override @@ -74,13 +79,14 @@ public boolean equals(Object o) GoogleStorageObjectMetadata that = (GoogleStorageObjectMetadata) o; return Objects.equals(bucket, that.bucket) && Objects.equals(name, that.name) - && Objects.equals(size, that.size); + && Objects.equals(size, that.size) + && Objects.equals(lastUpdateTimeMillis, that.getLastUpdateTimeMillis()); } @Override public int hashCode() { - return Objects.hash(bucket, name, size); + return Objects.hash(bucket, name, size, lastUpdateTimeMillis); } @Override @@ -90,7 +96,7 @@ public String toString() "bucket='" + bucket + '\'' + ", name='" + name + '\'' + ", size=" + size + - ", lastUpdateTime=" + lastUpdateTime + + ", lastUpdateTimeMillis=" + lastUpdateTimeMillis + '}'; } } diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java index 4f7444f8ea9e..a11694f4a2f6 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java @@ -190,13 +190,13 @@ public void killAll() throws IOException } @Override - public void killOlderThan(long timestamp) throws IOException + public void killOlderThan(long timestampMs) throws IOException { LOG.info( "Deleting all task logs from gs location [bucket: '%s' prefix: '%s'] older than %s.", config.getBucket(), config.getPrefix(), - new Date(timestamp) + new Date(timestampMs) ); try { GoogleUtils.deleteObjectsInPath( @@ -204,7 +204,7 @@ public void killOlderThan(long timestamp) throws IOException inputDataConfig, config.getBucket(), config.getPrefix(), - (object) -> object.getLastUpdateTime() < timestamp + (object) -> object.getLastUpdateTimeMillis() < timestampMs ); } catch (Exception e) { diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java index b93128cc2fa4..01ae094912cf 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java @@ -67,7 +67,7 @@ public URI getLatestVersion(URI descriptorBase, @Nullable Pattern pattern) if (pattern != null && !pattern.matcher(keyString).matches()) { continue; } - final long latestModified = objectMetadata.getLastUpdateTime(); + final long latestModified = objectMetadata.getLastUpdateTimeMillis(); if (latestModified >= mostRecent) { mostRecent = latestModified; latest = objectLocation.toUri(GoogleStorageDruidModule.SCHEME_GS); diff --git a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java index d92339f53c79..715628022072 100644 --- a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java +++ b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java @@ -124,7 +124,7 @@ public void testBatchDeleteFailure() } @Test - public void testGetMetadata() throws IOException + public void testGetMetadataMatch() throws IOException { EasyMock.expect(mockStorage.get( EasyMock.eq(BUCKET), @@ -140,7 +140,10 @@ public void testGetMetadata() throws IOException EasyMock.replay(mockStorage, blob); GoogleStorageObjectMetadata objectMetadata = googleStorage.getMetadata(BUCKET, PATH); - assertEquals(objectMetadata, new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE, UPDATE_TIME.toEpochSecond())); + assertEquals( + objectMetadata, + new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE, UPDATE_TIME.toEpochSecond() * 1000) + ); } @@ -243,13 +246,13 @@ public void testList() throws IOException bucket1, path1, size1, - updateTime1.toEpochSecond() + updateTime1.toEpochSecond() * 1000 ); GoogleStorageObjectMetadata objectMetadata2 = new GoogleStorageObjectMetadata( bucket2, path2, size2, - updateTime2.toEpochSecond() + updateTime2.toEpochSecond() * 1000 ); GoogleStorageObjectPage objectPage = googleStorage.list(BUCKET, PATH, null, null); diff --git a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java index b9417b7f7f0e..2c65d3863793 100644 --- a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java +++ b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java @@ -37,13 +37,13 @@ public void getLatestVersion() // object for directory prefix/dir/0/ final GoogleStorageObjectMetadata storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "//", 0); - storageObject1.setLastUpdateTime(System.currentTimeMillis()); + storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis()); final GoogleStorageObjectMetadata storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v1", 1); - storageObject2.setLastUpdateTime(System.currentTimeMillis()); + storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis()); final GoogleStorageObjectMetadata storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v2", 1); - storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100); + storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100); final GoogleStorageObjectMetadata storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/other", 4); - storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100); + storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100); final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4)); final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage); @@ -61,13 +61,13 @@ public void getLatestVersionTrailingSlashKeyPrefix() // object for directory prefix/dir/0/ final GoogleStorageObjectMetadata storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/", 0); - storageObject1.setLastUpdateTime(System.currentTimeMillis()); + storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis()); final GoogleStorageObjectMetadata storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v1", 1); - storageObject2.setLastUpdateTime(System.currentTimeMillis()); + storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis()); final GoogleStorageObjectMetadata storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v2", 1); - storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100); + storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100); final GoogleStorageObjectMetadata storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "other", 4); - storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100); + storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100); final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4)); final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java index 71945ad92301..26c2a9362a4a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java @@ -45,6 +45,10 @@ public class TaskLogAutoCleanerConfig @JsonProperty private final long durationToRetain; + /** + * Config for Task logs auto-cleaner. + * All time-related parameters should be in milliseconds. + */ @JsonCreator public TaskLogAutoCleanerConfig( @JsonProperty("enabled") boolean enabled, diff --git a/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java b/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java index d2a3d0e92fd2..76dc7b8e92f3 100644 --- a/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java +++ b/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java @@ -30,5 +30,11 @@ public interface TaskLogKiller { void killAll() throws IOException; + + /** + * Removes logs older than the provided timestamp + * @param timestamp Timestamp in milliseconds + * @throws IOException + */ void killOlderThan(long timestamp) throws IOException; }