[Backport] Change last update timestamp granularity of GCS objects from seconds to milliseconds

extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java

@@ -138,7 +138,7 @@ public GoogleStorageObjectMetadata getMetadata(
         blob.getName(),
         blob.getSize(),
         blob.getUpdateTimeOffsetDateTime()
-            .toEpochSecond()
+            .toEpochSecond() * 1000
     );
   }
 
@@ -234,7 +234,7 @@ public GoogleStorageObjectPage list(
                     blob.getName(),
                     blob.getSize(),
                     blob.getUpdateTimeOffsetDateTime()
-                        .toEpochSecond()
+                        .toEpochSecond() * 1000
                 ))
                 .collect(Collectors.toList());
 

extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java

@@ -26,19 +26,24 @@ public class GoogleStorageObjectMetadata
   final String bucket;
   final String name;
   final Long size;
-  Long lastUpdateTime;
+  Long lastUpdateTimeMillis;
 
-  public GoogleStorageObjectMetadata(final String bucket, final String name, final Long size, final Long lastUpdateTime)
+  public GoogleStorageObjectMetadata(
+      final String bucket,
+      final String name,
+      final Long size,
+      final Long lastUpdateTimeMillis
+  )
   {
     this.bucket = bucket;
     this.name = name;
     this.size = size;
-    this.lastUpdateTime = lastUpdateTime;
+    this.lastUpdateTimeMillis = lastUpdateTimeMillis;
   }
 
-  public void setLastUpdateTime(Long lastUpdateTime)
+  public void setLastUpdateTimeMillis(Long lastUpdateTimeMillis)
   {
-    this.lastUpdateTime = lastUpdateTime;
+    this.lastUpdateTimeMillis = lastUpdateTimeMillis;
   }
 
 
@@ -57,9 +62,9 @@ public Long getSize()
     return size;
   }
 
-  public Long getLastUpdateTime()
+  public Long getLastUpdateTimeMillis()
   {
-    return lastUpdateTime;
+    return lastUpdateTimeMillis;
   }
 
   @Override
@@ -74,13 +79,14 @@ public boolean equals(Object o)
     GoogleStorageObjectMetadata that = (GoogleStorageObjectMetadata) o;
     return Objects.equals(bucket, that.bucket)
            && Objects.equals(name, that.name)
-           && Objects.equals(size, that.size);
+           && Objects.equals(size, that.size)
+           && Objects.equals(lastUpdateTimeMillis, that.getLastUpdateTimeMillis());
   }
 
   @Override
   public int hashCode()
   {
-    return Objects.hash(bucket, name, size);
+    return Objects.hash(bucket, name, size, lastUpdateTimeMillis);
   }
 
   @Override
@@ -90,7 +96,7 @@ public String toString()
            "bucket='" + bucket + '\'' +
            ", name='" + name + '\'' +
            ", size=" + size +
-           ", lastUpdateTime=" + lastUpdateTime +
+           ", lastUpdateTimeMillis=" + lastUpdateTimeMillis +
            '}';
   }
 }

extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java

@@ -190,21 +190,21 @@ public void killAll() throws IOException
   }
 
   @Override
-  public void killOlderThan(long timestamp) throws IOException
+  public void killOlderThan(long timestampMs) throws IOException
   {
     LOG.info(
         "Deleting all task logs from gs location [bucket: '%s' prefix: '%s'] older than %s.",
         config.getBucket(),
         config.getPrefix(),
-        new Date(timestamp)
+        new Date(timestampMs)
     );
     try {
       GoogleUtils.deleteObjectsInPath(
           storage,
           inputDataConfig,
           config.getBucket(),
           config.getPrefix(),
-          (object) -> object.getLastUpdateTime() < timestamp
+          (object) -> object.getLastUpdateTimeMillis() < timestampMs
       );
     }
     catch (Exception e) {

extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java

@@ -67,7 +67,7 @@ public URI getLatestVersion(URI descriptorBase, @Nullable Pattern pattern)
         if (pattern != null && !pattern.matcher(keyString).matches()) {
           continue;
         }
-        final long latestModified = objectMetadata.getLastUpdateTime();
+        final long latestModified = objectMetadata.getLastUpdateTimeMillis();
         if (latestModified >= mostRecent) {
           mostRecent = latestModified;
           latest = objectLocation.toUri(GoogleStorageDruidModule.SCHEME_GS);

extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java

@@ -124,7 +124,7 @@ public void testBatchDeleteFailure()
   }
 
   @Test
-  public void testGetMetadata() throws IOException
+  public void testGetMetadataMatch() throws IOException
   {
     EasyMock.expect(mockStorage.get(
         EasyMock.eq(BUCKET),
@@ -140,7 +140,10 @@ public void testGetMetadata() throws IOException
     EasyMock.replay(mockStorage, blob);
 
     GoogleStorageObjectMetadata objectMetadata = googleStorage.getMetadata(BUCKET, PATH);
-    assertEquals(objectMetadata, new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE, UPDATE_TIME.toEpochSecond()));
+    assertEquals(
+        objectMetadata,
+        new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE, UPDATE_TIME.toEpochSecond() * 1000)
+    );
 
   }
 
@@ -243,13 +246,13 @@ public void testList() throws IOException
         bucket1,
         path1,
         size1,
-        updateTime1.toEpochSecond()
+        updateTime1.toEpochSecond() * 1000
     );
     GoogleStorageObjectMetadata objectMetadata2 = new GoogleStorageObjectMetadata(
         bucket2,
         path2,
         size2,
-        updateTime2.toEpochSecond()
+        updateTime2.toEpochSecond() * 1000
     );
 
     GoogleStorageObjectPage objectPage = googleStorage.list(BUCKET, PATH, null, null);

extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java

@@ -37,13 +37,13 @@ public void getLatestVersion()
 
     // object for directory prefix/dir/0/
     final GoogleStorageObjectMetadata storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "//", 0);
-    storageObject1.setLastUpdateTime(System.currentTimeMillis());
+    storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis());
     final GoogleStorageObjectMetadata storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v1", 1);
-    storageObject2.setLastUpdateTime(System.currentTimeMillis());
+    storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis());
     final GoogleStorageObjectMetadata storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v2", 1);
-    storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100);
+    storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
     final GoogleStorageObjectMetadata storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/other", 4);
-    storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100);
+    storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
     final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4));
 
     final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage);
@@ -61,13 +61,13 @@ public void getLatestVersionTrailingSlashKeyPrefix()
 
     // object for directory prefix/dir/0/
     final GoogleStorageObjectMetadata storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/", 0);
-    storageObject1.setLastUpdateTime(System.currentTimeMillis());
+    storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis());
     final GoogleStorageObjectMetadata storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v1", 1);
-    storageObject2.setLastUpdateTime(System.currentTimeMillis());
+    storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis());
     final GoogleStorageObjectMetadata storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v2", 1);
-    storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100);
+    storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
     final GoogleStorageObjectMetadata storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "other", 4);
-    storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100);
+    storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
     final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4));
 
     final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage);

indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java

@@ -45,6 +45,10 @@ public class TaskLogAutoCleanerConfig
   @JsonProperty
   private final long durationToRetain;
 
+  /**
+   * Config for Task logs auto-cleaner.
+   * All time-related parameters should be in milliseconds.
+   */
   @JsonCreator
   public TaskLogAutoCleanerConfig(
       @JsonProperty("enabled") boolean enabled,

processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java

@@ -30,5 +30,11 @@
 public interface TaskLogKiller
 {
   void killAll() throws IOException;
+
+  /**
+   * Removes logs older than the provided timestamp
+   * @param timestamp Timestamp in milliseconds
+   * @throws IOException
+   */
   void killOlderThan(long timestamp) throws IOException;
 }