From c916cd6363779797cc04d2b75a44493ffac9d3a9 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Tue, 12 Mar 2024 10:04:18 +0530 Subject: [PATCH 01/14] Add update() in TestDerbyConnectorRule --- ...exerSQLMetadataStorageCoordinatorTest.java | 15 +++----- .../metadata/SQLMetadataConnectorTest.java | 27 +-------------- .../SqlSegmentsMetadataManagerTest.java | 34 +++++++------------ .../druid/metadata/TestDerbyConnector.java | 20 +++++++++++ .../duty/KillUnusedSegmentsTest.java | 9 +++++ 5 files changed, 46 insertions(+), 59 deletions(-) diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java index da7bf4226921..53e9aa289539 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java @@ -383,17 +383,10 @@ private void markAllSegmentsUnused(Set segments, DateTime usedStatu for (final DataSegment segment : segments) { Assert.assertEquals( 1, - (int) derbyConnector.getDBI().withHandle( - handle -> { - String request = StringUtils.format( - "UPDATE %s SET used = false, used_status_last_updated = :used_status_last_updated WHERE id = :id", - derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable() - ); - return handle.createStatement(request) - .bind("id", segment.getId().toString()) - .bind("used_status_last_updated", usedStatusLastUpdatedTime.toString() - ).execute(); - } + (int) derbyConnectorRule.updateSegmentsTable( + "UPDATE %s SET used = false, used_status_last_updated = ? WHERE id = ?", + usedStatusLastUpdatedTime.toString(), + segment.getId().toString() ) ); } diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java index 1c8f6493e22d..fa8d75e4a49e 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java @@ -30,13 +30,11 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.skife.jdbi.v2.Batch; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; import org.skife.jdbi.v2.exceptions.CallbackFailedException; import org.skife.jdbi.v2.exceptions.UnableToExecuteStatementException; import org.skife.jdbi.v2.exceptions.UnableToObtainConnectionException; -import org.skife.jdbi.v2.tweak.HandleCallback; import java.sql.SQLException; import java.sql.SQLRecoverableException; @@ -47,7 +45,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -175,29 +172,7 @@ public void testGeIndexOnNoTable() public void testAlterSegmentTableAddLastUsed() { connector.createSegmentTable(); - - // Drop column used_status_last_updated to bring us in line with pre-upgrade state - derbyConnectorRule.getConnector().retryWithHandle( - new HandleCallback() - { - @Override - public Void withHandle(Handle handle) - { - final Batch batch = handle.createBatch(); - batch.add( - StringUtils.format( - "ALTER TABLE %1$s DROP COLUMN USED_STATUS_LAST_UPDATED", - derbyConnectorRule.metadataTablesConfigSupplier() - .get() - .getSegmentsTable() - .toUpperCase(Locale.ENGLISH) - ) - ); - batch.execute(); - return null; - } - } - ); + derbyConnectorRule.updateSegmentsTable("ALTER TABLE %1$s DROP COLUMN USED_STATUS_LAST_UPDATED"); connector.alterSegmentTableAddUsedFlagLastUpdated(); connector.tableHasColumn( diff --git a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java index 85e5021f6c6c..7871f2f18b8a 100644 --- a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java @@ -891,15 +891,6 @@ public void testPopulateUsedFlagLastUpdated() throws IOException Assert.assertEquals(0, getCountOfRowsWithLastUsedNull()); } - private void updateSegmentPayload(DataSegment segment, byte[] payload) - { - executeUpdate( - "UPDATE %1$s SET PAYLOAD = ? WHERE ID = ?", - payload, - segment.getId().toString() - ); - } - private int getCountOfRowsWithLastUsedNull() { return derbyConnectorRule.getConnector().retryWithHandle( @@ -912,9 +903,18 @@ private int getCountOfRowsWithLastUsedNull() ); } + private void updateSegmentPayload(DataSegment segment, byte[] payload) + { + derbyConnectorRule.updateSegmentsTable( + "UPDATE %1$s SET PAYLOAD = ? WHERE ID = ?", + payload, + segment.getId().toString() + ); + } + private void updateUsedStatusLastUpdated(DataSegment segment, DateTime newValue) { - executeUpdate( + derbyConnectorRule.updateSegmentsTable( "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = ? WHERE ID = ?", newValue.toString(), segment.getId().toString() @@ -923,22 +923,12 @@ private void updateUsedStatusLastUpdated(DataSegment segment, DateTime newValue) private void updateUsedStatusLastUpdatedToNull(DataSegment segment) { - executeUpdate( + derbyConnectorRule.updateSegmentsTable( "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = NULL WHERE ID = ?", segment.getId().toString() ); } - private void executeUpdate(String sqlFormat, Object... args) - { - derbyConnectorRule.getConnector().retryWithHandle( - handle -> handle.update( - StringUtils.format(sqlFormat, getSegmentsTable()), - args - ) - ); - } - /** * Alters the column used_status_last_updated to be nullable. This is used to * test backward compatibility with versions of Druid without this column @@ -946,7 +936,7 @@ private void executeUpdate(String sqlFormat, Object... args) */ private void allowUsedFlagLastUpdatedToBeNullable() { - executeUpdate("ALTER TABLE %1$s ALTER COLUMN USED_STATUS_LAST_UPDATED NULL"); + derbyConnectorRule.updateSegmentsTable("ALTER TABLE %1$s ALTER COLUMN USED_STATUS_LAST_UPDATED NULL"); } private String getSegmentsTable() diff --git a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java index e5460ce402b4..f8983c8a0002 100644 --- a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java +++ b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java @@ -29,6 +29,7 @@ import org.skife.jdbi.v2.exceptions.UnableToObtainConnectionException; import java.sql.SQLException; +import java.util.Locale; import java.util.UUID; public class TestDerbyConnector extends DerbyConnector @@ -135,5 +136,24 @@ public Supplier metadataTablesConfigSupplier() { return dbTables; } + + + public Integer updateSegmentsTable(String sqlFormat, Object... args) + { + return this.getConnector().retryWithHandle( + handle -> handle.update( + StringUtils.format(sqlFormat, getSegmentsTable()), + args + ) + ); + } + + private String getSegmentsTable() + { + return this.metadataTablesConfigSupplier() + .get() + .getSegmentsTable() + .toUpperCase(Locale.ENGLISH); + } } } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java index 649900c841fb..6144dbc360a1 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java @@ -926,6 +926,15 @@ void deleteLastKillTaskId(final String dataSource) } private void updateUsedStatusLastUpdated(DataSegment segment, DateTime lastUpdatedTime) + { + derbyConnectorRule.updateSegmentsTable( + "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = ? WHERE ID = ?", + lastUpdatedTime.toString(), + segment.getId().toString() + ); + } + + private void updateUsedStatusLastUpdatedBoilerPlate(DataSegment segment, DateTime lastUpdatedTime) { derbyConnectorRule.getConnector().retryWithHandle( handle -> handle.update( From 82dd841f7b523de96dac65626defb0f10275df7d Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Wed, 13 Mar 2024 12:02:45 +0530 Subject: [PATCH 02/14] use common function. --- .../IndexerSQLMetadataStorageCoordinator.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java index 3d8939c3e52e..44d09c7da4f9 100644 --- a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java +++ b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java @@ -419,16 +419,7 @@ public SegmentPublishResult commitSegmentsAndMetadata( @Nullable final DataSourceMetadata endMetadata ) throws IOException { - if (segments.isEmpty()) { - throw new IllegalArgumentException("segment set must not be empty"); - } - - final String dataSource = segments.iterator().next().getDataSource(); - for (DataSegment segment : segments) { - if (!dataSource.equals(segment.getDataSource())) { - throw new IllegalArgumentException("segments must all be from the same dataSource"); - } - } + verifySegmentsToCommit(segments); if ((startMetadata == null && endMetadata != null) || (startMetadata != null && endMetadata == null)) { throw new IllegalArgumentException("start/end metadata pair must be either null or non-null"); From e5e558c0d3dfd7cab8cfc7c9271ee24873b1224f Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 14 Mar 2024 12:34:35 +0530 Subject: [PATCH 03/14] fixup build. --- .../druid/metadata/IndexerSQLMetadataStorageCoordinator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java index 44d09c7da4f9..f89ed18d3a35 100644 --- a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java +++ b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java @@ -425,6 +425,8 @@ public SegmentPublishResult commitSegmentsAndMetadata( throw new IllegalArgumentException("start/end metadata pair must be either null or non-null"); } + final String dataSource = segments.iterator().next().getDataSource(); + // Find which segments are used (i.e. not overshadowed). final Set usedSegments = new HashSet<>(); List> segmentHolders = From a9d6b73e79a093231bc7a12216a36b80ec0d9b99 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 14 Mar 2024 12:42:42 +0530 Subject: [PATCH 04/14] fixup indentations. --- .../task/KillUnusedSegmentsTaskTest.java | 358 +++++++++--------- 1 file changed, 170 insertions(+), 188 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java index d8e7a006605e..ec1001e0006b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java @@ -86,17 +86,16 @@ public void testKill() throws Exception ) ); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2019-03-01/2019-04-01"), + null, + false, + null, + null, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -197,17 +196,16 @@ public void testKillWithMarkUnused() throws Exception ) ); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - true, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2019-03-01/2019-04-01"), + null, + true, + null, + null, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -234,17 +232,16 @@ public void testKillWithMarkUnused() throws Exception @Test public void testGetInputSourceResources() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - true, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2019-03-01/2019-04-01"), + null, + true, + null, + null, + null + ); Assert.assertTrue(task.getInputSourceResources().isEmpty()); } @@ -270,17 +267,16 @@ public void testKillBatchSizeOneAndLimit4() throws Exception ) ); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 1, - 4, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 1, + 4, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -348,17 +344,16 @@ public void testKillMultipleUnusedSegmentsWithNullMaxUsedStatusLastUpdatedTime() final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -440,17 +435,16 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task1 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime1 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -465,17 +459,16 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT Assert.assertEquals(ImmutableList.of(segment3), unusedSegments); Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); - final KillUnusedSegmentsTask task2 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime2 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -545,7 +538,6 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final DateTime maxUsedStatusLastUpdatedTime2 = DateTimes.nowUtc(); - final List segmentIntervals = segments.stream() .map(DataSegment::getInterval) .collect(Collectors.toList()); @@ -553,17 +545,16 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task1 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime1 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -578,17 +569,16 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT Assert.assertEquals(ImmutableList.of(segment2, segment3), unusedSegments); Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); - final KillUnusedSegmentsTask task2 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime2 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -617,17 +607,16 @@ public void testKillBatchSizeThree() throws Exception Assert.assertEquals(segments, announced); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - true, - 3, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + true, + 3, + null, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -648,102 +637,96 @@ public void testKillBatchSizeThree() throws Exception @Test public void testComputeNextBatchSizeDefault() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + null, + null, + null + ); Assert.assertEquals(100, task.computeNextBatchSize(50)); } @Test public void testComputeNextBatchSizeWithBatchSizeLargerThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 10, - 5, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 10, + 5, + null + ); Assert.assertEquals(5, task.computeNextBatchSize(0)); } @Test public void testComputeNextBatchSizeWithBatchSizeSmallerThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 5, + 10, + null + ); Assert.assertEquals(5, task.computeNextBatchSize(0)); } @Test public void testComputeNextBatchSizeWithRemainingLessThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 5, + 10, + null + ); Assert.assertEquals(3, task.computeNextBatchSize(7)); } @Test public void testGetNumTotalBatchesDefault() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + null, + null, + null + ); Assert.assertNull(task.getNumTotalBatches()); } @Test public void testGetNumTotalBatchesWithBatchSizeLargerThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 10, - 5, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 10, + 5, + null + ); Assert.assertEquals(1, (int) task.getNumTotalBatches()); } @@ -819,17 +802,16 @@ public void testInvalidMarkAsUnusedWithLimit() @Test public void testGetNumTotalBatchesWithBatchSizeSmallerThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 5, + 10, + null + ); Assert.assertEquals(2, (int) task.getNumTotalBatches()); } From f23b17501f78cd56fb2b64467539b05c38cf62db Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 14 Mar 2024 12:52:57 +0530 Subject: [PATCH 05/14] Revert "fixup indentations." This reverts commit a9d6b73e79a093231bc7a12216a36b80ec0d9b99. --- .../task/KillUnusedSegmentsTaskTest.java | 358 +++++++++--------- 1 file changed, 188 insertions(+), 170 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java index ec1001e0006b..d8e7a006605e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java @@ -86,16 +86,17 @@ public void testKill() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2019-03-01/2019-04-01"), + null, + false, + null, + null, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -196,16 +197,17 @@ public void testKillWithMarkUnused() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - true, - null, - null, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2019-03-01/2019-04-01"), + null, + true, + null, + null, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -232,16 +234,17 @@ public void testKillWithMarkUnused() throws Exception @Test public void testGetInputSourceResources() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - true, - null, - null, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2019-03-01/2019-04-01"), + null, + true, + null, + null, + null + ); Assert.assertTrue(task.getInputSourceResources().isEmpty()); } @@ -267,16 +270,17 @@ public void testKillBatchSizeOneAndLimit4() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 1, - 4, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 1, + 4, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -344,16 +348,17 @@ public void testKillMultipleUnusedSegmentsWithNullMaxUsedStatusLastUpdatedTime() final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -435,16 +440,17 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime1 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -459,16 +465,17 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT Assert.assertEquals(ImmutableList.of(segment3), unusedSegments); Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); - final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime2 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -538,6 +545,7 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final DateTime maxUsedStatusLastUpdatedTime2 = DateTimes.nowUtc(); + final List segmentIntervals = segments.stream() .map(DataSegment::getInterval) .collect(Collectors.toList()); @@ -545,16 +553,17 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime1 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -569,16 +578,17 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT Assert.assertEquals(ImmutableList.of(segment2, segment3), unusedSegments); Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); - final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime2 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -607,16 +617,17 @@ public void testKillBatchSizeThree() throws Exception Assert.assertEquals(segments, announced); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - true, - 3, - null, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + true, + 3, + null, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -637,96 +648,102 @@ public void testKillBatchSizeThree() throws Exception @Test public void testComputeNextBatchSizeDefault() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + null, + null, + null + ); Assert.assertEquals(100, task.computeNextBatchSize(50)); } @Test public void testComputeNextBatchSizeWithBatchSizeLargerThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 10, - 5, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 10, + 5, + null + ); Assert.assertEquals(5, task.computeNextBatchSize(0)); } @Test public void testComputeNextBatchSizeWithBatchSizeSmallerThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 5, + 10, + null + ); Assert.assertEquals(5, task.computeNextBatchSize(0)); } @Test public void testComputeNextBatchSizeWithRemainingLessThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 5, + 10, + null + ); Assert.assertEquals(3, task.computeNextBatchSize(7)); } @Test public void testGetNumTotalBatchesDefault() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + null, + null, + null + ); Assert.assertNull(task.getNumTotalBatches()); } @Test public void testGetNumTotalBatchesWithBatchSizeLargerThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 10, - 5, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 10, + 5, + null + ); Assert.assertEquals(1, (int) task.getNumTotalBatches()); } @@ -802,16 +819,17 @@ public void testInvalidMarkAsUnusedWithLimit() @Test public void testGetNumTotalBatchesWithBatchSizeSmallerThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = + new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + false, + 5, + 10, + null + ); Assert.assertEquals(2, (int) task.getNumTotalBatches()); } From 1ac2d969805b67e30d8df69086d733e66d05d895 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 14 Mar 2024 12:59:21 +0530 Subject: [PATCH 06/14] fixup indentataions. --- .../task/KillUnusedSegmentsTaskTest.java | 438 +++++++++--------- 1 file changed, 209 insertions(+), 229 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java index 2de9a0f10f2a..0ecfa0f824d8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java @@ -86,18 +86,17 @@ public void testKill() throws Exception ) ); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2019-03-01/2019-04-01"), + null, + null, + false, + null, + null, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -142,18 +141,17 @@ public void testKillWithMarkUnused() throws Exception ) ); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - null, - true, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2019-03-01/2019-04-01"), + null, + null, + true, + null, + null, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -394,18 +392,17 @@ public void testKillUnusedSegmentsWithUsedLoadSpec() throws Exception @Test public void testGetInputSourceResources() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - null, - true, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2019-03-01/2019-04-01"), + null, + null, + true, + null, + null, + null + ); Assert.assertTrue(task.getInputSourceResources().isEmpty()); } @@ -431,18 +428,17 @@ public void testKillBatchSizeOneAndLimit4() throws Exception ) ); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 1, - 4, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + null, + false, + 1, + 4, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -510,18 +506,17 @@ public void testKillMultipleUnusedSegmentsWithNullMaxUsedStatusLastUpdatedTime() final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + null, + false, + 1, + 10, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -603,18 +598,17 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task1 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime1 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -629,18 +623,17 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT Assert.assertEquals(ImmutableList.of(segment3), unusedSegments); Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); - final KillUnusedSegmentsTask task2 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime2 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -717,19 +710,17 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - - final KillUnusedSegmentsTask task1 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime1 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -744,18 +735,17 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT Assert.assertEquals(ImmutableList.of(segment2, segment3), unusedSegments); Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); - final KillUnusedSegmentsTask task2 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + null, + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime2 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -810,18 +800,17 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task1 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - ImmutableList.of(version.toString()), - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + ImmutableList.of(version.toString()), + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime1 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); Assert.assertEquals( @@ -839,18 +828,17 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime Assert.assertEquals(ImmutableSet.of(segment3, segment4, segment5), new HashSet<>(observedUnusedSegments)); - final KillUnusedSegmentsTask task2 = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - ImmutableList.of(version.toString()), - null, - false, - 1, - 10, - maxUsedStatusLastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + umbrellaInterval, + ImmutableList.of(version.toString()), + null, + false, + 1, + 10, + maxUsedStatusLastUpdatedTime2 + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); Assert.assertEquals( @@ -883,18 +871,17 @@ public void testKillBatchSizeThree() throws Exception Assert.assertEquals(segments, announced); - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - true, - 3, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + null, + true, + 3, + null, + null + ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -915,108 +902,102 @@ public void testKillBatchSizeThree() throws Exception @Test public void testComputeNextBatchSizeDefault() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + null, + false, + null, + null, + null + ); Assert.assertEquals(100, task.computeNextBatchSize(50)); } @Test public void testComputeNextBatchSizeWithBatchSizeLargerThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 10, - 5, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + null, + false, + 10, + 5, + null + ); Assert.assertEquals(5, task.computeNextBatchSize(0)); } @Test public void testComputeNextBatchSizeWithBatchSizeSmallerThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + null, + false, + 5, + 10, + null + ); Assert.assertEquals(5, task.computeNextBatchSize(0)); } @Test public void testComputeNextBatchSizeWithRemainingLessThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + null, + false, + 5, + 10, + null + ); Assert.assertEquals(3, task.computeNextBatchSize(7)); } @Test public void testGetNumTotalBatchesDefault() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + null, + false, + null, + null, + null + ); Assert.assertNull(task.getNumTotalBatches()); } @Test public void testGetNumTotalBatchesWithBatchSizeLargerThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 10, - 5, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + null, + false, + 10, + 5, + null + ); Assert.assertEquals(1, (int) task.getNumTotalBatches()); } @@ -1119,18 +1100,17 @@ public void testInvalidVersionWithMarkAsUnused() @Test public void testGetNumTotalBatchesWithBatchSizeSmallerThanLimit() { - final KillUnusedSegmentsTask task = - new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( + null, + DATA_SOURCE, + Intervals.of("2018-01-01/2020-01-01"), + null, + null, + false, + 5, + 10, + null + ); Assert.assertEquals(2, (int) task.getNumTotalBatches()); } From b63926d1aa435ce0159eb5940f450c4669f3ab19 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 14 Mar 2024 13:20:44 +0530 Subject: [PATCH 07/14] Remove Thread.sleep() by directly calling updateUsedStatusLastUpdated. --- .../task/KillUnusedSegmentsTaskTest.java | 80 ++++++++++--------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java index 0ecfa0f824d8..d242849da2fc 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java @@ -574,13 +574,11 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ) ); - // Capture the last updated time cutoff - final DateTime maxUsedStatusLastUpdatedTime1 = DateTimes.nowUtc(); + final DateTime lastUpdatedTime1 = DateTimes.nowUtc(); + updateUsedStatusLastUpdated(segment1, lastUpdatedTime1); + updateUsedStatusLastUpdated(segment4, lastUpdatedTime1); - // Delay for 1s, mark the segments as unused and then capture the last updated time cutoff again - Thread.sleep(1000); - - // now mark the third segment as unused + // Now mark the third segment as unused Assert.assertEquals( 1, getSegmentsMetadataManager().markAsUnusedSegmentsInInterval( @@ -589,8 +587,8 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ) ); - final DateTime maxUsedStatusLastUpdatedTime2 = DateTimes.nowUtc(); - + final DateTime lastUpdatedTime2 = DateTimes.nowUtc(); + updateUsedStatusLastUpdated(segment3, lastUpdatedTime2); final List segmentIntervals = segments.stream() .map(DataSegment::getInterval) @@ -607,7 +605,7 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT false, 1, 10, - maxUsedStatusLastUpdatedTime1 + lastUpdatedTime1 ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -632,7 +630,7 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT false, 1, 10, - maxUsedStatusLastUpdatedTime2 + lastUpdatedTime2 ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -686,10 +684,9 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ) ); - final DateTime maxUsedStatusLastUpdatedTime1 = DateTimes.nowUtc(); - - // Delay for 1s, mark the segments as unused and then capture the last updated time cutoff again - Thread.sleep(1000); + final DateTime lastUpdatedTime1 = DateTimes.nowUtc(); + updateUsedStatusLastUpdated(segment1, lastUpdatedTime1); + updateUsedStatusLastUpdated(segment4, lastUpdatedTime1); Assert.assertEquals( 2, @@ -701,8 +698,9 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ) ); - final DateTime maxUsedStatusLastUpdatedTime2 = DateTimes.nowUtc(); - + final DateTime lastUpdatedTime2 = DateTimes.nowUtc(); + updateUsedStatusLastUpdated(segment2, lastUpdatedTime2); + updateUsedStatusLastUpdated(segment3, lastUpdatedTime2); final List segmentIntervals = segments.stream() .map(DataSegment::getInterval) @@ -719,20 +717,20 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT false, 1, 10, - maxUsedStatusLastUpdatedTime1 + lastUpdatedTime1 ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); - final List unusedSegments = + final List observedUnusedSegments1 = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( DATA_SOURCE, umbrellaInterval, null, null - ); + ); - Assert.assertEquals(ImmutableList.of(segment2, segment3), unusedSegments); + Assert.assertEquals(ImmutableList.of(segment2, segment3), observedUnusedSegments1); Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( @@ -744,19 +742,19 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT false, 1, 10, - maxUsedStatusLastUpdatedTime2 + lastUpdatedTime2 ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); - final List unusedSegments2 = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( - DATA_SOURCE, - umbrellaInterval, - null, - null - ); + final List observedUnusedSegments2 = + getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( + DATA_SOURCE, umbrellaInterval, + null, + null + ); - Assert.assertEquals(ImmutableList.of(), unusedSegments2); + Assert.assertEquals(ImmutableList.of(), observedUnusedSegments2); Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); } @@ -780,11 +778,10 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime ) ); - // Capture the last updated time cutoff - final DateTime maxUsedStatusLastUpdatedTime1 = DateTimes.nowUtc(); - - // Delay for 1s, mark the segments as unused and then capture the last updated time cutoff again - Thread.sleep(1000); + final DateTime lastUpdatedTime1 = DateTimes.nowUtc(); + updateUsedStatusLastUpdated(segment1, lastUpdatedTime1); + updateUsedStatusLastUpdated(segment2, lastUpdatedTime1); + updateUsedStatusLastUpdated(segment4, lastUpdatedTime1); Assert.assertEquals( 2, @@ -792,7 +789,9 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime ImmutableSet.of(segment3.getId(), segment5.getId()) ) ); - final DateTime maxUsedStatusLastUpdatedTime2 = DateTimes.nowUtc(); + + final DateTime lastUpdatedTime2 = DateTimes.nowUtc(); + updateUsedStatusLastUpdated(segment4, lastUpdatedTime2); final List segmentIntervals = segments.stream() .map(DataSegment::getInterval) @@ -809,7 +808,7 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime false, 1, 10, - maxUsedStatusLastUpdatedTime1 + lastUpdatedTime1 ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -837,7 +836,7 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime false, 1, 10, - maxUsedStatusLastUpdatedTime2 + lastUpdatedTime2 ); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -1177,4 +1176,13 @@ private static DataSegment newSegment(Interval interval, String version, Map Date: Thu, 14 Mar 2024 13:30:11 +0530 Subject: [PATCH 08/14] another indentation slip. --- .../indexing/common/task/KillUnusedSegmentsTaskTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java index d242849da2fc..bd3db0fa44e4 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java @@ -100,7 +100,8 @@ public void testKill() throws Exception Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); - final List unusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( + final List observedUnusedSegments = + getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( DATA_SOURCE, Intervals.of("2019/2020"), null, @@ -108,7 +109,7 @@ public void testKill() throws Exception null ); - Assert.assertEquals(ImmutableList.of(newSegment(Intervals.of("2019-02-01/2019-03-01"), version)), unusedSegments); + Assert.assertEquals(ImmutableList.of(newSegment(Intervals.of("2019-02-01/2019-03-01"), version)), observedUnusedSegments); Assertions.assertThat(getMetadataStorageCoordinator().retrieveUsedSegmentsForInterval( DATA_SOURCE, Intervals.of("2019/2020"), @@ -749,7 +750,8 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final List observedUnusedSegments2 = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( - DATA_SOURCE, umbrellaInterval, + DATA_SOURCE, + umbrellaInterval, null, null ); From 886bb9f9c398f2359d82b4ed4fa46ab296b04f79 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 14 Mar 2024 13:54:29 +0530 Subject: [PATCH 09/14] Move common segment initialization to setup(). --- .../task/KillUnusedSegmentsTaskTest.java | 199 +++++++++--------- .../druid/metadata/TestDerbyConnector.java | 1 - .../duty/KillUnusedSegmentsTest.java | 21 -- 3 files changed, 97 insertions(+), 124 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java index bd3db0fa44e4..8a6adcd15426 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java @@ -55,34 +55,38 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase private TestTaskRunner taskRunner; + private DataSegment segment1; + private DataSegment segment2; + private DataSegment segment3; + private DataSegment segment4; + @Before public void setup() { taskRunner = new TestTaskRunner(); + + final String version = DateTimes.nowUtc().toString(); + segment1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), version); + segment2 = newSegment(Intervals.of("2019-02-01/2019-03-01"), version); + segment3 = newSegment(Intervals.of("2019-03-01/2019-04-01"), version); + segment4 = newSegment(Intervals.of("2019-04-01/2019-05-01"), version); } @Test public void testKill() throws Exception { - final String version = DateTimes.nowUtc().toString(); - final Set segments = ImmutableSet.of( - newSegment(Intervals.of("2019-01-01/2019-02-01"), version), - newSegment(Intervals.of("2019-02-01/2019-03-01"), version), - newSegment(Intervals.of("2019-03-01/2019-04-01"), version), - newSegment(Intervals.of("2019-04-01/2019-05-01"), version) - ); + final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4); final Set announced = getMetadataStorageCoordinator().commitSegments(segments); - Assert.assertEquals(segments, announced); Assert.assertTrue( getSegmentsMetadataManager().markSegmentAsUnused( - newSegment(Intervals.of("2019-02-01/2019-03-01"), version).getId() + segment2.getId() ) ); Assert.assertTrue( getSegmentsMetadataManager().markSegmentAsUnused( - newSegment(Intervals.of("2019-03-01/2019-04-01"), version).getId() + segment3.getId() ) ); @@ -109,36 +113,31 @@ public void testKill() throws Exception null ); - Assert.assertEquals(ImmutableList.of(newSegment(Intervals.of("2019-02-01/2019-03-01"), version)), observedUnusedSegments); - Assertions.assertThat(getMetadataStorageCoordinator().retrieveUsedSegmentsForInterval( - DATA_SOURCE, - Intervals.of("2019/2020"), - Segments.ONLY_VISIBLE) - ).containsExactlyInAnyOrder( - newSegment(Intervals.of("2019-01-01/2019-02-01"), version), - newSegment(Intervals.of("2019-04-01/2019-05-01"), version) - ); + Assert.assertEquals(ImmutableList.of(segment2), observedUnusedSegments); + Assertions.assertThat( + getMetadataStorageCoordinator().retrieveUsedSegmentsForInterval( + DATA_SOURCE, + Intervals.of("2019/2020"), + Segments.ONLY_VISIBLE + ) + ).containsExactlyInAnyOrder(segment1, segment4); - Assert.assertEquals(new KillTaskReport.Stats(1, 2, 0), getReportedStats()); + Assert.assertEquals( + new KillTaskReport.Stats(1, 2, 0), + getReportedStats() + ); } @Test public void testKillWithMarkUnused() throws Exception { - final String version = DateTimes.nowUtc().toString(); - final Set segments = ImmutableSet.of( - newSegment(Intervals.of("2019-01-01/2019-02-01"), version), - newSegment(Intervals.of("2019-02-01/2019-03-01"), version), - newSegment(Intervals.of("2019-03-01/2019-04-01"), version), - newSegment(Intervals.of("2019-04-01/2019-05-01"), version) - ); + final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4); final Set announced = getMetadataStorageCoordinator().commitSegments(segments); - Assert.assertEquals(segments, announced); Assert.assertTrue( getSegmentsMetadataManager().markSegmentAsUnused( - newSegment(Intervals.of("2019-02-01/2019-03-01"), version).getId() + segment2.getId() ) ); @@ -165,16 +164,19 @@ public void testKillWithMarkUnused() throws Exception null ); - Assert.assertEquals(ImmutableList.of(newSegment(Intervals.of("2019-02-01/2019-03-01"), version)), unusedSegments); + Assert.assertEquals(ImmutableList.of(segment2), unusedSegments); Assertions.assertThat( - getMetadataStorageCoordinator() - .retrieveUsedSegmentsForInterval(DATA_SOURCE, Intervals.of("2019/2020"), Segments.ONLY_VISIBLE) - ).containsExactlyInAnyOrder( - newSegment(Intervals.of("2019-01-01/2019-02-01"), version), - newSegment(Intervals.of("2019-04-01/2019-05-01"), version) - ); + getMetadataStorageCoordinator().retrieveUsedSegmentsForInterval( + DATA_SOURCE, + Intervals.of("2019/2020"), + Segments.ONLY_VISIBLE + ) + ).containsExactlyInAnyOrder(segment1, segment4); - Assert.assertEquals(new KillTaskReport.Stats(1, 2, 1), getReportedStats()); + Assert.assertEquals( + new KillTaskReport.Stats(1, 2, 1), + getReportedStats() + ); } @Test @@ -185,13 +187,13 @@ public void testKillSegmentsWithVersions() throws Exception final String v2 = now.minusHours(2).toString(); final String v3 = now.minusHours(3).toString(); - final DataSegment segment1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v1); - final DataSegment segment2 = newSegment(Intervals.of("2019-02-01/2019-03-01"), v1); - final DataSegment segment3 = newSegment(Intervals.of("2019-03-01/2019-04-01"), v1); - final DataSegment segment4 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v2); - final DataSegment segment5 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v3); + final DataSegment segment1V1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v1); + final DataSegment segment2V1 = newSegment(Intervals.of("2019-02-01/2019-03-01"), v1); + final DataSegment segment3V1 = newSegment(Intervals.of("2019-03-01/2019-04-01"), v1); + final DataSegment segment4V2 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v2); + final DataSegment segment5V3 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v3); - final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4, segment5); + final Set segments = ImmutableSet.of(segment1V1, segment2V1, segment3V1, segment4V2, segment5V3); Assert.assertEquals(segments, getMetadataStorageCoordinator().commitSegments(segments)); Assert.assertEquals( @@ -226,7 +228,7 @@ public void testKillSegmentsWithVersions() throws Exception null ); - Assert.assertEquals(ImmutableSet.of(segment5), new HashSet<>(observedUnusedSegments)); + Assert.assertEquals(ImmutableSet.of(segment5V3), new HashSet<>(observedUnusedSegments)); } @Test @@ -237,13 +239,13 @@ public void testKillSegmentsWithVersionsAndLimit() throws Exception final String v2 = now.minusHours(2).toString(); final String v3 = now.minusHours(3).toString(); - final DataSegment segment1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v1); - final DataSegment segment2 = newSegment(Intervals.of("2019-02-01/2019-03-01"), v1); - final DataSegment segment3 = newSegment(Intervals.of("2019-03-01/2019-04-01"), v1); - final DataSegment segment4 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v2); - final DataSegment segment5 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v3); + final DataSegment segment1V1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v1); + final DataSegment segment2V1 = newSegment(Intervals.of("2019-02-01/2019-03-01"), v1); + final DataSegment segment3V1 = newSegment(Intervals.of("2019-03-01/2019-04-01"), v1); + final DataSegment segment4V2 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v2); + final DataSegment segment5V3 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v3); - final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4, segment5); + final Set segments = ImmutableSet.of(segment1V1, segment2V1, segment3V1, segment4V2, segment5V3); Assert.assertEquals(segments, getMetadataStorageCoordinator().commitSegments(segments)); Assert.assertEquals( @@ -278,7 +280,7 @@ public void testKillSegmentsWithVersionsAndLimit() throws Exception null ); - Assert.assertEquals(ImmutableSet.of(segment3, segment4, segment5), new HashSet<>(observedUnusedSegments)); + Assert.assertEquals(ImmutableSet.of(segment3V1, segment4V2, segment5V3), new HashSet<>(observedUnusedSegments)); } @Test @@ -289,13 +291,13 @@ public void testKillWithNonExistentVersion() throws Exception final String v2 = now.minusHours(2).toString(); final String v3 = now.minusHours(3).toString(); - final DataSegment segment1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v1); - final DataSegment segment2 = newSegment(Intervals.of("2019-02-01/2019-03-01"), v1); - final DataSegment segment3 = newSegment(Intervals.of("2019-03-01/2019-04-01"), v1); - final DataSegment segment4 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v2); - final DataSegment segment5 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v3); + final DataSegment segment1V1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v1); + final DataSegment segment2V1 = newSegment(Intervals.of("2019-02-01/2019-03-01"), v1); + final DataSegment segment3V1 = newSegment(Intervals.of("2019-03-01/2019-04-01"), v1); + final DataSegment segment4V2 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v2); + final DataSegment segment5V3 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v3); - final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4, segment5); + final Set segments = ImmutableSet.of(segment1V1, segment2V1, segment3V1, segment4V2, segment5V3); Assert.assertEquals(segments, getMetadataStorageCoordinator().commitSegments(segments)); Assert.assertEquals( @@ -347,12 +349,12 @@ public void testKillUnusedSegmentsWithUsedLoadSpec() throws Exception final String v2 = now.minusHours(2).toString(); final String v3 = now.minusHours(3).toString(); - final DataSegment segment1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v1, ImmutableMap.of("foo", "1")); - final DataSegment segment2 = newSegment(Intervals.of("2019-02-01/2019-03-01"), v2, ImmutableMap.of("foo", "1")); - final DataSegment segment3 = newSegment(Intervals.of("2019-03-01/2019-04-01"), v3, ImmutableMap.of("foo", "1")); + final DataSegment segment1V1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), v1, ImmutableMap.of("foo", "1")); + final DataSegment segment2V2 = newSegment(Intervals.of("2019-02-01/2019-03-01"), v2, ImmutableMap.of("foo", "1")); + final DataSegment segment3V3 = newSegment(Intervals.of("2019-03-01/2019-04-01"), v3, ImmutableMap.of("foo", "1")); - final Set segments = ImmutableSet.of(segment1, segment2, segment3); - final Set unusedSegments = ImmutableSet.of(segment1, segment2); + final Set segments = ImmutableSet.of(segment1V1, segment2V2, segment3V3); + final Set unusedSegments = ImmutableSet.of(segment1V1, segment2V2); Assert.assertEquals(segments, getMetadataStorageCoordinator().commitSegments(segments)); Assert.assertEquals( @@ -412,10 +414,10 @@ public void testKillBatchSizeOneAndLimit4() throws Exception { final String version = DateTimes.nowUtc().toString(); final Set segments = ImmutableSet.of( - newSegment(Intervals.of("2019-01-01/2019-02-01"), version), - newSegment(Intervals.of("2019-02-01/2019-03-01"), version), - newSegment(Intervals.of("2019-03-01/2019-04-01"), version), - newSegment(Intervals.of("2019-04-01/2019-05-01"), version) + segment1, + segment2, + segment3, + segment4 ); final Set announced = getMetadataStorageCoordinator().commitSegments(segments); @@ -454,7 +456,10 @@ public void testKillBatchSizeOneAndLimit4() throws Exception ); Assert.assertEquals(Collections.emptyList(), unusedSegments); - Assert.assertEquals(new KillTaskReport.Stats(4, 4, 0), getReportedStats()); + Assert.assertEquals( + new KillTaskReport.Stats(4, 4, 0), + getReportedStats() + ); } /** @@ -465,12 +470,6 @@ public void testKillBatchSizeOneAndLimit4() throws Exception @Test public void testKillMultipleUnusedSegmentsWithNullMaxUsedStatusLastUpdatedTime() throws Exception { - final String version = DateTimes.nowUtc().toString(); - final DataSegment segment1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), version); - final DataSegment segment2 = newSegment(Intervals.of("2019-02-01/2019-03-01"), version); - final DataSegment segment3 = newSegment(Intervals.of("2019-03-01/2019-04-01"), version); - final DataSegment segment4 = newSegment(Intervals.of("2019-04-01/2019-05-01"), version); - final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4); final Set announced = getMetadataStorageCoordinator().commitSegments(segments); @@ -530,7 +529,10 @@ public void testKillMultipleUnusedSegmentsWithNullMaxUsedStatusLastUpdatedTime() ); Assert.assertEquals(ImmutableList.of(), unusedSegments); - Assert.assertEquals(new KillTaskReport.Stats(3, 4, 0), getReportedStats()); + Assert.assertEquals( + new KillTaskReport.Stats(3, 4, 0), + getReportedStats() + ); } /** @@ -548,12 +550,6 @@ public void testKillMultipleUnusedSegmentsWithNullMaxUsedStatusLastUpdatedTime() @Test public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedTime() throws Exception { - final String version = DateTimes.nowUtc().toString(); - final DataSegment segment1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), version); - final DataSegment segment2 = newSegment(Intervals.of("2019-02-01/2019-03-01"), version); - final DataSegment segment3 = newSegment(Intervals.of("2019-03-01/2019-04-01"), version); - final DataSegment segment4 = newSegment(Intervals.of("2019-04-01/2019-05-01"), version); - final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4); final Set announced = getMetadataStorageCoordinator().commitSegments(segments); @@ -620,7 +616,10 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ); Assert.assertEquals(ImmutableList.of(segment3), unusedSegments); - Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); + Assert.assertEquals( + new KillTaskReport.Stats(2, 3, 0), + getReportedStats() + ); final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( null, @@ -645,7 +644,10 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ); Assert.assertEquals(ImmutableList.of(), unusedSegments2); - Assert.assertEquals(new KillTaskReport.Stats(1, 2, 0), getReportedStats()); + Assert.assertEquals( + new KillTaskReport.Stats(1, 2, 0), + getReportedStats() + ); } /** @@ -664,12 +666,6 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT @Test public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedTime2() throws Exception { - final String version = DateTimes.nowUtc().toString(); - final DataSegment segment1 = newSegment(Intervals.of("2019-01-01/2019-02-01"), version); - final DataSegment segment2 = newSegment(Intervals.of("2019-02-01/2019-03-01"), version); - final DataSegment segment3 = newSegment(Intervals.of("2019-03-01/2019-04-01"), version); - final DataSegment segment4 = newSegment(Intervals.of("2019-04-01/2019-05-01"), version); - final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4); final Set announced = getMetadataStorageCoordinator().commitSegments(segments); @@ -732,7 +728,10 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ); Assert.assertEquals(ImmutableList.of(segment2, segment3), observedUnusedSegments1); - Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); + Assert.assertEquals( + new KillTaskReport.Stats(2, 3, 0), + getReportedStats() + ); final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( null, @@ -757,7 +756,10 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ); Assert.assertEquals(ImmutableList.of(), observedUnusedSegments2); - Assert.assertEquals(new KillTaskReport.Stats(2, 3, 0), getReportedStats()); + Assert.assertEquals( + new KillTaskReport.Stats(2, 3, 0), + getReportedStats() + ); } @Test @@ -861,15 +863,8 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime @Test public void testKillBatchSizeThree() throws Exception { - final String version = DateTimes.nowUtc().toString(); - final Set segments = ImmutableSet.of( - newSegment(Intervals.of("2019-01-01/2019-02-01"), version), - newSegment(Intervals.of("2019-02-01/2019-03-01"), version), - newSegment(Intervals.of("2019-03-01/2019-04-01"), version), - newSegment(Intervals.of("2019-04-01/2019-05-01"), version) - ); + final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4); final Set announced = getMetadataStorageCoordinator().commitSegments(segments); - Assert.assertEquals(segments, announced); final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( @@ -886,8 +881,6 @@ public void testKillBatchSizeThree() throws Exception Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); - // we expect ALL tasks to be deleted - final List unusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( DATA_SOURCE, Intervals.of("2019/2020"), @@ -896,8 +889,10 @@ public void testKillBatchSizeThree() throws Exception ); Assert.assertEquals(Collections.emptyList(), unusedSegments); - - Assert.assertEquals(new KillTaskReport.Stats(4, 3, 4), getReportedStats()); + Assert.assertEquals( + new KillTaskReport.Stats(4, 3, 4), + getReportedStats() + ); } @Test diff --git a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java index f8983c8a0002..48581fe080d8 100644 --- a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java +++ b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java @@ -137,7 +137,6 @@ public Supplier metadataTablesConfigSupplier() return dbTables; } - public Integer updateSegmentsTable(String sqlFormat, Object... args) { return this.getConnector().retryWithHandle( diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java index dce901ec4cbc..e34e2b9bb3d2 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java @@ -934,25 +934,4 @@ private void updateUsedStatusLastUpdated(DataSegment segment, DateTime lastUpdat segment.getId().toString() ); } - - private void updateUsedStatusLastUpdatedBoilerPlate(DataSegment segment, DateTime lastUpdatedTime) - { - derbyConnectorRule.getConnector().retryWithHandle( - handle -> handle.update( - StringUtils.format( - "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = ? WHERE ID = ?", getSegmentsTable() - ), - lastUpdatedTime.toString(), - segment.getId().toString() - ) - ); - } - - private String getSegmentsTable() - { - return derbyConnectorRule.metadataTablesConfigSupplier() - .get() - .getSegmentsTable() - .toUpperCase(Locale.ENGLISH); - } } From a38a5ed1589c082bef17880269dc6e426c12e225 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 14 Mar 2024 20:55:59 -0700 Subject: [PATCH 10/14] Fix for checkstyle. --- .../druid/server/coordinator/duty/KillUnusedSegmentsTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java index e34e2b9bb3d2..8e9bf6e69fd9 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java @@ -37,7 +37,6 @@ import org.apache.druid.java.util.common.CloseableIterators; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.metadata.SQLMetadataSegmentPublisher; import org.apache.druid.metadata.SegmentsMetadataManagerConfig; @@ -69,7 +68,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; public class KillUnusedSegmentsTest From b63c0568b8166f6286c454c5fdcb3d0be37f1994 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Fri, 15 Mar 2024 07:02:45 -0700 Subject: [PATCH 11/14] review comments: indentation fixes, type. --- .../task/KillUnusedSegmentsTaskTest.java | 97 ++++++++++--------- ...exerSQLMetadataStorageCoordinatorTest.java | 2 +- .../druid/metadata/TestDerbyConnector.java | 9 +- 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java index 8a6adcd15426..55132d0477e8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java @@ -106,11 +106,11 @@ public void testKill() throws Exception final List observedUnusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( - DATA_SOURCE, - Intervals.of("2019/2020"), - null, - null, - null + DATA_SOURCE, + Intervals.of("2019/2020"), + null, + null, + null ); Assert.assertEquals(ImmutableList.of(segment2), observedUnusedSegments); @@ -155,7 +155,7 @@ public void testKillWithMarkUnused() throws Exception Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); - final List unusedSegments = + final List observedUnusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( DATA_SOURCE, Intervals.of("2019/2020"), @@ -164,7 +164,7 @@ public void testKillWithMarkUnused() throws Exception null ); - Assert.assertEquals(ImmutableList.of(segment2), unusedSegments); + Assert.assertEquals(ImmutableList.of(segment2), observedUnusedSegments); Assertions.assertThat( getMetadataStorageCoordinator().retrieveUsedSegmentsForInterval( DATA_SOURCE, @@ -221,12 +221,13 @@ public void testKillSegmentsWithVersions() throws Exception getReportedStats() ); - final List observedUnusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( - DATA_SOURCE, - Intervals.of("2018/2020"), - null, - null - ); + final List observedUnusedSegments = + getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( + DATA_SOURCE, + Intervals.of("2018/2020"), + null, + null + ); Assert.assertEquals(ImmutableSet.of(segment5V3), new HashSet<>(observedUnusedSegments)); } @@ -273,12 +274,13 @@ public void testKillSegmentsWithVersionsAndLimit() throws Exception getReportedStats() ); - final List observedUnusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( - DATA_SOURCE, - Intervals.of("2018/2020"), - null, - null - ); + final List observedUnusedSegments = + getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( + DATA_SOURCE, + Intervals.of("2018/2020"), + null, + null + ); Assert.assertEquals(ImmutableSet.of(segment3V1, segment4V2, segment5V3), new HashSet<>(observedUnusedSegments)); } @@ -325,12 +327,13 @@ public void testKillWithNonExistentVersion() throws Exception getReportedStats() ); - final List observedUnusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( - DATA_SOURCE, - Intervals.of("2018/2020"), - null, - null - ); + final List observedUnusedSegments = + getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( + DATA_SOURCE, + Intervals.of("2018/2020"), + null, + null + ); Assert.assertEquals(segments, new HashSet<>(observedUnusedSegments)); } @@ -382,12 +385,13 @@ public void testKillUnusedSegmentsWithUsedLoadSpec() throws Exception getReportedStats() ); - final List observedUnusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( - DATA_SOURCE, - Intervals.of("2018/2020"), - null, - null - ); + final List observedUnusedSegments = + getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( + DATA_SOURCE, + Intervals.of("2018/2020"), + null, + null + ); Assert.assertEquals(ImmutableSet.of(), new HashSet<>(observedUnusedSegments)); } @@ -447,7 +451,7 @@ public void testKillBatchSizeOneAndLimit4() throws Exception // we expect ALL tasks to be deleted - final List unusedSegments = + final List observedUnusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( DATA_SOURCE, Intervals.of("2019/2020"), @@ -455,7 +459,7 @@ public void testKillBatchSizeOneAndLimit4() throws Exception null ); - Assert.assertEquals(Collections.emptyList(), unusedSegments); + Assert.assertEquals(Collections.emptyList(), observedUnusedSegments); Assert.assertEquals( new KillTaskReport.Stats(4, 4, 0), getReportedStats() @@ -520,7 +524,7 @@ public void testKillMultipleUnusedSegmentsWithNullMaxUsedStatusLastUpdatedTime() Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); - final List unusedSegments = + final List observedUnusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( DATA_SOURCE, umbrellaInterval, @@ -528,7 +532,7 @@ public void testKillMultipleUnusedSegmentsWithNullMaxUsedStatusLastUpdatedTime() null ); - Assert.assertEquals(ImmutableList.of(), unusedSegments); + Assert.assertEquals(ImmutableList.of(), observedUnusedSegments); Assert.assertEquals( new KillTaskReport.Stats(3, 4, 0), getReportedStats() @@ -607,7 +611,7 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); - final List unusedSegments = + final List observedUnusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( DATA_SOURCE, umbrellaInterval, @@ -615,7 +619,7 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT null ); - Assert.assertEquals(ImmutableList.of(segment3), unusedSegments); + Assert.assertEquals(ImmutableList.of(segment3), observedUnusedSegments); Assert.assertEquals( new KillTaskReport.Stats(2, 3, 0), getReportedStats() @@ -635,7 +639,7 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); - final List unusedSegments2 = + final List observedUnusedSegments2 = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( DATA_SOURCE, umbrellaInterval, @@ -643,7 +647,7 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT null ); - Assert.assertEquals(ImmutableList.of(), unusedSegments2); + Assert.assertEquals(ImmutableList.of(), observedUnusedSegments2); Assert.assertEquals( new KillTaskReport.Stats(1, 2, 0), getReportedStats() @@ -881,14 +885,15 @@ public void testKillBatchSizeThree() throws Exception Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); - final List unusedSegments = getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( - DATA_SOURCE, - Intervals.of("2019/2020"), - null, - null - ); + final List observedUnusedSegments = + getMetadataStorageCoordinator().retrieveUnusedSegmentsForInterval( + DATA_SOURCE, + Intervals.of("2019/2020"), + null, + null + ); - Assert.assertEquals(Collections.emptyList(), unusedSegments); + Assert.assertEquals(Collections.emptyList(), observedUnusedSegments); Assert.assertEquals( new KillTaskReport.Stats(4, 3, 4), getReportedStats() diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java index 706b825a82cf..dd4b11003609 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java @@ -383,7 +383,7 @@ private void markAllSegmentsUnused(Set segments, DateTime usedStatu for (final DataSegment segment : segments) { Assert.assertEquals( 1, - (int) derbyConnectorRule.updateSegmentsTable( + derbyConnectorRule.updateSegmentsTable( "UPDATE %s SET used = false, used_status_last_updated = ? WHERE id = ?", usedStatusLastUpdatedTime.toString(), segment.getId().toString() diff --git a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java index 48581fe080d8..a1a9d2f75022 100644 --- a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java +++ b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java @@ -137,7 +137,14 @@ public Supplier metadataTablesConfigSupplier() return dbTables; } - public Integer updateSegmentsTable(String sqlFormat, Object... args) + /** + * Updates the segments table with the supplied SQL query format and arguments. + * + * @param sqlFormat the SQL query format with %s placeholder for the table name and ? for each query {@code args} + * @param args the arguments to be substituted into the SQL query + * @return the number of rows affected by the update operation + */ + public int updateSegmentsTable(String sqlFormat, Object... args) { return this.getConnector().retryWithHandle( handle -> handle.update( From 5a311b604c219ecb40a854407315cc9ed10160ce Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Fri, 15 Mar 2024 07:10:03 -0700 Subject: [PATCH 12/14] Wrapper class for Segments table --- .../task/KillUnusedSegmentsTaskTest.java | 31 +++++-------- ...exerSQLMetadataStorageCoordinatorTest.java | 2 +- .../metadata/SQLMetadataConnectorTest.java | 2 +- .../SqlSegmentsMetadataManagerTest.java | 39 ++++++---------- .../druid/metadata/TestDerbyConnector.java | 44 ++++++++++++++++--- .../duty/KillUnusedSegmentsTest.java | 11 +---- 6 files changed, 67 insertions(+), 62 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java index 55132d0477e8..2f43f48856a8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java @@ -576,8 +576,8 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ); final DateTime lastUpdatedTime1 = DateTimes.nowUtc(); - updateUsedStatusLastUpdated(segment1, lastUpdatedTime1); - updateUsedStatusLastUpdated(segment4, lastUpdatedTime1); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment1.getId().toString(), lastUpdatedTime1); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment4.getId().toString(), lastUpdatedTime1); // Now mark the third segment as unused Assert.assertEquals( @@ -589,7 +589,7 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ); final DateTime lastUpdatedTime2 = DateTimes.nowUtc(); - updateUsedStatusLastUpdated(segment3, lastUpdatedTime2); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment3.getId().toString(), lastUpdatedTime2); final List segmentIntervals = segments.stream() .map(DataSegment::getInterval) @@ -686,8 +686,8 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ); final DateTime lastUpdatedTime1 = DateTimes.nowUtc(); - updateUsedStatusLastUpdated(segment1, lastUpdatedTime1); - updateUsedStatusLastUpdated(segment4, lastUpdatedTime1); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment1.getId().toString(), lastUpdatedTime1); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment4.getId().toString(), lastUpdatedTime1); Assert.assertEquals( 2, @@ -700,8 +700,8 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT ); final DateTime lastUpdatedTime2 = DateTimes.nowUtc(); - updateUsedStatusLastUpdated(segment2, lastUpdatedTime2); - updateUsedStatusLastUpdated(segment3, lastUpdatedTime2); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment2.getId().toString(), lastUpdatedTime2); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment3.getId().toString(), lastUpdatedTime2); final List segmentIntervals = segments.stream() .map(DataSegment::getInterval) @@ -787,9 +787,9 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime ); final DateTime lastUpdatedTime1 = DateTimes.nowUtc(); - updateUsedStatusLastUpdated(segment1, lastUpdatedTime1); - updateUsedStatusLastUpdated(segment2, lastUpdatedTime1); - updateUsedStatusLastUpdated(segment4, lastUpdatedTime1); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment1.getId().toString(), lastUpdatedTime1); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment2.getId().toString(), lastUpdatedTime1); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment4.getId().toString(), lastUpdatedTime1); Assert.assertEquals( 2, @@ -799,7 +799,7 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime ); final DateTime lastUpdatedTime2 = DateTimes.nowUtc(); - updateUsedStatusLastUpdated(segment4, lastUpdatedTime2); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment4.getId().toString(), lastUpdatedTime2); final List segmentIntervals = segments.stream() .map(DataSegment::getInterval) @@ -1178,13 +1178,4 @@ private static DataSegment newSegment(Interval interval, String version, Map segments, DateTime usedStatu for (final DataSegment segment : segments) { Assert.assertEquals( 1, - derbyConnectorRule.updateSegmentsTable( + derbyConnectorRule.segments().update( "UPDATE %s SET used = false, used_status_last_updated = ? WHERE id = ?", usedStatusLastUpdatedTime.toString(), segment.getId().toString() diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java index fa8d75e4a49e..d240b83dd07d 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java @@ -172,7 +172,7 @@ public void testGeIndexOnNoTable() public void testAlterSegmentTableAddLastUsed() { connector.createSegmentTable(); - derbyConnectorRule.updateSegmentsTable("ALTER TABLE %1$s DROP COLUMN USED_STATUS_LAST_UPDATED"); + derbyConnectorRule.segments().update("ALTER TABLE %1$s DROP COLUMN USED_STATUS_LAST_UPDATED"); connector.alterSegmentTableAddUsedFlagLastUpdated(); connector.tableHasColumn( diff --git a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java index f51dc74eeeaf..b177b40c5876 100644 --- a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java @@ -41,7 +41,6 @@ import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.NoneShardSpec; import org.hamcrest.MatcherAssert; -import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Interval; import org.joda.time.Period; @@ -52,7 +51,6 @@ import org.junit.Test; import java.io.IOException; -import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; @@ -401,7 +399,10 @@ public void testGetUnusedSegmentIntervals() throws IOException "2017-10-15T20:19:12.565Z" ); publishUnusedSegments(koalaSegment1); - updateUsedStatusLastUpdated(koalaSegment1, DateTimes.nowUtc().minus(Duration.standardHours(2))); + derbyConnectorRule.segments().updateUsedStatusLastUpdated( + koalaSegment1.getId().toString(), + DateTimes.nowUtc().minus(Duration.standardHours(2)) + ); // Publish an unused segment with used_status_last_updated 2 days ago final DataSegment koalaSegment2 = createSegment( @@ -410,7 +411,10 @@ public void testGetUnusedSegmentIntervals() throws IOException "2017-10-15T20:19:12.565Z" ); publishUnusedSegments(koalaSegment2); - updateUsedStatusLastUpdated(koalaSegment2, DateTimes.nowUtc().minus(Duration.standardDays(2))); + derbyConnectorRule.segments().updateUsedStatusLastUpdated( + koalaSegment2.getId().toString(), + DateTimes.nowUtc().minus(Duration.standardDays(2)) + ); // Publish an unused segment and set used_status_last_updated to null final DataSegment koalaSegment3 = createSegment( @@ -910,7 +914,7 @@ private int getCountOfRowsWithLastUsedNull() handle -> handle.select( StringUtils.format( "SELECT ID FROM %1$s WHERE USED_STATUS_LAST_UPDATED IS NULL", - getSegmentsTable() + derbyConnectorRule.segments().getTableName() ) ).size() ); @@ -918,25 +922,16 @@ private int getCountOfRowsWithLastUsedNull() private void updateSegmentPayload(DataSegment segment, byte[] payload) { - derbyConnectorRule.updateSegmentsTable( + derbyConnectorRule.segments().update( "UPDATE %1$s SET PAYLOAD = ? WHERE ID = ?", payload, segment.getId().toString() ); } - private void updateUsedStatusLastUpdated(DataSegment segment, DateTime newValue) - { - derbyConnectorRule.updateSegmentsTable( - "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = ? WHERE ID = ?", - newValue.toString(), - segment.getId().toString() - ); - } - private void updateUsedStatusLastUpdatedToNull(DataSegment segment) { - derbyConnectorRule.updateSegmentsTable( + derbyConnectorRule.segments().update( "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = NULL WHERE ID = ?", segment.getId().toString() ); @@ -949,14 +944,8 @@ private void updateUsedStatusLastUpdatedToNull(DataSegment segment) */ private void allowUsedFlagLastUpdatedToBeNullable() { - derbyConnectorRule.updateSegmentsTable("ALTER TABLE %1$s ALTER COLUMN USED_STATUS_LAST_UPDATED NULL"); - } - - private String getSegmentsTable() - { - return derbyConnectorRule.metadataTablesConfigSupplier() - .get() - .getSegmentsTable() - .toUpperCase(Locale.ENGLISH); + derbyConnectorRule.segments().update( + "ALTER TABLE %1$s ALTER COLUMN USED_STATUS_LAST_UPDATED NULL" + ); } } diff --git a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java index a1a9d2f75022..efa4972f2afe 100644 --- a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java +++ b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java @@ -23,6 +23,7 @@ import com.google.common.base.Suppliers; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.metadata.storage.derby.DerbyConnector; +import org.joda.time.DateTime; import org.junit.Assert; import org.junit.rules.ExternalResource; import org.skife.jdbi.v2.DBI; @@ -137,6 +138,24 @@ public Supplier metadataTablesConfigSupplier() return dbTables; } + public SegmentsTable segments() + { + return new SegmentsTable(this); + } + } + + /** + * A wrapper class for queries on the segments table. + */ + public static class SegmentsTable + { + private final DerbyConnectorRule rule; + + public SegmentsTable(DerbyConnectorRule rule) + { + this.rule = rule; + } + /** * Updates the segments table with the supplied SQL query format and arguments. * @@ -144,19 +163,34 @@ public Supplier metadataTablesConfigSupplier() * @param args the arguments to be substituted into the SQL query * @return the number of rows affected by the update operation */ - public int updateSegmentsTable(String sqlFormat, Object... args) + public int update(String sqlFormat, Object... args) { - return this.getConnector().retryWithHandle( + return this.rule.getConnector().retryWithHandle( handle -> handle.update( - StringUtils.format(sqlFormat, getSegmentsTable()), + StringUtils.format(sqlFormat, getTableName()), args ) ); } - private String getSegmentsTable() + /** + * Updates the last used status time for the supplied segment ID. + */ + public int updateUsedStatusLastUpdated(String segmentId, DateTime lastUpdatedTime) + { + return update( + "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = ? WHERE ID = ?", + lastUpdatedTime.toString(), + segmentId + ); + } + + /** + * @return the segment table name. + */ + public String getTableName() { - return this.metadataTablesConfigSupplier() + return this.rule.metadataTablesConfigSupplier() .get() .getSegmentsTable() .toUpperCase(Locale.ENGLISH); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java index 8e9bf6e69fd9..aa37913734ee 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java @@ -774,7 +774,7 @@ private void createAndAddUnusedSegment( throw new RuntimeException(e); } sqlSegmentsMetadataManager.markSegmentsAsUnused(ImmutableSet.of(segment.getId())); - updateUsedStatusLastUpdated(segment, lastUpdatedTime); + derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment.getId().toString(), lastUpdatedTime); } private DataSegment createSegment(final String dataSource, final Interval interval, final String version) @@ -923,13 +923,4 @@ void deleteLastKillTaskId(final String dataSource) observedDatasourceToLastKillTaskId.remove(dataSource); } } - - private void updateUsedStatusLastUpdated(DataSegment segment, DateTime lastUpdatedTime) - { - derbyConnectorRule.updateSegmentsTable( - "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = ? WHERE ID = ?", - lastUpdatedTime.toString(), - segment.getId().toString() - ); - } } From c0e097d488cd054d4718ac37ef7ec604af509fdd Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Fri, 15 Mar 2024 07:48:27 -0700 Subject: [PATCH 13/14] Add KillUnusedSegmentsTaskBuilder in test class --- .../task/KillUnusedSegmentsTaskTest.java | 555 ++++++++---------- 1 file changed, 248 insertions(+), 307 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java index 2f43f48856a8..1e36cc825a01 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java @@ -90,17 +90,10 @@ public void testKill() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2019-03-01/2019-04-01")) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -141,17 +134,11 @@ public void testKillWithMarkUnused() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - null, - true, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2019-03-01/2019-04-01")) + .markAsUnused(true) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -203,17 +190,12 @@ public void testKillSegmentsWithVersions() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018/2020"), - ImmutableList.of(v1, v2), - null, - false, - 3, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018/2020")) + .versions(ImmutableList.of(v1, v2)) + .batchSize(3) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); Assert.assertEquals( @@ -256,17 +238,13 @@ public void testKillSegmentsWithVersionsAndLimit() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018/2020"), - ImmutableList.of(v1), - null, - false, - 3, - 2, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018/2020")) + .versions(ImmutableList.of(v1)) + .batchSize(3) + .limit(2) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); Assert.assertEquals( @@ -309,17 +287,13 @@ public void testKillWithNonExistentVersion() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018/2020"), - ImmutableList.of(now.plusDays(100).toString()), - null, - false, - 3, - 2, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018/2020")) + .versions(ImmutableList.of(now.plusDays(100).toString())) + .batchSize(3) + .limit(2) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); Assert.assertEquals( @@ -367,17 +341,12 @@ public void testKillUnusedSegmentsWithUsedLoadSpec() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018/2020"), - ImmutableList.of(v1, v2), - null, - false, - null, - 100, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018/2020")) + .versions(ImmutableList.of(v1, v2)) + .limit(100) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); Assert.assertEquals( @@ -399,34 +368,21 @@ public void testKillUnusedSegmentsWithUsedLoadSpec() throws Exception @Test public void testGetInputSourceResources() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2019-03-01/2019-04-01"), - null, - null, - true, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2019-03-01/2019-04-01")) + .markAsUnused(true) + .build(); Assert.assertTrue(task.getInputSourceResources().isEmpty()); } @Test public void testKillBatchSizeOneAndLimit4() throws Exception { - final String version = DateTimes.nowUtc().toString(); - final Set segments = ImmutableSet.of( - segment1, - segment2, - segment3, - segment4 - ); + final Set segments = ImmutableSet.of(segment1, segment2, segment3, segment4); final Set announced = getMetadataStorageCoordinator().commitSegments(segments); Assert.assertEquals(segments, announced); - Assert.assertEquals( segments.size(), getSegmentsMetadataManager().markAsUnusedSegmentsInInterval( @@ -435,17 +391,12 @@ public void testKillBatchSizeOneAndLimit4() throws Exception ) ); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 1, - 4, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .batchSize(1) + .limit(4) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -509,18 +460,12 @@ public void testKillMultipleUnusedSegmentsWithNullMaxUsedStatusLastUpdatedTime() final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(umbrellaInterval) + .batchSize(1) + .limit(10) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -597,17 +542,13 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - lastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(umbrellaInterval) + .batchSize(1) + .limit(10) + .maxUsedStatusLastUpdatedTime(lastUpdatedTime1) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -625,17 +566,13 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT getReportedStats() ); - final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - lastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(umbrellaInterval) + .batchSize(1) + .limit(10) + .maxUsedStatusLastUpdatedTime(lastUpdatedTime2) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -709,17 +646,13 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - lastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(umbrellaInterval) + .batchSize(1) + .limit(10) + .maxUsedStatusLastUpdatedTime(lastUpdatedTime1) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); @@ -737,17 +670,13 @@ public void testKillMultipleUnusedSegmentsWithDifferentMaxUsedStatusLastUpdatedT getReportedStats() ); - final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - null, - null, - false, - 1, - 10, - lastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(umbrellaInterval) + .batchSize(1) + .limit(10) + .maxUsedStatusLastUpdatedTime(lastUpdatedTime2) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); @@ -807,17 +736,14 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - ImmutableList.of(version.toString()), - null, - false, - 1, - 10, - lastUpdatedTime1 - ); + final KillUnusedSegmentsTask task1 = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(umbrellaInterval) + .versions(ImmutableList.of(version.toString())) + .batchSize(1) + .limit(10) + .maxUsedStatusLastUpdatedTime(lastUpdatedTime1) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task1).get().getStatusCode()); Assert.assertEquals( @@ -835,17 +761,14 @@ public void testKillMultipleUnusedSegmentsWithVersionAndDifferentLastUpdatedTime Assert.assertEquals(ImmutableSet.of(segment3, segment4, segment5), new HashSet<>(observedUnusedSegments)); - final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - umbrellaInterval, - ImmutableList.of(version.toString()), - null, - false, - 1, - 10, - lastUpdatedTime2 - ); + final KillUnusedSegmentsTask task2 = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(umbrellaInterval) + .versions(ImmutableList.of(version.toString())) + .batchSize(1) + .limit(10) + .maxUsedStatusLastUpdatedTime(lastUpdatedTime2) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task2).get().getStatusCode()); Assert.assertEquals( @@ -871,17 +794,12 @@ public void testKillBatchSizeThree() throws Exception final Set announced = getMetadataStorageCoordinator().commitSegments(segments); Assert.assertEquals(segments, announced); - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - true, - 3, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .markAsUnused(true) + .batchSize(3) + .build(); Assert.assertEquals(TaskState.SUCCESS, taskRunner.run(task).get().getStatusCode()); @@ -903,102 +821,68 @@ public void testKillBatchSizeThree() throws Exception @Test public void testComputeNextBatchSizeDefault() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .build(); Assert.assertEquals(100, task.computeNextBatchSize(50)); } @Test public void testComputeNextBatchSizeWithBatchSizeLargerThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 10, - 5, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .batchSize(10) + .limit(5) + .build(); Assert.assertEquals(5, task.computeNextBatchSize(0)); } @Test public void testComputeNextBatchSizeWithBatchSizeSmallerThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .batchSize(5) + .limit(10) + .build(); Assert.assertEquals(5, task.computeNextBatchSize(0)); } @Test public void testComputeNextBatchSizeWithRemainingLessThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .batchSize(5) + .limit(10) + .build(); Assert.assertEquals(3, task.computeNextBatchSize(7)); } @Test public void testGetNumTotalBatchesDefault() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - null, - null, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .build(); Assert.assertNull(task.getNumTotalBatches()); } @Test public void testGetNumTotalBatchesWithBatchSizeLargerThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 10, - 5, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .batchSize(10) + .limit(5) + .build(); Assert.assertEquals(1, (int) task.getNumTotalBatches()); } @@ -1008,17 +892,11 @@ public void testInvalidLimit() MatcherAssert.assertThat( Assert.assertThrows( DruidException.class, - () -> new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 10, - 0, - null - ) + () -> new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .limit(0) + .build() ), DruidExceptionMatcher.invalidInput().expectMessageIs( "limit[0] must be a positive integer." @@ -1032,17 +910,11 @@ public void testInvalidBatchSize() MatcherAssert.assertThat( Assert.assertThrows( DruidException.class, - () -> new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 0, - 10, - null - ) + () -> new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .batchSize(0) + .build() ), DruidExceptionMatcher.invalidInput().expectMessageIs( "batchSize[0] must be a positive integer." @@ -1056,17 +928,13 @@ public void testInvalidLimitWithMarkAsUnused() MatcherAssert.assertThat( Assert.assertThrows( DruidException.class, - () -> new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - true, - 10, - 10, - null - ) + () -> new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .markAsUnused(true) + .batchSize(10) + .limit(10) + .build() ), DruidExceptionMatcher.invalidInput().expectMessageIs( "limit[10] cannot be provided when markAsUnused is enabled." @@ -1075,22 +943,17 @@ public void testInvalidLimitWithMarkAsUnused() } @Test - public void testInvalidVersionWithMarkAsUnused() + public void testInvalidVersionsWithMarkAsUnused() { MatcherAssert.assertThat( Assert.assertThrows( DruidException.class, - () -> new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - ImmutableList.of("foo"), - null, - true, - 10, - null, - null - ) + () -> new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .markAsUnused(true) + .versions(ImmutableList.of("foo")) + .build() ), DruidExceptionMatcher.invalidInput().expectMessageIs( "versions[[foo]] cannot be provided when markAsUnused is enabled." @@ -1101,17 +964,13 @@ public void testInvalidVersionWithMarkAsUnused() @Test public void testGetNumTotalBatchesWithBatchSizeSmallerThanLimit() { - final KillUnusedSegmentsTask task = new KillUnusedSegmentsTask( - null, - DATA_SOURCE, - Intervals.of("2018-01-01/2020-01-01"), - null, - null, - false, - 5, - 10, - null - ); + final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder() + .dataSource(DATA_SOURCE) + .interval(Intervals.of("2018-01-01/2020-01-01")) + .versions(ImmutableList.of("foo")) + .batchSize(5) + .limit(10) + .build(); Assert.assertEquals(2, (int) task.getNumTotalBatches()); } @@ -1133,6 +992,88 @@ public void testKillTaskReportSerde() throws Exception Assert.assertEquals(stats, deserializedKillReport.getPayload()); } + private static class KillUnusedSegmentsTaskBuilder + { + private String id; + private String dataSource; + private Interval interval; + private List versions; + private Map context; + private Boolean markAsUnused; + private Integer batchSize; + private Integer limit; + private DateTime maxUsedStatusLastUpdatedTime; + + public KillUnusedSegmentsTaskBuilder id(String id) + { + this.id = id; + return this; + } + + public KillUnusedSegmentsTaskBuilder dataSource(String dataSource) + { + this.dataSource = dataSource; + return this; + } + + public KillUnusedSegmentsTaskBuilder interval(Interval interval) + { + this.interval = interval; + return this; + } + + public KillUnusedSegmentsTaskBuilder versions(List versions) + { + this.versions = versions; + return this; + } + + public KillUnusedSegmentsTaskBuilder context(Map context) + { + this.context = context; + return this; + } + + public KillUnusedSegmentsTaskBuilder markAsUnused(Boolean markAsUnused) + { + this.markAsUnused = markAsUnused; + return this; + } + + public KillUnusedSegmentsTaskBuilder batchSize(Integer batchSize) + { + this.batchSize = batchSize; + return this; + } + + public KillUnusedSegmentsTaskBuilder limit(Integer limit) + { + this.limit = limit; + return this; + } + + public KillUnusedSegmentsTaskBuilder maxUsedStatusLastUpdatedTime(DateTime maxUsedStatusLastUpdatedTime) + { + this.maxUsedStatusLastUpdatedTime = maxUsedStatusLastUpdatedTime; + return this; + } + + public KillUnusedSegmentsTask build() + { + return new KillUnusedSegmentsTask( + id, + dataSource, + interval, + versions, + context, + markAsUnused, + batchSize, + limit, + maxUsedStatusLastUpdatedTime + ); + } + } + private KillTaskReport.Stats getReportedStats() { try { From 674962f94c8d3c12f65f8875f88010d885cbefb7 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Fri, 15 Mar 2024 07:55:42 -0700 Subject: [PATCH 14/14] Remove javadocs for self-explanatory methods. --- .../java/org/apache/druid/metadata/TestDerbyConnector.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java index efa4972f2afe..1fa2ef2302c9 100644 --- a/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java +++ b/server/src/test/java/org/apache/druid/metadata/TestDerbyConnector.java @@ -173,9 +173,6 @@ public int update(String sqlFormat, Object... args) ); } - /** - * Updates the last used status time for the supplied segment ID. - */ public int updateUsedStatusLastUpdated(String segmentId, DateTime lastUpdatedTime) { return update( @@ -185,9 +182,6 @@ public int updateUsedStatusLastUpdated(String segmentId, DateTime lastUpdatedTim ); } - /** - * @return the segment table name. - */ public String getTableName() { return this.rule.metadataTablesConfigSupplier()