From 3fe99b932ee669a70e5678ee734de45eedabcaf4 Mon Sep 17 00:00:00 2001 From: Maytas Monsereenusorn Date: Fri, 10 Dec 2021 10:41:51 -0800 Subject: [PATCH 1/8] add impl --- .../duty/ITAutoCompactionTest.java | 1330 +++++++++-------- .../ClientCompactionIntervalSpec.java | 2 +- .../HttpIndexingServiceClientTest.java | 72 + 3 files changed, 765 insertions(+), 639 deletions(-) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java index 6440f56b588a..c217f4646aa0 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java @@ -25,6 +25,7 @@ import org.apache.commons.io.IOUtils; import org.apache.druid.data.input.MaxSizeSplitHintSpec; import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; import org.apache.druid.indexer.partitions.HashedPartitionsSpec; import org.apache.druid.indexer.partitions.PartitionsSpec; @@ -102,672 +103,725 @@ public void setup() throws Exception fullDatasourceName = "wikipedia_index_test_" + UUID.randomUUID() + config.getExtraDatasourceNameSuffix(); } - @Test - public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, Period.days(1)); - //...compacted into 1 new segment for 1 day. 1 day compacted and 1 day skipped/remains uncompacted. (3 total) - forceTriggerAutoCompaction(3); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); - checkCompactionIntervals(intervalsBeforeCompaction); - getAndAssertCompactionStatus( - fullDatasourceName, - AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, - 0, - 14370, - 0, - 0, - 2, - 0, - 0, - 1, - 0); - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); - //...compacted into 1 new segment for the remaining one day. 2 day compacted and 0 day uncompacted. (2 total) - forceTriggerAutoCompaction(2); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); - checkCompactionIntervals(intervalsBeforeCompaction); - getAndAssertCompactionStatus( - fullDatasourceName, - AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, - 0, - 22568, - 0, - 0, - 3, - 0, - 0, - 2, - 0); - } - } - - @Test - public void testAutoCompactionDutyCanUpdateCompactionConfig() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - // Dummy compaction config which will be overwritten - submitCompactionConfig(10000, NO_SKIP_OFFSET); - // New compaction config should overwrites the existing compaction config - submitCompactionConfig(1, NO_SKIP_OFFSET); - - LOG.info("Auto compaction test with dynamic partitioning"); - - // Instead of merging segments, the updated config will split segments! - //...compacted into 10 new segments across 2 days. 5 new segments each day (10 total) - forceTriggerAutoCompaction(10); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(10, 1); - checkCompactionIntervals(intervalsBeforeCompaction); - - LOG.info("Auto compaction test with hash partitioning"); - - final HashedPartitionsSpec hashedPartitionsSpec = new HashedPartitionsSpec(null, 3, null); - submitCompactionConfig(hashedPartitionsSpec, NO_SKIP_OFFSET, 1, null, null, null, false); - // 2 segments published per day after compaction. - forceTriggerAutoCompaction(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(hashedPartitionsSpec, 4); - checkCompactionIntervals(intervalsBeforeCompaction); - - LOG.info("Auto compaction test with range partitioning"); - - final SingleDimensionPartitionsSpec rangePartitionsSpec = new SingleDimensionPartitionsSpec( - 5, - null, - "city", - false - ); - submitCompactionConfig(rangePartitionsSpec, NO_SKIP_OFFSET, 1, null, null, null, false); - forceTriggerAutoCompaction(2); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(rangePartitionsSpec, 2); - checkCompactionIntervals(intervalsBeforeCompaction); - } - } +// @Test +// public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, Period.days(1)); +// //...compacted into 1 new segment for 1 day. 1 day compacted and 1 day skipped/remains uncompacted. (3 total) +// forceTriggerAutoCompaction(3); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); +// checkCompactionIntervals(intervalsBeforeCompaction); +// getAndAssertCompactionStatus( +// fullDatasourceName, +// AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, +// 0, +// 14370, +// 0, +// 0, +// 2, +// 0, +// 0, +// 1, +// 0); +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); +// //...compacted into 1 new segment for the remaining one day. 2 day compacted and 0 day uncompacted. (2 total) +// forceTriggerAutoCompaction(2); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// checkCompactionIntervals(intervalsBeforeCompaction); +// getAndAssertCompactionStatus( +// fullDatasourceName, +// AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, +// 0, +// 22568, +// 0, +// 0, +// 3, +// 0, +// 0, +// 2, +// 0); +// } +// } +// +// @Test +// public void testAutoCompactionDutyCanUpdateCompactionConfig() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// // Dummy compaction config which will be overwritten +// submitCompactionConfig(10000, NO_SKIP_OFFSET); +// // New compaction config should overwrites the existing compaction config +// submitCompactionConfig(1, NO_SKIP_OFFSET); +// +// LOG.info("Auto compaction test with dynamic partitioning"); +// +// // Instead of merging segments, the updated config will split segments! +// //...compacted into 10 new segments across 2 days. 5 new segments each day (10 total) +// forceTriggerAutoCompaction(10); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(10, 1); +// checkCompactionIntervals(intervalsBeforeCompaction); +// +// LOG.info("Auto compaction test with hash partitioning"); +// +// final HashedPartitionsSpec hashedPartitionsSpec = new HashedPartitionsSpec(null, 3, null); +// submitCompactionConfig(hashedPartitionsSpec, NO_SKIP_OFFSET, 1, null, null, null, false); +// // 2 segments published per day after compaction. +// forceTriggerAutoCompaction(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(hashedPartitionsSpec, 4); +// checkCompactionIntervals(intervalsBeforeCompaction); +// +// LOG.info("Auto compaction test with range partitioning"); +// +// final SingleDimensionPartitionsSpec rangePartitionsSpec = new SingleDimensionPartitionsSpec( +// 5, +// null, +// "city", +// false +// ); +// submitCompactionConfig(rangePartitionsSpec, NO_SKIP_OFFSET, 1, null, null, null, false); +// forceTriggerAutoCompaction(2); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(rangePartitionsSpec, 2); +// checkCompactionIntervals(intervalsBeforeCompaction); +// } +// } +// +// @Test +// public void testAutoCompactionDutyCanDeleteCompactionConfig() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); +// deleteCompactionConfig(); +// +// // ...should remains unchanged (4 total) +// forceTriggerAutoCompaction(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(0, null); +// // Auto compaction stats should be deleted as compacation config was deleted +// Assert.assertNull(compactionResource.getCompactionStatus(fullDatasourceName)); +// checkCompactionIntervals(intervalsBeforeCompaction); +// } +// } +// +// @Test +// public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception +// { +// // Set compactionTaskSlotRatio to 0 to prevent any compaction +// updateCompactionTaskSlot(0, 0); +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); +// // ...should remains unchanged (4 total) +// forceTriggerAutoCompaction(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(0, null); +// checkCompactionIntervals(intervalsBeforeCompaction); +// Assert.assertNull(compactionResource.getCompactionStatus(fullDatasourceName)); +// // Update compaction slots to be 1 +// updateCompactionTaskSlot(1, 1); +// // One day compacted (1 new segment) and one day remains uncompacted. (3 total) +// forceTriggerAutoCompaction(3); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); +// checkCompactionIntervals(intervalsBeforeCompaction); +// getAndAssertCompactionStatus( +// fullDatasourceName, +// AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, +// 14370, +// 14369, +// 0, +// 2, +// 2, +// 0, +// 1, +// 1, +// 0); +// Assert.assertEquals(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize"), "14370"); +// // Run compaction again to compact the remaining day +// // Remaining day compacted (1 new segment). Now both days compacted (2 total) +// forceTriggerAutoCompaction(2); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// checkCompactionIntervals(intervalsBeforeCompaction); +// getAndAssertCompactionStatus( +// fullDatasourceName, +// AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, +// 0, +// 22568, +// 0, +// 0, +// 3, +// 0, +// 0, +// 2, +// 0); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithSegmentGranularityAndWithDropExistingTrue() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// Granularity newGranularity = Granularities.YEAR; +// // Set dropExisting to true +// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); +// +// LOG.info("Auto compaction test with YEAR segment granularity"); +// +// List expectedIntervalAfterCompaction = new ArrayList<>(); +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// forceTriggerAutoCompaction(1); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(1, 1000); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// +// newGranularity = Granularities.DAY; +// // Set dropExisting to true +// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); +// +// LOG.info("Auto compaction test with DAY segment granularity"); +// +// // Since dropExisting is set to true... +// // The earlier segment with YEAR granularity will be dropped post-compaction +// // Hence, we will only have 2013-08-31 to 2013-09-01 and 2013-09-01 to 2013-09-02. +// expectedIntervalAfterCompaction = new ArrayList<>(); +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// forceTriggerAutoCompaction(2); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(2, 1000); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithSegmentGranularityAndWithDropExistingFalse() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// Granularity newGranularity = Granularities.YEAR; +// // Set dropExisting to false +// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); +// +// LOG.info("Auto compaction test with YEAR segment granularity"); +// +// List expectedIntervalAfterCompaction = new ArrayList<>(); +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// forceTriggerAutoCompaction(1); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(1, 1000); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// +// newGranularity = Granularities.DAY; +// // Set dropExisting to false +// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); +// +// LOG.info("Auto compaction test with DAY segment granularity"); +// +// // Since dropExisting is set to false... +// // The earlier segment with YEAR granularity is still 'used' as it’s not fully overshaowed. +// // This is because we only have newer version on 2013-08-31 to 2013-09-01 and 2013-09-01 to 2013-09-02. +// // The version for the YEAR segment is still the latest for 2013-01-01 to 2013-08-31 and 2013-09-02 to 2014-01-01. +// // Hence, all three segments are available and the expected intervals are combined from the DAY and YEAR segment granularities +// // (which are 2013-08-31 to 2013-09-01, 2013-09-01 to 2013-09-02 and 2013-01-01 to 2014-01-01) +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// forceTriggerAutoCompaction(3); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(3, 1000); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithSegmentGranularityAndMixedVersion() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, Period.days(1)); +// //...compacted into 1 new segment for 1 day. 1 day compacted and 1 day skipped/remains uncompacted. (3 total) +// forceTriggerAutoCompaction(3); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); +// +// Granularity newGranularity = Granularities.YEAR; +// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); +// +// LOG.info("Auto compaction test with YEAR segment granularity"); +// +// List expectedIntervalAfterCompaction = new ArrayList<>(); +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// // Since the new segmentGranularity is YEAR, it will have mixed versions inside the same time chunk +// // There will be an old version (for the first day interval) from the initial ingestion and +// // a newer version (for the second day interval) from the first compaction +// forceTriggerAutoCompaction(1); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(1, 1000); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveSameSegmentGranularity() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// // Compacted without SegmentGranularity in auto compaction config +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); +// forceTriggerAutoCompaction(2); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// +// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// +// // Segments were compacted and already has DAY granularity since it was initially ingested with DAY granularity. +// // Now set auto compaction with DAY granularity in the granularitySpec +// Granularity newGranularity = Granularities.DAY; +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); +// forceTriggerAutoCompaction(2); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// // should be no new compaction task as segmentGranularity is already DAY +// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveDifferentSegmentGranularity() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// // Compacted without SegmentGranularity in auto compaction config +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); +// forceTriggerAutoCompaction(2); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// +// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// +// // Segments were compacted and already has DAY granularity since it was initially ingested with DAY granularity. +// // Now set auto compaction with DAY granularity in the granularitySpec +// Granularity newGranularity = Granularities.YEAR; +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); +// forceTriggerAutoCompaction(1); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); +// +// // There should be new compaction tasks since SegmentGranularity changed from DAY to YEAR +// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// Assert.assertTrue(compactTasksAfter.size() > compactTasksBefore.size()); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithSegmentGranularityAndSmallerSegmentGranularityCoveringMultipleSegmentsInTimelineAndDropExistingTrue() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// Granularity newGranularity = Granularities.YEAR; +// // Set dropExisting to true +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); +// +// List expectedIntervalAfterCompaction = new ArrayList<>(); +// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// forceTriggerAutoCompaction(1); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// +// loadData(INDEX_TASK); +// verifySegmentsCount(5); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// // 5 segments. 1 compacted YEAR segment and 4 newly ingested DAY segments across 2 days +// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from the compaction earlier +// // two segments with interval of 2013-08-31/2013-09-01 (newly ingested with DAY) +// // and two segments with interval of 2013-09-01/2013-09-02 (newly ingested with DAY) +// expectedIntervalAfterCompaction.addAll(intervalsBeforeCompaction); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// +// newGranularity = Granularities.MONTH; +// // Set dropExisting to true +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); +// // Since dropExisting is set to true... +// // This will submit a single compaction task for interval of 2013-01-01/2014-01-01 with MONTH granularity +// expectedIntervalAfterCompaction = new ArrayList<>(); +// // The previous segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) will be dropped +// // We will only have one segments with interval of 2013-09-01/2013-10-01 (compacted with MONTH) +// // and one segments with interval of 2013-10-01/2013-11-01 (compacted with MONTH) +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : Granularities.MONTH.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// forceTriggerAutoCompaction(2); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithSegmentGranularityAndSmallerSegmentGranularityCoveringMultipleSegmentsInTimelineAndDropExistingFalse() throws Exception +// { +// loadData(INDEX_TASK); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// +// Granularity newGranularity = Granularities.YEAR; +// // Set dropExisting to false +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); +// +// List expectedIntervalAfterCompaction = new ArrayList<>(); +// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// forceTriggerAutoCompaction(1); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// +// loadData(INDEX_TASK); +// verifySegmentsCount(5); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// // 5 segments. 1 compacted YEAR segment and 4 newly ingested DAY segments across 2 days +// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from the compaction earlier +// // two segments with interval of 2013-08-31/2013-09-01 (newly ingested with DAY) +// // and two segments with interval of 2013-09-01/2013-09-02 (newly ingested with DAY) +// expectedIntervalAfterCompaction.addAll(intervalsBeforeCompaction); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// +// newGranularity = Granularities.MONTH; +// // Set dropExisting to false +// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); +// // Since dropExisting is set to true... +// // This will submit a single compaction task for interval of 2013-01-01/2014-01-01 with MONTH granularity +// expectedIntervalAfterCompaction = new ArrayList<>(); +// // Since dropExisting is set to false... +// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from before the compaction +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : Granularities.YEAR.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// // one segments with interval of 2013-09-01/2013-10-01 (compacted with MONTH) +// // and one segments with interval of 2013-10-01/2013-11-01 (compacted with MONTH) +// for (String interval : intervalsBeforeCompaction) { +// for (Interval newinterval : Granularities.MONTH.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { +// expectedIntervalAfterCompaction.add(newinterval.toString()); +// } +// } +// +// forceTriggerAutoCompaction(3); +// verifyQuery(INDEX_QUERIES_RESOURCE); +// verifySegmentsCompacted(3, MAX_ROWS_PER_SEGMENT_COMPACTED); +// checkCompactionIntervals(expectedIntervalAfterCompaction); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithRollup() throws Exception +// { +// final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles")); +// Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.DAY, Granularities.DAY, false, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); +// loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// Map expectedResult = ImmutableMap.of( +// "%%EXPECTED_COUNT_RESULT%%", 2, +// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) +// ); +// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); +// submitCompactionConfig( +// MAX_ROWS_PER_SEGMENT_COMPACTED, +// NO_SKIP_OFFSET, +// new UserCompactionTaskGranularityConfig(null, null, true), +// false +// ); +// forceTriggerAutoCompaction(2); +// expectedResult = ImmutableMap.of( +// "%%EXPECTED_COUNT_RESULT%%", 1, +// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) +// ); +// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// +// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// // Verify rollup segments does not get compacted again +// forceTriggerAutoCompaction(2); +// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithQueryGranularity() throws Exception +// { +// final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles")); +// Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, true, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); +// loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// Map expectedResult = ImmutableMap.of( +// "%%EXPECTED_COUNT_RESULT%%", 2, +// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) +// ); +// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); +// submitCompactionConfig( +// MAX_ROWS_PER_SEGMENT_COMPACTED, +// NO_SKIP_OFFSET, +// new UserCompactionTaskGranularityConfig(null, Granularities.DAY, null), +// false +// ); +// forceTriggerAutoCompaction(2); +// expectedResult = ImmutableMap.of( +// "%%EXPECTED_COUNT_RESULT%%", 1, +// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) +// ); +// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// +// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// // Verify rollup segments does not get compacted again +// forceTriggerAutoCompaction(2); +// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithDimensionsSpec() throws Exception +// { +// // Index data with dimensions "page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous", +// // "namespace", "continent", "country", "region", "city" +// loadData(INDEX_TASK_WITH_DIMENSION_SPEC); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// +// // Result is not rollup +// Map expectedResult = ImmutableMap.of( +// "%%EXPECTED_COUNT_RESULT%%", 2, +// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) +// ); +// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); +// +// // Compact and change dimension to only "language" +// submitCompactionConfig( +// MAX_ROWS_PER_SEGMENT_COMPACTED, +// NO_SKIP_OFFSET, +// null, +// new UserCompactionTaskDimensionsConfig(DimensionsSpec.getDefaultSchemas(ImmutableList.of("language"))), +// null, +// false +// ); +// forceTriggerAutoCompaction(2); +// +// // Result should rollup on language dimension +// expectedResult = ImmutableMap.of( +// "%%EXPECTED_COUNT_RESULT%%", 1, +// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) +// ); +// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// +// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// // Verify compacted segments does not get compacted again +// forceTriggerAutoCompaction(2); +// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); +// } +// } +// +// @Test +// public void testAutoCompactionDutyWithFilter() throws Exception +// { +// loadData(INDEX_TASK_WITH_DIMENSION_SPEC); +// try (final Closeable ignored = unloader(fullDatasourceName)) { +// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); +// intervalsBeforeCompaction.sort(null); +// // 4 segments across 2 days (4 total)... +// verifySegmentsCount(4); +// +// // Result is not rollup +// // For dim "page", result has values "Gypsy Danger" and "Striker Eureka" +// Map expectedResult = ImmutableMap.of( +// "%%EXPECTED_COUNT_RESULT%%", 2, +// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) +// ); +// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); +// +// // Compact and filter with selector on dim "page" and value "Striker Eureka" +// submitCompactionConfig( +// MAX_ROWS_PER_SEGMENT_COMPACTED, +// NO_SKIP_OFFSET, +// null, +// null, +// new UserCompactionTaskTransformConfig(new SelectorDimFilter("page", "Striker Eureka", null)), +// false +// ); +// forceTriggerAutoCompaction(2); +// +// // For dim "page", result should only contain value "Striker Eureka" +// expectedResult = ImmutableMap.of( +// "%%EXPECTED_COUNT_RESULT%%", 1, +// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(459.0)))) +// ); +// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); +// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); +// +// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// // Verify compacted segments does not get compacted again +// forceTriggerAutoCompaction(2); +// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); +// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); +// } +// } @Test - public void testAutoCompactionDutyCanDeleteCompactionConfig() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); - deleteCompactionConfig(); - - // ...should remains unchanged (4 total) - forceTriggerAutoCompaction(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(0, null); - // Auto compaction stats should be deleted as compacation config was deleted - Assert.assertNull(compactionResource.getCompactionStatus(fullDatasourceName)); - checkCompactionIntervals(intervalsBeforeCompaction); - } - } - - @Test - public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception - { - // Set compactionTaskSlotRatio to 0 to prevent any compaction - updateCompactionTaskSlot(0, 0); - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); - // ...should remains unchanged (4 total) - forceTriggerAutoCompaction(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(0, null); - checkCompactionIntervals(intervalsBeforeCompaction); - Assert.assertNull(compactionResource.getCompactionStatus(fullDatasourceName)); - // Update compaction slots to be 1 - updateCompactionTaskSlot(1, 1); - // One day compacted (1 new segment) and one day remains uncompacted. (3 total) - forceTriggerAutoCompaction(3); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); - checkCompactionIntervals(intervalsBeforeCompaction); - getAndAssertCompactionStatus( - fullDatasourceName, - AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, - 14370, - 14369, - 0, - 2, - 2, - 0, - 1, - 1, - 0); - Assert.assertEquals(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize"), "14370"); - // Run compaction again to compact the remaining day - // Remaining day compacted (1 new segment). Now both days compacted (2 total) - forceTriggerAutoCompaction(2); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); - checkCompactionIntervals(intervalsBeforeCompaction); - getAndAssertCompactionStatus( - fullDatasourceName, - AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, - 0, - 22568, - 0, - 0, - 3, - 0, - 0, - 2, - 0); - } - } - - @Test - public void testAutoCompactionDutyWithSegmentGranularityAndWithDropExistingTrue() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - Granularity newGranularity = Granularities.YEAR; - // Set dropExisting to true - submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); - - LOG.info("Auto compaction test with YEAR segment granularity"); - - List expectedIntervalAfterCompaction = new ArrayList<>(); - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - forceTriggerAutoCompaction(1); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(1, 1000); - checkCompactionIntervals(expectedIntervalAfterCompaction); - - newGranularity = Granularities.DAY; - // Set dropExisting to true - submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); - - LOG.info("Auto compaction test with DAY segment granularity"); - - // Since dropExisting is set to true... - // The earlier segment with YEAR granularity will be dropped post-compaction - // Hence, we will only have 2013-08-31 to 2013-09-01 and 2013-09-01 to 2013-09-02. - expectedIntervalAfterCompaction = new ArrayList<>(); - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - forceTriggerAutoCompaction(2); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(2, 1000); - checkCompactionIntervals(expectedIntervalAfterCompaction); - } - } - - @Test - public void testAutoCompactionDutyWithSegmentGranularityAndWithDropExistingFalse() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - Granularity newGranularity = Granularities.YEAR; - // Set dropExisting to false - submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); - - LOG.info("Auto compaction test with YEAR segment granularity"); - - List expectedIntervalAfterCompaction = new ArrayList<>(); - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - forceTriggerAutoCompaction(1); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(1, 1000); - checkCompactionIntervals(expectedIntervalAfterCompaction); - - newGranularity = Granularities.DAY; - // Set dropExisting to false - submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); - - LOG.info("Auto compaction test with DAY segment granularity"); - - // Since dropExisting is set to false... - // The earlier segment with YEAR granularity is still 'used' as it’s not fully overshaowed. - // This is because we only have newer version on 2013-08-31 to 2013-09-01 and 2013-09-01 to 2013-09-02. - // The version for the YEAR segment is still the latest for 2013-01-01 to 2013-08-31 and 2013-09-02 to 2014-01-01. - // Hence, all three segments are available and the expected intervals are combined from the DAY and YEAR segment granularities - // (which are 2013-08-31 to 2013-09-01, 2013-09-01 to 2013-09-02 and 2013-01-01 to 2014-01-01) - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - forceTriggerAutoCompaction(3); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(3, 1000); - checkCompactionIntervals(expectedIntervalAfterCompaction); - } - } - - @Test - public void testAutoCompactionDutyWithSegmentGranularityAndMixedVersion() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, Period.days(1)); - //...compacted into 1 new segment for 1 day. 1 day compacted and 1 day skipped/remains uncompacted. (3 total) - forceTriggerAutoCompaction(3); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); - - Granularity newGranularity = Granularities.YEAR; - submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); - - LOG.info("Auto compaction test with YEAR segment granularity"); - - List expectedIntervalAfterCompaction = new ArrayList<>(); - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - // Since the new segmentGranularity is YEAR, it will have mixed versions inside the same time chunk - // There will be an old version (for the first day interval) from the initial ingestion and - // a newer version (for the second day interval) from the first compaction - forceTriggerAutoCompaction(1); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(1, 1000); - checkCompactionIntervals(expectedIntervalAfterCompaction); - } - } - - @Test - public void testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveSameSegmentGranularity() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - // Compacted without SegmentGranularity in auto compaction config - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); - forceTriggerAutoCompaction(2); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); - - List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); - - // Segments were compacted and already has DAY granularity since it was initially ingested with DAY granularity. - // Now set auto compaction with DAY granularity in the granularitySpec - Granularity newGranularity = Granularities.DAY; - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); - forceTriggerAutoCompaction(2); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); - // should be no new compaction task as segmentGranularity is already DAY - List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); - Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); - } - } - - @Test - public void testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveDifferentSegmentGranularity() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - // Compacted without SegmentGranularity in auto compaction config - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); - forceTriggerAutoCompaction(2); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); - - List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); - - // Segments were compacted and already has DAY granularity since it was initially ingested with DAY granularity. - // Now set auto compaction with DAY granularity in the granularitySpec - Granularity newGranularity = Granularities.YEAR; - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); - forceTriggerAutoCompaction(1); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); - - // There should be new compaction tasks since SegmentGranularity changed from DAY to YEAR - List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); - Assert.assertTrue(compactTasksAfter.size() > compactTasksBefore.size()); - } - } - - @Test - public void testAutoCompactionDutyWithSegmentGranularityAndSmallerSegmentGranularityCoveringMultipleSegmentsInTimelineAndDropExistingTrue() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - Granularity newGranularity = Granularities.YEAR; - // Set dropExisting to true - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); - - List expectedIntervalAfterCompaction = new ArrayList<>(); - // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - forceTriggerAutoCompaction(1); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); - checkCompactionIntervals(expectedIntervalAfterCompaction); - - loadData(INDEX_TASK); - verifySegmentsCount(5); - verifyQuery(INDEX_QUERIES_RESOURCE); - // 5 segments. 1 compacted YEAR segment and 4 newly ingested DAY segments across 2 days - // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from the compaction earlier - // two segments with interval of 2013-08-31/2013-09-01 (newly ingested with DAY) - // and two segments with interval of 2013-09-01/2013-09-02 (newly ingested with DAY) - expectedIntervalAfterCompaction.addAll(intervalsBeforeCompaction); - checkCompactionIntervals(expectedIntervalAfterCompaction); - - newGranularity = Granularities.MONTH; - // Set dropExisting to true - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); - // Since dropExisting is set to true... - // This will submit a single compaction task for interval of 2013-01-01/2014-01-01 with MONTH granularity - expectedIntervalAfterCompaction = new ArrayList<>(); - // The previous segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) will be dropped - // We will only have one segments with interval of 2013-09-01/2013-10-01 (compacted with MONTH) - // and one segments with interval of 2013-10-01/2013-11-01 (compacted with MONTH) - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : Granularities.MONTH.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - forceTriggerAutoCompaction(2); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); - checkCompactionIntervals(expectedIntervalAfterCompaction); - } - } - - @Test - public void testAutoCompactionDutyWithSegmentGranularityAndSmallerSegmentGranularityCoveringMultipleSegmentsInTimelineAndDropExistingFalse() throws Exception - { - loadData(INDEX_TASK); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - verifyQuery(INDEX_QUERIES_RESOURCE); - - Granularity newGranularity = Granularities.YEAR; - // Set dropExisting to false - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); - - List expectedIntervalAfterCompaction = new ArrayList<>(); - // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - forceTriggerAutoCompaction(1); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); - checkCompactionIntervals(expectedIntervalAfterCompaction); - - loadData(INDEX_TASK); - verifySegmentsCount(5); - verifyQuery(INDEX_QUERIES_RESOURCE); - // 5 segments. 1 compacted YEAR segment and 4 newly ingested DAY segments across 2 days - // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from the compaction earlier - // two segments with interval of 2013-08-31/2013-09-01 (newly ingested with DAY) - // and two segments with interval of 2013-09-01/2013-09-02 (newly ingested with DAY) - expectedIntervalAfterCompaction.addAll(intervalsBeforeCompaction); - checkCompactionIntervals(expectedIntervalAfterCompaction); - - newGranularity = Granularities.MONTH; - // Set dropExisting to false - submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); - // Since dropExisting is set to true... - // This will submit a single compaction task for interval of 2013-01-01/2014-01-01 with MONTH granularity - expectedIntervalAfterCompaction = new ArrayList<>(); - // Since dropExisting is set to false... - // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from before the compaction - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : Granularities.YEAR.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - // one segments with interval of 2013-09-01/2013-10-01 (compacted with MONTH) - // and one segments with interval of 2013-10-01/2013-11-01 (compacted with MONTH) - for (String interval : intervalsBeforeCompaction) { - for (Interval newinterval : Granularities.MONTH.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { - expectedIntervalAfterCompaction.add(newinterval.toString()); - } - } - - forceTriggerAutoCompaction(3); - verifyQuery(INDEX_QUERIES_RESOURCE); - verifySegmentsCompacted(3, MAX_ROWS_PER_SEGMENT_COMPACTED); - checkCompactionIntervals(expectedIntervalAfterCompaction); - } - } - - @Test - public void testAutoCompactionDutyWithRollup() throws Exception + public void testAutoCompactionDutyWithOverlappingInterval() throws Exception { final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles")); - Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.DAY, Granularities.DAY, false, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); + Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.WEEK, Granularities.NONE, false, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); + // Create WEEK segment with 2013-08-26 to 2013-09-20 loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs); - try (final Closeable ignored = unloader(fullDatasourceName)) { - Map expectedResult = ImmutableMap.of( - "%%EXPECTED_COUNT_RESULT%%", 2, - "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) - ); - verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); - submitCompactionConfig( - MAX_ROWS_PER_SEGMENT_COMPACTED, - NO_SKIP_OFFSET, - new UserCompactionTaskGranularityConfig(null, null, true), - false - ); - forceTriggerAutoCompaction(2); - expectedResult = ImmutableMap.of( - "%%EXPECTED_COUNT_RESULT%%", 1, - "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) - ); - verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); - - List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); - // Verify rollup segments does not get compacted again - forceTriggerAutoCompaction(2); - List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); - Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); - } - } - - @Test - public void testAutoCompactionDutyWithQueryGranularity() throws Exception - { - final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles")); - Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, true, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); + specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.MONTH, Granularities.NONE, false, ImmutableList.of(new Interval("2013-09-01/2013-09-02", chrono)))); + // Create MONTH segment with 2013-09-01 to 2013-10-01 loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs); - try (final Closeable ignored = unloader(fullDatasourceName)) { - Map expectedResult = ImmutableMap.of( - "%%EXPECTED_COUNT_RESULT%%", 2, - "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) - ); - verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); - submitCompactionConfig( - MAX_ROWS_PER_SEGMENT_COMPACTED, - NO_SKIP_OFFSET, - new UserCompactionTaskGranularityConfig(null, Granularities.DAY, null), - false - ); - forceTriggerAutoCompaction(2); - expectedResult = ImmutableMap.of( - "%%EXPECTED_COUNT_RESULT%%", 1, - "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) - ); - verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); - List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); - // Verify rollup segments does not get compacted again - forceTriggerAutoCompaction(2); - List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); - Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); - } - } - - @Test - public void testAutoCompactionDutyWithDimensionsSpec() throws Exception - { - // Index data with dimensions "page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous", - // "namespace", "continent", "country", "region", "city" - loadData(INDEX_TASK_WITH_DIMENSION_SPEC); try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); + verifySegmentsCount(2); // Result is not rollup + // For dim "page", result has values "Gypsy Danger" and "Striker Eureka" Map expectedResult = ImmutableMap.of( - "%%EXPECTED_COUNT_RESULT%%", 2, - "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) + "%%EXPECTED_COUNT_RESULT%%", + 2, + "%%EXPECTED_SCAN_RESULT%%", + ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) ); verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); - // Compact and change dimension to only "language" submitCompactionConfig( MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, null, - new UserCompactionTaskDimensionsConfig(DimensionsSpec.getDefaultSchemas(ImmutableList.of("language"))), + null, null, false ); + // Compact the MONTH segment forceTriggerAutoCompaction(2); - - // Result should rollup on language dimension - expectedResult = ImmutableMap.of( - "%%EXPECTED_COUNT_RESULT%%", 1, - "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) - ); - verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); - - List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); - // Verify compacted segments does not get compacted again - forceTriggerAutoCompaction(2); - List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); - Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); - } - } - - @Test - public void testAutoCompactionDutyWithFilter() throws Exception - { - loadData(INDEX_TASK_WITH_DIMENSION_SPEC); - try (final Closeable ignored = unloader(fullDatasourceName)) { - final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); - intervalsBeforeCompaction.sort(null); - // 4 segments across 2 days (4 total)... - verifySegmentsCount(4); - - // Result is not rollup - // For dim "page", result has values "Gypsy Danger" and "Striker Eureka" - Map expectedResult = ImmutableMap.of( - "%%EXPECTED_COUNT_RESULT%%", 2, - "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) - ); verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); - // Compact and filter with selector on dim "page" and value "Striker Eureka" - submitCompactionConfig( - MAX_ROWS_PER_SEGMENT_COMPACTED, - NO_SKIP_OFFSET, - null, - null, - new UserCompactionTaskTransformConfig(new SelectorDimFilter("page", "Striker Eureka", null)), - false - ); + // Compact the WEEK segment forceTriggerAutoCompaction(2); - - // For dim "page", result should only contain value "Striker Eureka" - expectedResult = ImmutableMap.of( - "%%EXPECTED_COUNT_RESULT%%", 1, - "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(459.0)))) - ); verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); - verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + // Verify all task succeed List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); + for (TaskResponseObject taskResponseObject : compactTasksBefore) { + Assert.assertEquals(TaskState.SUCCESS, taskResponseObject.getStatus()); + } + // Verify compacted segments does not get compacted again forceTriggerAutoCompaction(2); List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); diff --git a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java index ddec63568341..637aac4fae01 100644 --- a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java +++ b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java @@ -51,7 +51,7 @@ public static ClientCompactionIntervalSpec fromSegments(List segmen JodaUtils.umbrellaInterval( segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()) ), - SegmentUtils.hashIds(segments) + null ); } diff --git a/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java b/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java index b2fed8528e3d..6bd53804f9b4 100644 --- a/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java +++ b/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java @@ -24,8 +24,13 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.discovery.DruidLeaderClient; import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.http.client.Request; import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.NoneShardSpec; +import org.easymock.Capture; import org.easymock.EasyMock; import org.jboss.netty.buffer.BigEndianHeapChannelBuffer; import org.jboss.netty.buffer.ChannelBuffer; @@ -48,6 +53,7 @@ public class HttpIndexingServiceClientTest private HttpIndexingServiceClient httpIndexingServiceClient; private ObjectMapper jsonMapper; private DruidLeaderClient druidLeaderClient; + private ObjectMapper mockMapper; @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -57,6 +63,8 @@ public void setup() { jsonMapper = new DefaultObjectMapper(); druidLeaderClient = EasyMock.createMock(DruidLeaderClient.class); + mockMapper = EasyMock.createMock(ObjectMapper.class); + httpIndexingServiceClient = new HttpIndexingServiceClient( jsonMapper, druidLeaderClient @@ -268,5 +276,69 @@ public void testGetTaskReportEmpty() throws Exception EasyMock.verify(druidLeaderClient, response); } + @Test + public void testCompact() throws Exception + { + DataSegment segment = new DataSegment( + "test", + Intervals.of("2015-04-12/2015-04-13"), + "1", + ImmutableMap.of("bucket", "bucket", "path", "test/2015-04-12T00:00:00.000Z_2015-04-13T00:00:00.000Z/1/0/index.zip"), + null, + null, + NoneShardSpec.instance(), + 0, + 1 + ); + Capture captureTask = EasyMock.newCapture(); + HttpResponse response = EasyMock.createMock(HttpResponse.class); + EasyMock.expect(response.getStatus()).andReturn(HttpResponseStatus.OK).anyTimes(); + EasyMock.expect(response.getContent()).andReturn(new BigEndianHeapChannelBuffer(0)); + EasyMock.replay(response); + + StringFullResponseHolder responseHolder = new StringFullResponseHolder( + response, + StandardCharsets.UTF_8 + ).addChunk(jsonMapper.writeValueAsString(ImmutableMap.of("task", "aaa"))); + + EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.POST, "/druid/indexer/v1/task")) + .andReturn(new Request(HttpMethod.POST, new URL("http://localhost:8090/druid/indexer/v1/task"))) + .anyTimes(); + EasyMock.expect(druidLeaderClient.go(EasyMock.anyObject(Request.class))) + .andReturn(responseHolder) + .anyTimes(); + EasyMock.expect(mockMapper.writeValueAsBytes(EasyMock.capture(captureTask))) + .andReturn(new byte[]{1, 2, 3}) + .anyTimes(); + EasyMock.expect(mockMapper.readValue(EasyMock.anyString(), EasyMock.eq(JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT))) + .andReturn(ImmutableMap.of()) + .anyTimes(); + EasyMock.replay(druidLeaderClient, mockMapper); + + HttpIndexingServiceClient httpIndexingServiceClient = new HttpIndexingServiceClient( + mockMapper, + druidLeaderClient + ); + + try { + httpIndexingServiceClient.compactSegments( + "test-compact", + ImmutableList.of(segment), + 50, + null, + null, + null, + null, + null, + null + ); + } catch (Exception e) { + // Ignore IllegalStateException as taskId is internally generated and returned task id will failed check + Assert.assertEquals(IllegalStateException.class.getName(), e.getCause().getClass().getName()); + } + + ClientCompactionTaskQuery taskQuery = (ClientCompactionTaskQuery) captureTask.getValue(); + Assert.assertNull(taskQuery.getIoConfig().getInputSpec().getSha256OfSortedSegmentIds()); + } } From 62a18065abef9a4fdc6c457b5ef0571a9f1239c7 Mon Sep 17 00:00:00 2001 From: Maytas Monsereenusorn Date: Fri, 10 Dec 2021 10:59:40 -0800 Subject: [PATCH 2/8] add impl --- .../apache/druid/server/coordinator/duty/CompactSegments.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 634caf302a49..4a0ed532b648 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -354,6 +354,7 @@ private CoordinatorStats doRun( ); } else { granularitySpec = null; + xxx } // Create dimensionsSpec to send to compaction task ClientCompactionTaskDimensionsSpec dimensionsSpec; From a0ec4751d2ebbb7d6250880381c8523ed21726e8 Mon Sep 17 00:00:00 2001 From: Maytas Monsereenusorn Date: Mon, 13 Dec 2021 23:25:26 -0800 Subject: [PATCH 3/8] fix more bugs --- .../duty/ITAutoCompactionTest.java | 1344 ++++++++--------- .../coordinator/duty/CompactSegments.java | 38 +- 2 files changed, 701 insertions(+), 681 deletions(-) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java index c217f4646aa0..5d46e58f039d 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java @@ -103,678 +103,678 @@ public void setup() throws Exception fullDatasourceName = "wikipedia_index_test_" + UUID.randomUUID() + config.getExtraDatasourceNameSuffix(); } -// @Test -// public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, Period.days(1)); -// //...compacted into 1 new segment for 1 day. 1 day compacted and 1 day skipped/remains uncompacted. (3 total) -// forceTriggerAutoCompaction(3); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); -// checkCompactionIntervals(intervalsBeforeCompaction); -// getAndAssertCompactionStatus( -// fullDatasourceName, -// AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, -// 0, -// 14370, -// 0, -// 0, -// 2, -// 0, -// 0, -// 1, -// 0); -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); -// //...compacted into 1 new segment for the remaining one day. 2 day compacted and 0 day uncompacted. (2 total) -// forceTriggerAutoCompaction(2); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// checkCompactionIntervals(intervalsBeforeCompaction); -// getAndAssertCompactionStatus( -// fullDatasourceName, -// AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, -// 0, -// 22568, -// 0, -// 0, -// 3, -// 0, -// 0, -// 2, -// 0); -// } -// } -// -// @Test -// public void testAutoCompactionDutyCanUpdateCompactionConfig() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// // Dummy compaction config which will be overwritten -// submitCompactionConfig(10000, NO_SKIP_OFFSET); -// // New compaction config should overwrites the existing compaction config -// submitCompactionConfig(1, NO_SKIP_OFFSET); -// -// LOG.info("Auto compaction test with dynamic partitioning"); -// -// // Instead of merging segments, the updated config will split segments! -// //...compacted into 10 new segments across 2 days. 5 new segments each day (10 total) -// forceTriggerAutoCompaction(10); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(10, 1); -// checkCompactionIntervals(intervalsBeforeCompaction); -// -// LOG.info("Auto compaction test with hash partitioning"); -// -// final HashedPartitionsSpec hashedPartitionsSpec = new HashedPartitionsSpec(null, 3, null); -// submitCompactionConfig(hashedPartitionsSpec, NO_SKIP_OFFSET, 1, null, null, null, false); -// // 2 segments published per day after compaction. -// forceTriggerAutoCompaction(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(hashedPartitionsSpec, 4); -// checkCompactionIntervals(intervalsBeforeCompaction); -// -// LOG.info("Auto compaction test with range partitioning"); -// -// final SingleDimensionPartitionsSpec rangePartitionsSpec = new SingleDimensionPartitionsSpec( -// 5, -// null, -// "city", -// false -// ); -// submitCompactionConfig(rangePartitionsSpec, NO_SKIP_OFFSET, 1, null, null, null, false); -// forceTriggerAutoCompaction(2); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(rangePartitionsSpec, 2); -// checkCompactionIntervals(intervalsBeforeCompaction); -// } -// } -// -// @Test -// public void testAutoCompactionDutyCanDeleteCompactionConfig() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); -// deleteCompactionConfig(); -// -// // ...should remains unchanged (4 total) -// forceTriggerAutoCompaction(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(0, null); -// // Auto compaction stats should be deleted as compacation config was deleted -// Assert.assertNull(compactionResource.getCompactionStatus(fullDatasourceName)); -// checkCompactionIntervals(intervalsBeforeCompaction); -// } -// } -// -// @Test -// public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception -// { -// // Set compactionTaskSlotRatio to 0 to prevent any compaction -// updateCompactionTaskSlot(0, 0); -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); -// // ...should remains unchanged (4 total) -// forceTriggerAutoCompaction(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(0, null); -// checkCompactionIntervals(intervalsBeforeCompaction); -// Assert.assertNull(compactionResource.getCompactionStatus(fullDatasourceName)); -// // Update compaction slots to be 1 -// updateCompactionTaskSlot(1, 1); -// // One day compacted (1 new segment) and one day remains uncompacted. (3 total) -// forceTriggerAutoCompaction(3); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); -// checkCompactionIntervals(intervalsBeforeCompaction); -// getAndAssertCompactionStatus( -// fullDatasourceName, -// AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, -// 14370, -// 14369, -// 0, -// 2, -// 2, -// 0, -// 1, -// 1, -// 0); -// Assert.assertEquals(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize"), "14370"); -// // Run compaction again to compact the remaining day -// // Remaining day compacted (1 new segment). Now both days compacted (2 total) -// forceTriggerAutoCompaction(2); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// checkCompactionIntervals(intervalsBeforeCompaction); -// getAndAssertCompactionStatus( -// fullDatasourceName, -// AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, -// 0, -// 22568, -// 0, -// 0, -// 3, -// 0, -// 0, -// 2, -// 0); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithSegmentGranularityAndWithDropExistingTrue() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// Granularity newGranularity = Granularities.YEAR; -// // Set dropExisting to true -// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); -// -// LOG.info("Auto compaction test with YEAR segment granularity"); -// -// List expectedIntervalAfterCompaction = new ArrayList<>(); -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// forceTriggerAutoCompaction(1); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(1, 1000); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// -// newGranularity = Granularities.DAY; -// // Set dropExisting to true -// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); -// -// LOG.info("Auto compaction test with DAY segment granularity"); -// -// // Since dropExisting is set to true... -// // The earlier segment with YEAR granularity will be dropped post-compaction -// // Hence, we will only have 2013-08-31 to 2013-09-01 and 2013-09-01 to 2013-09-02. -// expectedIntervalAfterCompaction = new ArrayList<>(); -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// forceTriggerAutoCompaction(2); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(2, 1000); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithSegmentGranularityAndWithDropExistingFalse() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// Granularity newGranularity = Granularities.YEAR; -// // Set dropExisting to false -// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); -// -// LOG.info("Auto compaction test with YEAR segment granularity"); -// -// List expectedIntervalAfterCompaction = new ArrayList<>(); -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// forceTriggerAutoCompaction(1); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(1, 1000); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// -// newGranularity = Granularities.DAY; -// // Set dropExisting to false -// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); -// -// LOG.info("Auto compaction test with DAY segment granularity"); -// -// // Since dropExisting is set to false... -// // The earlier segment with YEAR granularity is still 'used' as it’s not fully overshaowed. -// // This is because we only have newer version on 2013-08-31 to 2013-09-01 and 2013-09-01 to 2013-09-02. -// // The version for the YEAR segment is still the latest for 2013-01-01 to 2013-08-31 and 2013-09-02 to 2014-01-01. -// // Hence, all three segments are available and the expected intervals are combined from the DAY and YEAR segment granularities -// // (which are 2013-08-31 to 2013-09-01, 2013-09-01 to 2013-09-02 and 2013-01-01 to 2014-01-01) -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// forceTriggerAutoCompaction(3); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(3, 1000); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithSegmentGranularityAndMixedVersion() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, Period.days(1)); -// //...compacted into 1 new segment for 1 day. 1 day compacted and 1 day skipped/remains uncompacted. (3 total) -// forceTriggerAutoCompaction(3); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); -// -// Granularity newGranularity = Granularities.YEAR; -// submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); -// -// LOG.info("Auto compaction test with YEAR segment granularity"); -// -// List expectedIntervalAfterCompaction = new ArrayList<>(); -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// // Since the new segmentGranularity is YEAR, it will have mixed versions inside the same time chunk -// // There will be an old version (for the first day interval) from the initial ingestion and -// // a newer version (for the second day interval) from the first compaction -// forceTriggerAutoCompaction(1); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(1, 1000); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveSameSegmentGranularity() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// // Compacted without SegmentGranularity in auto compaction config -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); -// forceTriggerAutoCompaction(2); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// -// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// -// // Segments were compacted and already has DAY granularity since it was initially ingested with DAY granularity. -// // Now set auto compaction with DAY granularity in the granularitySpec -// Granularity newGranularity = Granularities.DAY; -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); -// forceTriggerAutoCompaction(2); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// // should be no new compaction task as segmentGranularity is already DAY -// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveDifferentSegmentGranularity() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// // Compacted without SegmentGranularity in auto compaction config -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); -// forceTriggerAutoCompaction(2); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// -// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// -// // Segments were compacted and already has DAY granularity since it was initially ingested with DAY granularity. -// // Now set auto compaction with DAY granularity in the granularitySpec -// Granularity newGranularity = Granularities.YEAR; -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); -// forceTriggerAutoCompaction(1); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); -// -// // There should be new compaction tasks since SegmentGranularity changed from DAY to YEAR -// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// Assert.assertTrue(compactTasksAfter.size() > compactTasksBefore.size()); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithSegmentGranularityAndSmallerSegmentGranularityCoveringMultipleSegmentsInTimelineAndDropExistingTrue() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// Granularity newGranularity = Granularities.YEAR; -// // Set dropExisting to true -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); -// -// List expectedIntervalAfterCompaction = new ArrayList<>(); -// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// forceTriggerAutoCompaction(1); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// -// loadData(INDEX_TASK); -// verifySegmentsCount(5); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// // 5 segments. 1 compacted YEAR segment and 4 newly ingested DAY segments across 2 days -// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from the compaction earlier -// // two segments with interval of 2013-08-31/2013-09-01 (newly ingested with DAY) -// // and two segments with interval of 2013-09-01/2013-09-02 (newly ingested with DAY) -// expectedIntervalAfterCompaction.addAll(intervalsBeforeCompaction); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// -// newGranularity = Granularities.MONTH; -// // Set dropExisting to true -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); -// // Since dropExisting is set to true... -// // This will submit a single compaction task for interval of 2013-01-01/2014-01-01 with MONTH granularity -// expectedIntervalAfterCompaction = new ArrayList<>(); -// // The previous segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) will be dropped -// // We will only have one segments with interval of 2013-09-01/2013-10-01 (compacted with MONTH) -// // and one segments with interval of 2013-10-01/2013-11-01 (compacted with MONTH) -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : Granularities.MONTH.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// forceTriggerAutoCompaction(2); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithSegmentGranularityAndSmallerSegmentGranularityCoveringMultipleSegmentsInTimelineAndDropExistingFalse() throws Exception -// { -// loadData(INDEX_TASK); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// -// Granularity newGranularity = Granularities.YEAR; -// // Set dropExisting to false -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); -// -// List expectedIntervalAfterCompaction = new ArrayList<>(); -// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// forceTriggerAutoCompaction(1); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// -// loadData(INDEX_TASK); -// verifySegmentsCount(5); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// // 5 segments. 1 compacted YEAR segment and 4 newly ingested DAY segments across 2 days -// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from the compaction earlier -// // two segments with interval of 2013-08-31/2013-09-01 (newly ingested with DAY) -// // and two segments with interval of 2013-09-01/2013-09-02 (newly ingested with DAY) -// expectedIntervalAfterCompaction.addAll(intervalsBeforeCompaction); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// -// newGranularity = Granularities.MONTH; -// // Set dropExisting to false -// submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); -// // Since dropExisting is set to true... -// // This will submit a single compaction task for interval of 2013-01-01/2014-01-01 with MONTH granularity -// expectedIntervalAfterCompaction = new ArrayList<>(); -// // Since dropExisting is set to false... -// // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from before the compaction -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : Granularities.YEAR.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// // one segments with interval of 2013-09-01/2013-10-01 (compacted with MONTH) -// // and one segments with interval of 2013-10-01/2013-11-01 (compacted with MONTH) -// for (String interval : intervalsBeforeCompaction) { -// for (Interval newinterval : Granularities.MONTH.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { -// expectedIntervalAfterCompaction.add(newinterval.toString()); -// } -// } -// -// forceTriggerAutoCompaction(3); -// verifyQuery(INDEX_QUERIES_RESOURCE); -// verifySegmentsCompacted(3, MAX_ROWS_PER_SEGMENT_COMPACTED); -// checkCompactionIntervals(expectedIntervalAfterCompaction); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithRollup() throws Exception -// { -// final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles")); -// Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.DAY, Granularities.DAY, false, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); -// loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// Map expectedResult = ImmutableMap.of( -// "%%EXPECTED_COUNT_RESULT%%", 2, -// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) -// ); -// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); -// submitCompactionConfig( -// MAX_ROWS_PER_SEGMENT_COMPACTED, -// NO_SKIP_OFFSET, -// new UserCompactionTaskGranularityConfig(null, null, true), -// false -// ); -// forceTriggerAutoCompaction(2); -// expectedResult = ImmutableMap.of( -// "%%EXPECTED_COUNT_RESULT%%", 1, -// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) -// ); -// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// -// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// // Verify rollup segments does not get compacted again -// forceTriggerAutoCompaction(2); -// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithQueryGranularity() throws Exception -// { -// final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles")); -// Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, true, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); -// loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// Map expectedResult = ImmutableMap.of( -// "%%EXPECTED_COUNT_RESULT%%", 2, -// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) -// ); -// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); -// submitCompactionConfig( -// MAX_ROWS_PER_SEGMENT_COMPACTED, -// NO_SKIP_OFFSET, -// new UserCompactionTaskGranularityConfig(null, Granularities.DAY, null), -// false -// ); -// forceTriggerAutoCompaction(2); -// expectedResult = ImmutableMap.of( -// "%%EXPECTED_COUNT_RESULT%%", 1, -// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) -// ); -// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// -// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// // Verify rollup segments does not get compacted again -// forceTriggerAutoCompaction(2); -// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithDimensionsSpec() throws Exception -// { -// // Index data with dimensions "page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous", -// // "namespace", "continent", "country", "region", "city" -// loadData(INDEX_TASK_WITH_DIMENSION_SPEC); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// -// // Result is not rollup -// Map expectedResult = ImmutableMap.of( -// "%%EXPECTED_COUNT_RESULT%%", 2, -// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) -// ); -// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); -// -// // Compact and change dimension to only "language" -// submitCompactionConfig( -// MAX_ROWS_PER_SEGMENT_COMPACTED, -// NO_SKIP_OFFSET, -// null, -// new UserCompactionTaskDimensionsConfig(DimensionsSpec.getDefaultSchemas(ImmutableList.of("language"))), -// null, -// false -// ); -// forceTriggerAutoCompaction(2); -// -// // Result should rollup on language dimension -// expectedResult = ImmutableMap.of( -// "%%EXPECTED_COUNT_RESULT%%", 1, -// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) -// ); -// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// -// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// // Verify compacted segments does not get compacted again -// forceTriggerAutoCompaction(2); -// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); -// } -// } -// -// @Test -// public void testAutoCompactionDutyWithFilter() throws Exception -// { -// loadData(INDEX_TASK_WITH_DIMENSION_SPEC); -// try (final Closeable ignored = unloader(fullDatasourceName)) { -// final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); -// intervalsBeforeCompaction.sort(null); -// // 4 segments across 2 days (4 total)... -// verifySegmentsCount(4); -// -// // Result is not rollup -// // For dim "page", result has values "Gypsy Danger" and "Striker Eureka" -// Map expectedResult = ImmutableMap.of( -// "%%EXPECTED_COUNT_RESULT%%", 2, -// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) -// ); -// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); -// -// // Compact and filter with selector on dim "page" and value "Striker Eureka" -// submitCompactionConfig( -// MAX_ROWS_PER_SEGMENT_COMPACTED, -// NO_SKIP_OFFSET, -// null, -// null, -// new UserCompactionTaskTransformConfig(new SelectorDimFilter("page", "Striker Eureka", null)), -// false -// ); -// forceTriggerAutoCompaction(2); -// -// // For dim "page", result should only contain value "Striker Eureka" -// expectedResult = ImmutableMap.of( -// "%%EXPECTED_COUNT_RESULT%%", 1, -// "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(459.0)))) -// ); -// verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); -// verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); -// -// List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// // Verify compacted segments does not get compacted again -// forceTriggerAutoCompaction(2); -// List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); -// Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); -// } -// } + @Test + public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, Period.days(1)); + //...compacted into 1 new segment for 1 day. 1 day compacted and 1 day skipped/remains uncompacted. (3 total) + forceTriggerAutoCompaction(3); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); + checkCompactionIntervals(intervalsBeforeCompaction); + getAndAssertCompactionStatus( + fullDatasourceName, + AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, + 0, + 14370, + 0, + 0, + 2, + 0, + 0, + 1, + 0); + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); + //...compacted into 1 new segment for the remaining one day. 2 day compacted and 0 day uncompacted. (2 total) + forceTriggerAutoCompaction(2); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + checkCompactionIntervals(intervalsBeforeCompaction); + getAndAssertCompactionStatus( + fullDatasourceName, + AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, + 0, + 22568, + 0, + 0, + 3, + 0, + 0, + 2, + 0); + } + } + + @Test + public void testAutoCompactionDutyCanUpdateCompactionConfig() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + // Dummy compaction config which will be overwritten + submitCompactionConfig(10000, NO_SKIP_OFFSET); + // New compaction config should overwrites the existing compaction config + submitCompactionConfig(1, NO_SKIP_OFFSET); + + LOG.info("Auto compaction test with dynamic partitioning"); + + // Instead of merging segments, the updated config will split segments! + //...compacted into 10 new segments across 2 days. 5 new segments each day (10 total) + forceTriggerAutoCompaction(10); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(10, 1); + checkCompactionIntervals(intervalsBeforeCompaction); + + LOG.info("Auto compaction test with hash partitioning"); + + final HashedPartitionsSpec hashedPartitionsSpec = new HashedPartitionsSpec(null, 3, null); + submitCompactionConfig(hashedPartitionsSpec, NO_SKIP_OFFSET, 1, null, null, null, false); + // 2 segments published per day after compaction. + forceTriggerAutoCompaction(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(hashedPartitionsSpec, 4); + checkCompactionIntervals(intervalsBeforeCompaction); + + LOG.info("Auto compaction test with range partitioning"); + + final SingleDimensionPartitionsSpec rangePartitionsSpec = new SingleDimensionPartitionsSpec( + 5, + null, + "city", + false + ); + submitCompactionConfig(rangePartitionsSpec, NO_SKIP_OFFSET, 1, null, null, null, false); + forceTriggerAutoCompaction(2); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(rangePartitionsSpec, 2); + checkCompactionIntervals(intervalsBeforeCompaction); + } + } + + @Test + public void testAutoCompactionDutyCanDeleteCompactionConfig() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); + deleteCompactionConfig(); + + // ...should remains unchanged (4 total) + forceTriggerAutoCompaction(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(0, null); + // Auto compaction stats should be deleted as compacation config was deleted + Assert.assertNull(compactionResource.getCompactionStatus(fullDatasourceName)); + checkCompactionIntervals(intervalsBeforeCompaction); + } + } + + @Test + public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception + { + // Set compactionTaskSlotRatio to 0 to prevent any compaction + updateCompactionTaskSlot(0, 0); + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); + // ...should remains unchanged (4 total) + forceTriggerAutoCompaction(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(0, null); + checkCompactionIntervals(intervalsBeforeCompaction); + Assert.assertNull(compactionResource.getCompactionStatus(fullDatasourceName)); + // Update compaction slots to be 1 + updateCompactionTaskSlot(1, 1); + // One day compacted (1 new segment) and one day remains uncompacted. (3 total) + forceTriggerAutoCompaction(3); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); + checkCompactionIntervals(intervalsBeforeCompaction); + getAndAssertCompactionStatus( + fullDatasourceName, + AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, + 14370, + 14369, + 0, + 2, + 2, + 0, + 1, + 1, + 0); + Assert.assertEquals(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize"), "14370"); + // Run compaction again to compact the remaining day + // Remaining day compacted (1 new segment). Now both days compacted (2 total) + forceTriggerAutoCompaction(2); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + checkCompactionIntervals(intervalsBeforeCompaction); + getAndAssertCompactionStatus( + fullDatasourceName, + AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, + 0, + 22568, + 0, + 0, + 3, + 0, + 0, + 2, + 0); + } + } + + @Test + public void testAutoCompactionDutyWithSegmentGranularityAndWithDropExistingTrue() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + Granularity newGranularity = Granularities.YEAR; + // Set dropExisting to true + submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); + + LOG.info("Auto compaction test with YEAR segment granularity"); + + List expectedIntervalAfterCompaction = new ArrayList<>(); + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + forceTriggerAutoCompaction(1); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(1, 1000); + checkCompactionIntervals(expectedIntervalAfterCompaction); + + newGranularity = Granularities.DAY; + // Set dropExisting to true + submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); + + LOG.info("Auto compaction test with DAY segment granularity"); + + // Since dropExisting is set to true... + // The earlier segment with YEAR granularity will be dropped post-compaction + // Hence, we will only have 2013-08-31 to 2013-09-01 and 2013-09-01 to 2013-09-02. + expectedIntervalAfterCompaction = new ArrayList<>(); + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + forceTriggerAutoCompaction(2); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(2, 1000); + checkCompactionIntervals(expectedIntervalAfterCompaction); + } + } + + @Test + public void testAutoCompactionDutyWithSegmentGranularityAndWithDropExistingFalse() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + Granularity newGranularity = Granularities.YEAR; + // Set dropExisting to false + submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); + + LOG.info("Auto compaction test with YEAR segment granularity"); + + List expectedIntervalAfterCompaction = new ArrayList<>(); + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + forceTriggerAutoCompaction(1); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(1, 1000); + checkCompactionIntervals(expectedIntervalAfterCompaction); + + newGranularity = Granularities.DAY; + // Set dropExisting to false + submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); + + LOG.info("Auto compaction test with DAY segment granularity"); + + // Since dropExisting is set to false... + // The earlier segment with YEAR granularity is still 'used' as it’s not fully overshaowed. + // This is because we only have newer version on 2013-08-31 to 2013-09-01 and 2013-09-01 to 2013-09-02. + // The version for the YEAR segment is still the latest for 2013-01-01 to 2013-08-31 and 2013-09-02 to 2014-01-01. + // Hence, all three segments are available and the expected intervals are combined from the DAY and YEAR segment granularities + // (which are 2013-08-31 to 2013-09-01, 2013-09-01 to 2013-09-02 and 2013-01-01 to 2014-01-01) + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + forceTriggerAutoCompaction(3); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(3, 1000); + checkCompactionIntervals(expectedIntervalAfterCompaction); + } + } + + @Test + public void testAutoCompactionDutyWithSegmentGranularityAndMixedVersion() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, Period.days(1)); + //...compacted into 1 new segment for 1 day. 1 day compacted and 1 day skipped/remains uncompacted. (3 total) + forceTriggerAutoCompaction(3); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); + + Granularity newGranularity = Granularities.YEAR; + submitCompactionConfig(1000, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); + + LOG.info("Auto compaction test with YEAR segment granularity"); + + List expectedIntervalAfterCompaction = new ArrayList<>(); + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + // Since the new segmentGranularity is YEAR, it will have mixed versions inside the same time chunk + // There will be an old version (for the first day interval) from the initial ingestion and + // a newer version (for the second day interval) from the first compaction + forceTriggerAutoCompaction(1); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(1, 1000); + checkCompactionIntervals(expectedIntervalAfterCompaction); + } + } + + @Test + public void testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveSameSegmentGranularity() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + // Compacted without SegmentGranularity in auto compaction config + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); + forceTriggerAutoCompaction(2); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + + List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); + + // Segments were compacted and already has DAY granularity since it was initially ingested with DAY granularity. + // Now set auto compaction with DAY granularity in the granularitySpec + Granularity newGranularity = Granularities.DAY; + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); + forceTriggerAutoCompaction(2); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + // should be no new compaction task as segmentGranularity is already DAY + List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); + Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); + } + } + + @Test + public void testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveDifferentSegmentGranularity() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + // Compacted without SegmentGranularity in auto compaction config + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET); + forceTriggerAutoCompaction(2); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + + List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); + + // Segments were compacted and already has DAY granularity since it was initially ingested with DAY granularity. + // Now set auto compaction with DAY granularity in the granularitySpec + Granularity newGranularity = Granularities.YEAR; + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null)); + forceTriggerAutoCompaction(1); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); + + // There should be new compaction tasks since SegmentGranularity changed from DAY to YEAR + List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); + Assert.assertTrue(compactTasksAfter.size() > compactTasksBefore.size()); + } + } + + @Test + public void testAutoCompactionDutyWithSegmentGranularityAndSmallerSegmentGranularityCoveringMultipleSegmentsInTimelineAndDropExistingTrue() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + Granularity newGranularity = Granularities.YEAR; + // Set dropExisting to true + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); + + List expectedIntervalAfterCompaction = new ArrayList<>(); + // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + forceTriggerAutoCompaction(1); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); + checkCompactionIntervals(expectedIntervalAfterCompaction); + + loadData(INDEX_TASK); + verifySegmentsCount(5); + verifyQuery(INDEX_QUERIES_RESOURCE); + // 5 segments. 1 compacted YEAR segment and 4 newly ingested DAY segments across 2 days + // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from the compaction earlier + // two segments with interval of 2013-08-31/2013-09-01 (newly ingested with DAY) + // and two segments with interval of 2013-09-01/2013-09-02 (newly ingested with DAY) + expectedIntervalAfterCompaction.addAll(intervalsBeforeCompaction); + checkCompactionIntervals(expectedIntervalAfterCompaction); + + newGranularity = Granularities.MONTH; + // Set dropExisting to true + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), true); + // Since dropExisting is set to true... + // This will submit a single compaction task for interval of 2013-01-01/2014-01-01 with MONTH granularity + expectedIntervalAfterCompaction = new ArrayList<>(); + // The previous segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) will be dropped + // We will only have one segments with interval of 2013-09-01/2013-10-01 (compacted with MONTH) + // and one segments with interval of 2013-10-01/2013-11-01 (compacted with MONTH) + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : Granularities.MONTH.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + forceTriggerAutoCompaction(2); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + checkCompactionIntervals(expectedIntervalAfterCompaction); + } + } + + @Test + public void testAutoCompactionDutyWithSegmentGranularityAndSmallerSegmentGranularityCoveringMultipleSegmentsInTimelineAndDropExistingFalse() throws Exception + { + loadData(INDEX_TASK); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + verifyQuery(INDEX_QUERIES_RESOURCE); + + Granularity newGranularity = Granularities.YEAR; + // Set dropExisting to false + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); + + List expectedIntervalAfterCompaction = new ArrayList<>(); + // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : newGranularity.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + forceTriggerAutoCompaction(1); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(1, MAX_ROWS_PER_SEGMENT_COMPACTED); + checkCompactionIntervals(expectedIntervalAfterCompaction); + + loadData(INDEX_TASK); + verifySegmentsCount(5); + verifyQuery(INDEX_QUERIES_RESOURCE); + // 5 segments. 1 compacted YEAR segment and 4 newly ingested DAY segments across 2 days + // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from the compaction earlier + // two segments with interval of 2013-08-31/2013-09-01 (newly ingested with DAY) + // and two segments with interval of 2013-09-01/2013-09-02 (newly ingested with DAY) + expectedIntervalAfterCompaction.addAll(intervalsBeforeCompaction); + checkCompactionIntervals(expectedIntervalAfterCompaction); + + newGranularity = Granularities.MONTH; + // Set dropExisting to false + submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null), false); + // Since dropExisting is set to true... + // This will submit a single compaction task for interval of 2013-01-01/2014-01-01 with MONTH granularity + expectedIntervalAfterCompaction = new ArrayList<>(); + // Since dropExisting is set to false... + // We wil have one segment with interval of 2013-01-01/2014-01-01 (compacted with YEAR) from before the compaction + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : Granularities.YEAR.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + // one segments with interval of 2013-09-01/2013-10-01 (compacted with MONTH) + // and one segments with interval of 2013-10-01/2013-11-01 (compacted with MONTH) + for (String interval : intervalsBeforeCompaction) { + for (Interval newinterval : Granularities.MONTH.getIterable(new Interval(interval, ISOChronology.getInstanceUTC()))) { + expectedIntervalAfterCompaction.add(newinterval.toString()); + } + } + + forceTriggerAutoCompaction(3); + verifyQuery(INDEX_QUERIES_RESOURCE); + verifySegmentsCompacted(3, MAX_ROWS_PER_SEGMENT_COMPACTED); + checkCompactionIntervals(expectedIntervalAfterCompaction); + } + } + + @Test + public void testAutoCompactionDutyWithRollup() throws Exception + { + final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles")); + Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.DAY, Granularities.DAY, false, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); + loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs); + try (final Closeable ignored = unloader(fullDatasourceName)) { + Map expectedResult = ImmutableMap.of( + "%%EXPECTED_COUNT_RESULT%%", 2, + "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) + ); + verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); + submitCompactionConfig( + MAX_ROWS_PER_SEGMENT_COMPACTED, + NO_SKIP_OFFSET, + new UserCompactionTaskGranularityConfig(null, null, true), + false + ); + forceTriggerAutoCompaction(2); + expectedResult = ImmutableMap.of( + "%%EXPECTED_COUNT_RESULT%%", 1, + "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) + ); + verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + + List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); + // Verify rollup segments does not get compacted again + forceTriggerAutoCompaction(2); + List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); + Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); + } + } + + @Test + public void testAutoCompactionDutyWithQueryGranularity() throws Exception + { + final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles")); + Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, true, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); + loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs); + try (final Closeable ignored = unloader(fullDatasourceName)) { + Map expectedResult = ImmutableMap.of( + "%%EXPECTED_COUNT_RESULT%%", 2, + "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) + ); + verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); + submitCompactionConfig( + MAX_ROWS_PER_SEGMENT_COMPACTED, + NO_SKIP_OFFSET, + new UserCompactionTaskGranularityConfig(null, Granularities.DAY, null), + false + ); + forceTriggerAutoCompaction(2); + expectedResult = ImmutableMap.of( + "%%EXPECTED_COUNT_RESULT%%", 1, + "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) + ); + verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + + List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); + // Verify rollup segments does not get compacted again + forceTriggerAutoCompaction(2); + List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); + Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); + } + } + + @Test + public void testAutoCompactionDutyWithDimensionsSpec() throws Exception + { + // Index data with dimensions "page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous", + // "namespace", "continent", "country", "region", "city" + loadData(INDEX_TASK_WITH_DIMENSION_SPEC); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + + // Result is not rollup + Map expectedResult = ImmutableMap.of( + "%%EXPECTED_COUNT_RESULT%%", 2, + "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) + ); + verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); + + // Compact and change dimension to only "language" + submitCompactionConfig( + MAX_ROWS_PER_SEGMENT_COMPACTED, + NO_SKIP_OFFSET, + null, + new UserCompactionTaskDimensionsConfig(DimensionsSpec.getDefaultSchemas(ImmutableList.of("language"))), + null, + false + ); + forceTriggerAutoCompaction(2); + + // Result should rollup on language dimension + expectedResult = ImmutableMap.of( + "%%EXPECTED_COUNT_RESULT%%", 1, + "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))) + ); + verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + + List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); + // Verify compacted segments does not get compacted again + forceTriggerAutoCompaction(2); + List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); + Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); + } + } + + @Test + public void testAutoCompactionDutyWithFilter() throws Exception + { + loadData(INDEX_TASK_WITH_DIMENSION_SPEC); + try (final Closeable ignored = unloader(fullDatasourceName)) { + final List intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName); + intervalsBeforeCompaction.sort(null); + // 4 segments across 2 days (4 total)... + verifySegmentsCount(4); + + // Result is not rollup + // For dim "page", result has values "Gypsy Danger" and "Striker Eureka" + Map expectedResult = ImmutableMap.of( + "%%EXPECTED_COUNT_RESULT%%", 2, + "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))) + ); + verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); + + // Compact and filter with selector on dim "page" and value "Striker Eureka" + submitCompactionConfig( + MAX_ROWS_PER_SEGMENT_COMPACTED, + NO_SKIP_OFFSET, + null, + null, + new UserCompactionTaskTransformConfig(new SelectorDimFilter("page", "Striker Eureka", null)), + false + ); + forceTriggerAutoCompaction(2); + + // For dim "page", result should only contain value "Striker Eureka" + expectedResult = ImmutableMap.of( + "%%EXPECTED_COUNT_RESULT%%", 1, + "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(459.0)))) + ); + verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult); + verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED); + + List compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName); + // Verify compacted segments does not get compacted again + forceTriggerAutoCompaction(2); + List compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName); + Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size()); + } + } @Test public void testAutoCompactionDutyWithOverlappingInterval() throws Exception diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 4a0ed532b648..8320e9c45c29 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -32,7 +32,9 @@ import org.apache.druid.indexer.TaskStatusPlus; import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.java.util.common.granularity.GranularityType; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.coordinator.AutoCompactionSnapshot; import org.apache.druid.server.coordinator.CompactionStatistics; @@ -343,19 +345,37 @@ private CoordinatorStats doRun( snapshotBuilder.incrementSegmentCountCompacted(segmentsToCompact.size()); final DataSourceCompactionConfig config = compactionConfigs.get(dataSourceName); + // Create granularitySpec to send to compaction task ClientCompactionTaskGranularitySpec granularitySpec; - if (config.getGranularitySpec() != null) { - granularitySpec = new ClientCompactionTaskGranularitySpec( - config.getGranularitySpec().getSegmentGranularity(), - config.getGranularitySpec().getQueryGranularity(), - config.getGranularitySpec().isRollup() - - ); + Granularity segmentGranularityToUse = null; + if (config.getGranularitySpec() == null ||config.getGranularitySpec().getSegmentGranularity() == null) { + // Determines segmentGranularity from the segmentsToCompact + boolean allSegmentsOverlapped = true; + Interval union = null; + for (DataSegment segment : segmentsToCompact) { + if (union == null) { + union = segment.getInterval(); + } else if (union.overlaps(segment.getInterval())) { + union = Intervals.utc(union.getStartMillis(), Math.max(union.getEndMillis(), segment.getInterval().getEndMillis())); + } else { + allSegmentsOverlapped = false; + break; + } + } + if (allSegmentsOverlapped && union != null) { + segmentGranularityToUse = GranularityType.fromPeriod(union.toPeriod()).getDefaultGranularity(); + } } else { - granularitySpec = null; - xxx + segmentGranularityToUse = config.getGranularitySpec().getSegmentGranularity(); } + granularitySpec = new ClientCompactionTaskGranularitySpec( + segmentGranularityToUse, + config.getGranularitySpec() != null ? config.getGranularitySpec().getQueryGranularity() : null, + config.getGranularitySpec() != null ? config.getGranularitySpec().isRollup() : null + + ); + // Create dimensionsSpec to send to compaction task ClientCompactionTaskDimensionsSpec dimensionsSpec; if (config.getDimensionsSpec() != null) { From ebd3ea2e07dd69cc0d27195c14185d8eaa4423af Mon Sep 17 00:00:00 2001 From: Maytas Monsereenusorn Date: Tue, 14 Dec 2021 00:50:08 -0800 Subject: [PATCH 4/8] add tests --- .../coordinator/duty/CompactSegments.java | 14 +- .../coordinator/duty/CompactSegmentsTest.java | 192 +++++++++++++++++- 2 files changed, 204 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 8320e9c45c29..8d308ab217ac 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -31,6 +31,7 @@ import org.apache.druid.client.indexing.TaskPayloadResponse; import org.apache.druid.indexer.TaskStatusPlus; import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; +import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularity; @@ -351,6 +352,7 @@ private CoordinatorStats doRun( Granularity segmentGranularityToUse = null; if (config.getGranularitySpec() == null ||config.getGranularitySpec().getSegmentGranularity() == null) { // Determines segmentGranularity from the segmentsToCompact + // Each batch of segmentToCompact from CompactionSegmentIterator will contains a single time chunk boolean allSegmentsOverlapped = true; Interval union = null; for (DataSegment segment : segmentsToCompact) { @@ -359,12 +361,22 @@ private CoordinatorStats doRun( } else if (union.overlaps(segment.getInterval())) { union = Intervals.utc(union.getStartMillis(), Math.max(union.getEndMillis(), segment.getInterval().getEndMillis())); } else { + // Each batch of segmentsToCompact for auto compaction should only contains one time chuck. + // Hence, all segments should overlap as they are from the same time chuck. + // This check is in case the above behavior changes and to fall back to setting segmentGranularity as null + // which will determine segmentGranularity later when the compaction task is run allSegmentsOverlapped = false; break; } } if (allSegmentsOverlapped && union != null) { - segmentGranularityToUse = GranularityType.fromPeriod(union.toPeriod()).getDefaultGranularity(); + try { + segmentGranularityToUse = GranularityType.fromPeriod(union.toPeriod()).getDefaultGranularity(); + } catch (IAE iae) { + // This case can happen if the existing segment interval result in complicated periods. + // Fall back to setting segmentGranularity as null + LOG.warn("Cannot determine segmentGranularity from interval [%s]", union); + } } } else { segmentGranularityToUse = config.getGranularitySpec().getSegmentGranularity(); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index 68b9eb2ca736..8fb46a442ffc 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -702,7 +702,9 @@ public void testCompactWithoutGranularitySpec() ); // Only the same amount of segments as the original PARTITION_PER_TIME_INTERVAL since segment granulartity is the same Assert.assertEquals(PARTITION_PER_TIME_INTERVAL, segmentsCaptor.getValue().size()); - Assert.assertNull(granularitySpecArgumentCaptor.getValue()); + Assert.assertNull(granularitySpecArgumentCaptor.getValue().getSegmentGranularity()); + Assert.assertNull(granularitySpecArgumentCaptor.getValue().getQueryGranularity()); + Assert.assertNull(granularitySpecArgumentCaptor.getValue().isRollup()); } @Test @@ -1390,6 +1392,194 @@ public void testRunWithLockedIntervalsNoSkip() ); } + @Test + public void testDetermineSegmentGranularityFromSegmentsToCompact() + { + String dataSourceName = DATA_SOURCE_PREFIX + 1; + List segments = new ArrayList<>(); + segments.add( + new DataSegment( + dataSourceName, + Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), + "1", + null, + ImmutableList.of(), + ImmutableList.of(), + shardSpecFactory.apply(0, 2), + 0, + 10L + ) + ); + segments.add( + new DataSegment( + dataSourceName, + Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), + "1", + null, + ImmutableList.of(), + ImmutableList.of(), + shardSpecFactory.apply(1, 2), + 0, + 10L + ) + ); + dataSources = DataSourcesSnapshot + .fromUsedSegments(segments, ImmutableMap.of()) + .getUsedSegmentsTimelinesPerDataSource(); + + + final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class); + final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient); + final List compactionConfigs = new ArrayList<>(); + compactionConfigs.add( + new DataSourceCompactionConfig( + dataSourceName, + 0, + 500L, + null, + new Period("PT0H"), // smaller than segment interval + new UserCompactionTaskQueryTuningConfig( + null, + null, + null, + null, + partitionsSpec, + null, + null, + null, + null, + null, + 3, + null, + null, + null, + null, + null, + null + ), + null, + null, + null, + null, + null + ) + ); + doCompactSegments(compactSegments, compactionConfigs); + ArgumentCaptor> segmentsCaptor = ArgumentCaptor.forClass(List.class); + ArgumentCaptor granularitySpecArgumentCaptor = ArgumentCaptor.forClass( + ClientCompactionTaskGranularitySpec.class); + Mockito.verify(mockIndexingServiceClient).compactSegments( + ArgumentMatchers.anyString(), + segmentsCaptor.capture(), + ArgumentMatchers.anyInt(), + ArgumentMatchers.any(), + granularitySpecArgumentCaptor.capture(), + ArgumentMatchers.any(), + ArgumentMatchers.any(), + ArgumentMatchers.any(), + ArgumentMatchers.any() + ); + Assert.assertEquals(2, segmentsCaptor.getValue().size()); + ClientCompactionTaskGranularitySpec actual = granularitySpecArgumentCaptor.getValue(); + Assert.assertNotNull(actual); + ClientCompactionTaskGranularitySpec expected = new ClientCompactionTaskGranularitySpec(Granularities.DAY, null, null); + Assert.assertEquals(expected, actual); + } + + @Test + public void testDetermineSegmentGranularityFromSegmentGranularityInCompactionConfig() + { + String dataSourceName = DATA_SOURCE_PREFIX + 1; + List segments = new ArrayList<>(); + segments.add( + new DataSegment( + dataSourceName, + Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), + "1", + null, + ImmutableList.of(), + ImmutableList.of(), + shardSpecFactory.apply(0, 2), + 0, + 10L + ) + ); + segments.add( + new DataSegment( + dataSourceName, + Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), + "1", + null, + ImmutableList.of(), + ImmutableList.of(), + shardSpecFactory.apply(1, 2), + 0, + 10L + ) + ); + dataSources = DataSourcesSnapshot + .fromUsedSegments(segments, ImmutableMap.of()) + .getUsedSegmentsTimelinesPerDataSource(); + + + final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class); + final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient); + final List compactionConfigs = new ArrayList<>(); + compactionConfigs.add( + new DataSourceCompactionConfig( + dataSourceName, + 0, + 500L, + null, + new Period("PT0H"), // smaller than segment interval + new UserCompactionTaskQueryTuningConfig( + null, + null, + null, + null, + partitionsSpec, + null, + null, + null, + null, + null, + 3, + null, + null, + null, + null, + null, + null + ), + new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null), + null, + null, + null, + null + ) + ); + doCompactSegments(compactSegments, compactionConfigs); + ArgumentCaptor> segmentsCaptor = ArgumentCaptor.forClass(List.class); + ArgumentCaptor granularitySpecArgumentCaptor = ArgumentCaptor.forClass( + ClientCompactionTaskGranularitySpec.class); + Mockito.verify(mockIndexingServiceClient).compactSegments( + ArgumentMatchers.anyString(), + segmentsCaptor.capture(), + ArgumentMatchers.anyInt(), + ArgumentMatchers.any(), + granularitySpecArgumentCaptor.capture(), + ArgumentMatchers.any(), + ArgumentMatchers.any(), + ArgumentMatchers.any(), + ArgumentMatchers.any() + ); + Assert.assertEquals(2, segmentsCaptor.getValue().size()); + ClientCompactionTaskGranularitySpec actual = granularitySpecArgumentCaptor.getValue(); + Assert.assertNotNull(actual); + ClientCompactionTaskGranularitySpec expected = new ClientCompactionTaskGranularitySpec(Granularities.YEAR, null, null); + Assert.assertEquals(expected, actual); + } + private void verifySnapshot( CompactSegments compactSegments, AutoCompactionSnapshot.AutoCompactionScheduleStatus scheduleStatus, From c787990d14f14fd138bc3d2b510dd23546f34fd4 Mon Sep 17 00:00:00 2001 From: Maytas Monsereenusorn Date: Tue, 14 Dec 2021 00:54:10 -0800 Subject: [PATCH 5/8] fix checkstyle --- .../druid/client/indexing/ClientCompactionIntervalSpec.java | 1 - .../druid/server/coordinator/duty/CompactSegments.java | 5 +++-- .../druid/client/indexing/HttpIndexingServiceClientTest.java | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java index 637aac4fae01..6aaa197a63bf 100644 --- a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java +++ b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.JodaUtils; -import org.apache.druid.segment.SegmentUtils; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 8d308ab217ac..398c98b556d7 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -350,7 +350,7 @@ private CoordinatorStats doRun( // Create granularitySpec to send to compaction task ClientCompactionTaskGranularitySpec granularitySpec; Granularity segmentGranularityToUse = null; - if (config.getGranularitySpec() == null ||config.getGranularitySpec().getSegmentGranularity() == null) { + if (config.getGranularitySpec() == null || config.getGranularitySpec().getSegmentGranularity() == null) { // Determines segmentGranularity from the segmentsToCompact // Each batch of segmentToCompact from CompactionSegmentIterator will contains a single time chunk boolean allSegmentsOverlapped = true; @@ -372,7 +372,8 @@ private CoordinatorStats doRun( if (allSegmentsOverlapped && union != null) { try { segmentGranularityToUse = GranularityType.fromPeriod(union.toPeriod()).getDefaultGranularity(); - } catch (IAE iae) { + } + catch (IAE iae) { // This case can happen if the existing segment interval result in complicated periods. // Fall back to setting segmentGranularity as null LOG.warn("Cannot determine segmentGranularity from interval [%s]", union); diff --git a/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java b/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java index 6bd53804f9b4..cf1a84f7487b 100644 --- a/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java +++ b/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java @@ -315,7 +315,7 @@ public void testCompact() throws Exception .anyTimes(); EasyMock.replay(druidLeaderClient, mockMapper); - HttpIndexingServiceClient httpIndexingServiceClient = new HttpIndexingServiceClient( + HttpIndexingServiceClient httpIndexingServiceClient = new HttpIndexingServiceClient( mockMapper, druidLeaderClient ); @@ -332,7 +332,8 @@ public void testCompact() throws Exception null, null ); - } catch (Exception e) { + } + catch (Exception e) { // Ignore IllegalStateException as taskId is internally generated and returned task id will failed check Assert.assertEquals(IllegalStateException.class.getName(), e.getCause().getClass().getName()); } From c299c6617938314249c8160c0c7e3196e63cd402 Mon Sep 17 00:00:00 2001 From: Maytas Monsereenusorn Date: Mon, 3 Jan 2022 14:46:45 -0800 Subject: [PATCH 6/8] address comments --- .../duty/ITAutoCompactionTest.java | 2 +- .../coordinator/duty/CompactSegments.java | 33 ++++--------------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java index 5d46e58f039d..a3fa80f86d04 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java @@ -781,7 +781,7 @@ public void testAutoCompactionDutyWithOverlappingInterval() throws Exception { final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles")); Map specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.WEEK, Granularities.NONE, false, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono)))); - // Create WEEK segment with 2013-08-26 to 2013-09-20 + // Create WEEK segment with 2013-08-26 to 2013-09-02 loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs); specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.MONTH, Granularities.NONE, false, ImmutableList.of(new Interval("2013-09-01/2013-09-02", chrono)))); // Create MONTH segment with 2013-09-01 to 2013-10-01 diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 398c98b556d7..10a1d055e440 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -352,32 +352,13 @@ private CoordinatorStats doRun( Granularity segmentGranularityToUse = null; if (config.getGranularitySpec() == null || config.getGranularitySpec().getSegmentGranularity() == null) { // Determines segmentGranularity from the segmentsToCompact - // Each batch of segmentToCompact from CompactionSegmentIterator will contains a single time chunk - boolean allSegmentsOverlapped = true; - Interval union = null; - for (DataSegment segment : segmentsToCompact) { - if (union == null) { - union = segment.getInterval(); - } else if (union.overlaps(segment.getInterval())) { - union = Intervals.utc(union.getStartMillis(), Math.max(union.getEndMillis(), segment.getInterval().getEndMillis())); - } else { - // Each batch of segmentsToCompact for auto compaction should only contains one time chuck. - // Hence, all segments should overlap as they are from the same time chuck. - // This check is in case the above behavior changes and to fall back to setting segmentGranularity as null - // which will determine segmentGranularity later when the compaction task is run - allSegmentsOverlapped = false; - break; - } - } - if (allSegmentsOverlapped && union != null) { - try { - segmentGranularityToUse = GranularityType.fromPeriod(union.toPeriod()).getDefaultGranularity(); - } - catch (IAE iae) { - // This case can happen if the existing segment interval result in complicated periods. - // Fall back to setting segmentGranularity as null - LOG.warn("Cannot determine segmentGranularity from interval [%s]", union); - } + // Each batch of segmentToCompact from CompactionSegmentIterator will contains the same interval as + // segmentGranularity is not set in the compaction config + Interval interval = segmentsToCompact.get(0).getInterval(); + if (segmentsToCompact.stream().allMatch(segment -> interval.overlaps(segment.getInterval()))) { + segmentGranularityToUse = GranularityType.fromPeriod(interval.toPeriod()).getDefaultGranularity(); + } else { + LOG.warn("segmentsToCompact does not have the same interval. Fallback to not setting segmentGranularity"); } } else { segmentGranularityToUse = config.getGranularitySpec().getSegmentGranularity(); From dc7004f871642acbab24b0b5449ff03b76665af0 Mon Sep 17 00:00:00 2001 From: Maytas Monsereenusorn Date: Mon, 3 Jan 2022 14:52:59 -0800 Subject: [PATCH 7/8] address comments --- .../apache/druid/server/coordinator/duty/CompactSegments.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 10a1d055e440..224e32af1407 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -31,9 +31,7 @@ import org.apache.druid.client.indexing.TaskPayloadResponse; import org.apache.druid.indexer.TaskStatusPlus; import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; -import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.granularity.GranularityType; import org.apache.druid.java.util.common.logger.Logger; From adf6c98a2a54e1fc1752ac5dea0b31965f98e2fa Mon Sep 17 00:00:00 2001 From: Maytas Monsereenusorn Date: Mon, 3 Jan 2022 21:51:50 -0800 Subject: [PATCH 8/8] fix test --- .../server/coordinator/duty/CompactSegments.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 224e32af1407..f5c8b3a4250b 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -31,6 +31,7 @@ import org.apache.druid.client.indexing.TaskPayloadResponse; import org.apache.druid.indexer.TaskStatusPlus; import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; +import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.granularity.GranularityType; @@ -354,9 +355,16 @@ private CoordinatorStats doRun( // segmentGranularity is not set in the compaction config Interval interval = segmentsToCompact.get(0).getInterval(); if (segmentsToCompact.stream().allMatch(segment -> interval.overlaps(segment.getInterval()))) { - segmentGranularityToUse = GranularityType.fromPeriod(interval.toPeriod()).getDefaultGranularity(); + try { + segmentGranularityToUse = GranularityType.fromPeriod(interval.toPeriod()).getDefaultGranularity(); + } + catch (IAE iae) { + // This case can happen if the existing segment interval result in complicated periods. + // Fall back to setting segmentGranularity as null + LOG.warn("Cannot determine segmentGranularity from interval [%s]", interval); + } } else { - LOG.warn("segmentsToCompact does not have the same interval. Fallback to not setting segmentGranularity"); + LOG.warn("segmentsToCompact does not have the same interval. Fallback to not setting segmentGranularity for auto compaction task"); } } else { segmentGranularityToUse = config.getGranularitySpec().getSegmentGranularity();