diff --git a/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java b/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java index c9c4599fad76..091cfcc8b9e9 100644 --- a/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java @@ -21,7 +21,10 @@ import com.google.common.collect.ImmutableList; import org.apache.druid.client.DataSourcesSnapshot; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.segment.metadata.DefaultIndexingStateFingerprintMapper; +import org.apache.druid.segment.metadata.NoopIndexingStateCache; import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy; import org.apache.druid.server.compaction.CompactionSegmentIterator; import org.apache.druid.server.compaction.NewestSegmentFirstPolicy; @@ -135,7 +138,8 @@ public void measureNewestSegmentFirstPolicy(Blackhole blackhole) policy, compactionConfigs, dataSources, - Collections.emptyMap() + Collections.emptyMap(), + new DefaultIndexingStateFingerprintMapper(new NoopIndexingStateCache(), new DefaultObjectMapper()) ); for (int i = 0; i < numCompactionTaskSlots && iterator.hasNext(); i++) { blackhole.consume(iterator.next()); diff --git a/docs/api-reference/automatic-compaction-api.md b/docs/api-reference/automatic-compaction-api.md index f3744a45f02b..6864aae4735c 100644 --- a/docs/api-reference/automatic-compaction-api.md +++ b/docs/api-reference/automatic-compaction-api.md @@ -889,6 +889,7 @@ This includes the following fields: |`compactionPolicy`|Policy to choose intervals for compaction. Currently, the only supported policy is [Newest segment first](#compaction-policy-newestsegmentfirst).|Newest segment first| |`useSupervisors`|Whether compaction should be run on Overlord using supervisors instead of Coordinator duties.|false| |`engine`|Engine used for running compaction tasks, unless overridden in the datasource-level compaction config. Possible values are `native` and `msq`. `msq` engine can be used for compaction only if `useSupervisors` is `true`.|`native`| +|`storeCompactionStatePerSegment`|**This configuration only takes effect if `useSupervisors` is `true`.** Whether to persist the full compaction state in segment metadata. When `true` (default), compaction state is stored in both the segment metadata and the indexing states table. This is historically how Druid has worked. When `false`, only a fingerprint reference is stored in the segment metadata, reducing storage overhead in the segments table. The actual compaction state is stored in the indexing states table and can be referenced with the aforementioned fingerprint. Eventually this configuration will be removed and all compaction will use the fingerprint method only. This configuration exists for operators to opt into this future pattern early. **WARNING: if you set this to false and then compact data, rolling back to a Druid version that predates indexing state fingerprinting (< Druid 37) will result in missing compaction states and trigger compaction on segments that may already be compacted.**|`true`| #### Compaction policy `newestSegmentFirst` diff --git a/docs/configuration/index.md b/docs/configuration/index.md index a1d2d3070f61..6e0f34583c69 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -389,6 +389,7 @@ These properties specify the JDBC connection and other configuration around the |`druid.metadata.storage.tables.segments`|The table to use to look for segments.|`druid_segments`| |`druid.metadata.storage.tables.rules`|The table to use to look for segment load/drop rules.|`druid_rules`| |`druid.metadata.storage.tables.config`|The table to use to look for configs.|`druid_config`| +|`druid.metadata.storage.tables.indexingStates`|The table that stores indexing state payloads and fingerprints.|`druid_indexingStates`| |`druid.metadata.storage.tables.tasks`|Used by the indexing service to store tasks.|`druid_tasks`| |`druid.metadata.storage.tables.taskLog`|Used by the indexing service to store task logs.|`druid_tasklogs`| |`druid.metadata.storage.tables.taskLock`|Used by the indexing service to store task locks.|`druid_tasklocks`| diff --git a/docs/data-management/automatic-compaction.md b/docs/data-management/automatic-compaction.md index 97d7074586cb..f82c66d70fea 100644 --- a/docs/data-management/automatic-compaction.md +++ b/docs/data-management/automatic-compaction.md @@ -241,12 +241,18 @@ You can run automatic compaction using compaction supervisors on the Overlord ra * Can use either the native compaction engine or the [MSQ task engine](#use-msq-for-auto-compaction) * More reactive and submits tasks as soon as a compaction slot is available * Tracked compaction task status to avoid re-compacting an interval repeatedly +* Uses new Indexing State Fingerprinting mechanisms to store less data per segment in metadata storage -To use compaction supervisors, update the [compaction dynamic config](../api-reference/automatic-compaction-api.md#update-cluster-level-compaction-config) and set: +To use compaction supervisors, the following configuration requirements must be met: -* `useSupervisors` to `true` so that compaction tasks can be run as supervisor tasks -* `engine` to `msq` to use the MSQ task engine as the compaction engine or to `native` (default value) to use the native engine. +* You must be using incremental segment metadata caching: + * `druid.manager.segments.useIncrementalCache` set to `always` or `ifSynced` in your Overlord and Coordinator runtime properties. + * See [Segment metadata caching](../configuration/index.md#metadata-retrieval) for full configuration documentation. + +* update the [compaction dynamic config](../api-reference/automatic-compaction-api.md#update-cluster-level-compaction-config) and set: + * `useSupervisors` to `true` so that compaction tasks can be run as supervisor tasks + * `engine` to `msq` to use the MSQ task engine as the compaction engine or to `native` (default value) to use the native engine. Compaction supervisors use the same syntax as auto-compaction using Coordinator duties with one key difference: you submit the auto-compaction as a supervisor spec. In the spec, set the `type` to `autocompact` and include the auto-compaction config in the `spec`. diff --git a/docs/operations/clean-metadata-store.md b/docs/operations/clean-metadata-store.md index 65de31123023..71f673d2a3aa 100644 --- a/docs/operations/clean-metadata-store.md +++ b/docs/operations/clean-metadata-store.md @@ -34,6 +34,7 @@ The metadata store includes the following: - Compaction configuration records - Datasource records created by supervisors - Indexer task logs +- Indexing State records When you delete some entities from Apache Druid, records related to the entity may remain in the metadata store. If you have a high datasource churn rate, meaning you frequently create and delete many short-lived datasources or other related entities like compaction configuration or rules, the leftover records can fill your metadata store and cause performance issues. @@ -59,7 +60,7 @@ If you have compliance requirements to keep audit records and you enable automat ## Configure automated metadata cleanup You can configure cleanup for each entity separately, as described in this section. -Define the properties in the `coordinator/runtime.properties` file. +Unless otherwise specified, define the properties in the `coordinator/runtime.properties` file. The cleanup of one entity may depend on the cleanup of another entity as follows: - You have to configure a [kill task for segment records](#segment-records-and-segments-in-deep-storage-kill-task) before you can configure automated cleanup for [rules](#rules-records) or [compaction configuration](#compaction-configuration-records). @@ -131,6 +132,27 @@ Compaction configuration cleanup uses the following configuration: If you already have an extremely large compaction configuration, you may not be able to delete compaction configuration due to size limits with the audit log. In this case you can set `druid.audit.manager.maxPayloadSizeBytes` and `druid.audit.manager.skipNullField` to avoid the auditing issue. See [Audit logging](../configuration/index.md#audit-logging). ::: +### Indexing State Records + +:::info +Indexing State Records are cleaned up by the overlord. Therefore, this section should be configured in the `overlord/runtime.properties` file. +::: + +:::info +Indexing State Records are only created if you are using automatic compaction supervisors. +::: + +Indexing State records become eligible for deletion in the following scenarios: +- When no `used` segments have an `indexing_state_fingerprint` that is equal to the `fingerprint` of the record. +- When a record has `pending` state set to `true` + +Indexing State cleanup uses the following configuration: + - `druid.overlord.kill.indexingStates.on`: When `true`, enables cleanup for indexing state records. + - `druid.overlord.kill.indexingStates.period`: Defines the frequency in [ISO 8601 format](https://en.wikipedia.org/wiki/ISO_8601#Durations) for the cleanup job to check for and delete eligible indexing state records. Defaults to `P1D`. + - `druid.overlord.kill.indexingStates.durationToRetain`: Defines the retention period in [ISO 8601 format](https://en.wikipedia.org/wiki/ISO_8601#Durations) after indexing state records are marked as `used=false` become eligible for deletion. Defaults to `P7D`. + - `druid.overlord.kill.indexingStates.pendingDurationToRetain`: Defines the retention period in [ISO 8601 format](https://en.wikipedia.org/wiki/ISO_8601#Durations) after creation that pending indexing state records become eligible for deletion. Defaults to `P7D`. + - It is recommended that this value be greater than the maximum expected duration of compaction tasks to avoid pending records being deleted prematurely. + ### Datasource records created by supervisors Datasource records created by supervisors become eligible for deletion when the supervisor is terminated or does not exist in the `druid_supervisors` table and the `durationToRetain` time has passed since their creation. @@ -160,7 +182,9 @@ For more detail, see [Task logging](../configuration/index.md#task-logging). ## Disable automated metadata cleanup Druid automatically cleans up metadata records, excluding compaction configuration records and indexer task logs. -To disable automated metadata cleanup, set the following properties in the `coordinator/runtime.properties` file: +To disable automated metadata cleanup + +set the following properties in the `coordinator/runtime.properties` file: ```properties # Keep unused segments @@ -178,11 +202,19 @@ druid.coordinator.kill.rule.on=false # Keep datasource records created by supervisors druid.coordinator.kill.datasource.on=false ``` +set the following properties in the `overlord/runtime.properties` file: + +```properties +# Keep indexing state records +druid.overlord.kill.indexingStates.on=false +``` ## Example configuration for automated metadata cleanup Consider a scenario where you have scripts to create and delete hundreds of datasources and related entities a day. You do not want to fill your metadata store with leftover records. The datasources and related entities tend to persist for only one or two days. Therefore, you want to run a cleanup job that identifies and removes leftover records that are at least four days old after a seven day buffer period in case you want to recover the data. The exception is for audit logs, which you need to retain for 30 days: +Coordinator configuration (`coordinator/runtime.properties`): + ```properties ... # Schedule the metadata management store task for every hour: @@ -226,6 +258,18 @@ druid.coordinator.kill.datasource.durationToRetain=P4D ... ``` +Overlord configuration - if using automatic compaction supervisors (`overlord/runtime.properties`): + +```properties +... +# Poll every day to delete pending or unreferenced indexing state records > 4 days old +druid.overlord.kill.indexingStates.on=true +druid.overlord.kill.indexingStates.period=P1D +druid.overlord.kill.indexingStates.durationToRetain=P4D +druid.overlord.kill.indexingStates.pendingDurationToRetain=P4D +... +``` + ## Learn more See the following topics for more information: - [Metadata management](../configuration/index.md#metadata-management) for metadata store configuration reference. diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/AutoCompactionTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/AutoCompactionTest.java index 0c7f4a93ff74..4bc72870a1e5 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/AutoCompactionTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/AutoCompactionTest.java @@ -1508,7 +1508,7 @@ public void testAutoCompactionDutyWithDimensionsSpec(CompactionEngine engine) th @ParameterizedTest(name = "useSupervisors={0}") public void testAutoCompactionDutyWithFilter(boolean useSupervisors) throws Exception { - updateClusterConfig(new ClusterCompactionConfig(0.5, 10, null, useSupervisors, null)); + updateClusterConfig(new ClusterCompactionConfig(0.5, 10, null, useSupervisors, null, true)); loadData(INDEX_TASK); try (final Closeable ignored = unloader(fullDatasourceName)) { @@ -1552,7 +1552,7 @@ public void testAutoCompactionDutyWithFilter(boolean useSupervisors) throws Exce @ParameterizedTest(name = "useSupervisors={0}") public void testAutoCompationDutyWithMetricsSpec(boolean useSupervisors) throws Exception { - updateClusterConfig(new ClusterCompactionConfig(0.5, 10, null, useSupervisors, null)); + updateClusterConfig(new ClusterCompactionConfig(0.5, 10, null, useSupervisors, null, true)); loadData(INDEX_TASK); try (final Closeable ignored = unloader(fullDatasourceName)) { @@ -1854,7 +1854,7 @@ private void forceTriggerAutoCompaction( ).collect(Collectors.toList()) ); updateClusterConfig( - new ClusterCompactionConfig(0.5, intervals.size(), policy, true, null) + new ClusterCompactionConfig(0.5, intervals.size(), policy, true, null, true) ); // Wait for scheduler to pick up the compaction job @@ -1864,7 +1864,7 @@ private void forceTriggerAutoCompaction( // Disable all compaction updateClusterConfig( - new ClusterCompactionConfig(0.5, intervals.size(), COMPACT_NOTHING_POLICY, true, null) + new ClusterCompactionConfig(0.5, intervals.size(), COMPACT_NOTHING_POLICY, true, null, true) ); } else { forceTriggerAutoCompaction(numExpectedSegmentsAfterCompaction); @@ -1956,7 +1956,8 @@ private void updateCompactionTaskSlot(double compactionTaskSlotRatio, int maxCom maxCompactionTaskSlots, oldConfig.getCompactionPolicy(), oldConfig.isUseSupervisors(), - oldConfig.getEngine() + oldConfig.getEngine(), + oldConfig.isStoreCompactionStatePerSegment() ) ); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java index c3796ccaa151..5355f1601ae4 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionSupervisorTest.java @@ -27,10 +27,14 @@ import org.apache.druid.indexing.common.task.IndexTask; import org.apache.druid.indexing.compact.CompactionSupervisorSpec; import org.apache.druid.indexing.overlord.Segments; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.DruidMetrics; import org.apache.druid.rpc.UpdateResponse; +import org.apache.druid.segment.metadata.DefaultIndexingStateFingerprintMapper; +import org.apache.druid.segment.metadata.IndexingStateCache; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; import org.apache.druid.server.coordinator.ClusterCompactionConfig; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; @@ -98,14 +102,14 @@ public EmbeddedDruidCluster createCluster() private void configureCompaction(CompactionEngine compactionEngine) { final UpdateResponse updateResponse = cluster.callApi().onLeaderOverlord( - o -> o.updateClusterCompactionConfig(new ClusterCompactionConfig(1.0, 100, null, true, compactionEngine)) + o -> o.updateClusterCompactionConfig(new ClusterCompactionConfig(1.0, 100, null, true, compactionEngine, true)) ); Assertions.assertTrue(updateResponse.isSuccess()); } @MethodSource("getEngine") @ParameterizedTest(name = "compactionEngine={0}") - public void test_ingestDayGranularity_andCompactToMonthGranularity_withInlineConfig(CompactionEngine compactionEngine) + public void test_ingestDayGranularity_andCompactToMonthGranularity_andCompactToYearGranularity_withInlineConfig(CompactionEngine compactionEngine) { configureCompaction(compactionEngine); @@ -119,7 +123,7 @@ public void test_ingestDayGranularity_andCompactToMonthGranularity_withInlineCon Assertions.assertEquals(3, getNumSegmentsWith(Granularities.DAY)); // Create a compaction config with MONTH granularity - InlineSchemaDataSourceCompactionConfig compactionConfig = + InlineSchemaDataSourceCompactionConfig monthGranularityConfig = InlineSchemaDataSourceCompactionConfig .builder() .forDataSource(dataSource) @@ -152,11 +156,159 @@ public void test_ingestDayGranularity_andCompactToMonthGranularity_withInlineCon ) .build(); - runCompactionWithSpec(compactionConfig); + runCompactionWithSpec(monthGranularityConfig); waitForAllCompactionTasksToFinish(); Assertions.assertEquals(0, getNumSegmentsWith(Granularities.DAY)); Assertions.assertEquals(1, getNumSegmentsWith(Granularities.MONTH)); + + verifyCompactedSegmentsHaveFingerprints(monthGranularityConfig); + + InlineSchemaDataSourceCompactionConfig yearGranConfig = + InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(dataSource) + .withSkipOffsetFromLatest(Period.seconds(0)) + .withGranularitySpec( + new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null) + ) + .withTuningConfig( + new UserCompactionTaskQueryTuningConfig( + null, + null, + null, + null, + null, + new DimensionRangePartitionsSpec(null, 5000, List.of("item"), false), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ) + .build(); + + overlord.latchableEmitter().flush(); // flush events so wait for works correctly on the next round of compaction + runCompactionWithSpec(yearGranConfig); + waitForAllCompactionTasksToFinish(); + + Assertions.assertEquals(0, getNumSegmentsWith(Granularities.DAY)); + Assertions.assertEquals(0, getNumSegmentsWith(Granularities.MONTH)); + Assertions.assertEquals(1, getNumSegmentsWith(Granularities.YEAR)); + + verifyCompactedSegmentsHaveFingerprints(yearGranConfig); + } + + @MethodSource("getEngine") + @ParameterizedTest(name = "compactionEngine={0}") + public void test_compaction_withPersistLastCompactionStateFalse_storesOnlyFingerprint(CompactionEngine compactionEngine) + { + // Configure cluster with storeCompactionStatePerSegment=false + final UpdateResponse updateResponse = cluster.callApi().onLeaderOverlord( + o -> o.updateClusterCompactionConfig( + new ClusterCompactionConfig(1.0, 100, null, true, compactionEngine, false) + ) + ); + Assertions.assertTrue(updateResponse.isSuccess()); + + // Ingest data at DAY granularity + runIngestionAtGranularity( + "DAY", + "2025-06-01T00:00:00.000Z,shirt,105\n" + + "2025-06-02T00:00:00.000Z,trousers,210" + ); + Assertions.assertEquals(2, getNumSegmentsWith(Granularities.DAY)); + + // Create compaction config to compact to MONTH granularity + InlineSchemaDataSourceCompactionConfig monthConfig = + InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(dataSource) + .withSkipOffsetFromLatest(Period.seconds(0)) + .withGranularitySpec( + new UserCompactionTaskGranularityConfig(Granularities.MONTH, null, null) + ) + .withTuningConfig( + new UserCompactionTaskQueryTuningConfig( + null, + null, + null, + null, + null, + new DimensionRangePartitionsSpec(1000, null, List.of("item"), false), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ) + .build(); + + runCompactionWithSpec(monthConfig); + waitForAllCompactionTasksToFinish(); + + verifySegmentsHaveNullLastCompactionStateAndNonNullFingerprint(); + } + + private void verifySegmentsHaveNullLastCompactionStateAndNonNullFingerprint() + { + overlord + .bindings() + .segmentsMetadataStorage() + .retrieveAllUsedSegments(dataSource, Segments.ONLY_VISIBLE) + .forEach(segment -> { + Assertions.assertNull( + segment.getLastCompactionState(), + "Segment " + segment.getId() + " should have null lastCompactionState" + ); + Assertions.assertNotNull( + segment.getIndexingStateFingerprint(), + "Segment " + segment.getId() + " should have non-null indexingStateFingerprint" + ); + }); + } + + private void verifyCompactedSegmentsHaveFingerprints(DataSourceCompactionConfig compactionConfig) + { + IndexingStateCache cache = overlord.bindings().getInstance(IndexingStateCache.class); + IndexingStateFingerprintMapper fingerprintMapper = new DefaultIndexingStateFingerprintMapper( + cache, + new DefaultObjectMapper() + ); + String expectedFingerprint = fingerprintMapper.generateFingerprint( + dataSource, + compactionConfig.toCompactionState() + ); + + overlord + .bindings() + .segmentsMetadataStorage() + .retrieveAllUsedSegments(dataSource, Segments.ONLY_VISIBLE) + .forEach(segment -> { + Assertions.assertEquals( + expectedFingerprint, + segment.getIndexingStateFingerprint(), + "Segment " + segment.getId() + " fingerprint should match expected fingerprint" + ); + }); } private void runCompactionWithSpec(DataSourceCompactionConfig config) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java index 8200bb335566..9d2af4d2b21c 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java @@ -212,7 +212,7 @@ public void test_ingestClusterMetrics_withConcurrentCompactionSupervisor_andSkip ); final ClusterCompactionConfig updatedCompactionConfig - = new ClusterCompactionConfig(1.0, 10, null, true, null); + = new ClusterCompactionConfig(1.0, 10, null, true, null, null); final UpdateResponse updateResponse = cluster.callApi().onLeaderOverlord( o -> o.updateClusterCompactionConfig(updatedCompactionConfig) ); @@ -323,7 +323,7 @@ public void test_ingestClusterMetrics_compactionSkipsLockedIntervals() ); final ClusterCompactionConfig updatedCompactionConfig - = new ClusterCompactionConfig(1.0, 10, null, true, null); + = new ClusterCompactionConfig(1.0, 10, null, true, null, null); final UpdateResponse updateResponse = cluster.callApi().onLeaderOverlord( o -> o.updateClusterCompactionConfig(updatedCompactionConfig) ); diff --git a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java index 0e225bf92fa5..212a1e8529de 100644 --- a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java +++ b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java @@ -53,6 +53,7 @@ import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.segment.realtime.ChatHandlerProvider; import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector; @@ -120,7 +121,8 @@ public void setUp() derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); metadataSupervisorManager = EasyMock.createMock(MetadataSupervisorManager.class); taskQueue = EasyMock.createMock(TaskQueue.class); diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java index 1ec98359db79..7ce32d1f1f1a 100644 --- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java +++ b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java @@ -55,6 +55,7 @@ import org.apache.druid.query.topn.TopNQueryBuilder; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.segment.realtime.appenderator.SegmentSchemas; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -129,7 +130,8 @@ public void setUp() throws Exception derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); setupServerAndCurator(); diff --git a/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/storage/sqlserver/SQLServerConnector.java b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/storage/sqlserver/SQLServerConnector.java index 7b3c94f49926..be7147cc5d23 100644 --- a/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/storage/sqlserver/SQLServerConnector.java +++ b/extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/storage/sqlserver/SQLServerConnector.java @@ -294,4 +294,23 @@ protected boolean connectorIsTransientException(Throwable e) } return false; } + + @Override + public boolean isUniqueConstraintViolation(Throwable t) + { + Throwable cause = t; + while (cause != null) { + if (cause instanceof SQLException) { + SQLException sqlException = (SQLException) cause; + String sqlState = sqlException.getSQLState(); + + // SQL standard unique constraint violation code is 23000 for Sql Server + if ("23000".equals(sqlState)) { + return true; + } + } + cause = cause.getCause(); + } + return false; + } } diff --git a/extensions-contrib/sqlserver-metadata-storage/src/test/java/org/apache/druid/metadata/storage/sqlserver/SQLServerConnectorTest.java b/extensions-contrib/sqlserver-metadata-storage/src/test/java/org/apache/druid/metadata/storage/sqlserver/SQLServerConnectorTest.java index 1c44edd18fe5..9903871b573e 100644 --- a/extensions-contrib/sqlserver-metadata-storage/src/test/java/org/apache/druid/metadata/storage/sqlserver/SQLServerConnectorTest.java +++ b/extensions-contrib/sqlserver-metadata-storage/src/test/java/org/apache/druid/metadata/storage/sqlserver/SQLServerConnectorTest.java @@ -54,6 +54,36 @@ public void testIsTransientException() Assert.assertFalse(connector.isTransientException(new Throwable("Throwable with reason only"))); } + @Test + public void testIsUniqueConstraintViolation() + { + SQLServerConnector connector = new SQLServerConnector( + Suppliers.ofInstance(new MetadataStorageConnectorConfig()), + Suppliers.ofInstance( + MetadataStorageTablesConfig.fromBase(null) + ), + CentralizedDatasourceSchemaConfig.create() + ); + + // SQL Server integrity_constraint_violation SQL state (23000) + Assert.assertTrue(connector.isUniqueConstraintViolation( + new SQLException("Violation of UNIQUE KEY constraint", "23000") + )); + + // Different SQL state should return false + Assert.assertFalse(connector.isUniqueConstraintViolation( + new SQLException("some other error", "42000") + )); + + // SQLException wrapped in another exception (tests cause chain traversal) + Assert.assertTrue(connector.isUniqueConstraintViolation( + new RuntimeException(new SQLException("Duplicate key", "23000")) + )); + + // Non-SQLException exception + Assert.assertFalse(connector.isUniqueConstraintViolation(new Exception("not a SQLException"))); + } + @Test public void testLimitClause() { diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/compact/CatalogCompactionTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/compact/CatalogCompactionTest.java index ece6a9ef0573..d5093d4ffafe 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/compact/CatalogCompactionTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/compact/CatalogCompactionTest.java @@ -164,7 +164,7 @@ private IndexTask createIndexTaskForInlineData(String taskId) private void enableCompactionSupervisor() { final UpdateResponse updateResponse = cluster.callApi().onLeaderOverlord( - o -> o.updateClusterCompactionConfig(new ClusterCompactionConfig(1.0, 10, null, true, null)) + o -> o.updateClusterCompactionConfig(new ClusterCompactionConfig(1.0, 10, null, true, null, null)) ); Assertions.assertTrue(updateResponse.isSuccess()); } diff --git a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java index b6a05b300f03..8c4f928be76b 100644 --- a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java +++ b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java @@ -271,6 +271,25 @@ public DBI getDBI() return dbi; } + @Override + public boolean isUniqueConstraintViolation(Throwable t) + { + Throwable cause = t; + while (cause != null) { + if (cause instanceof SQLException) { + SQLException sqlException = (SQLException) cause; + String sqlState = sqlException.getSQLState(); + + // SQL standard unique constraint violation code is 23000 for MySQL + if ("23000".equals(sqlState)) { + return true; + } + } + cause = cause.getCause(); + } + return false; + } + @Nullable private Class tryLoadDriverClass(String className, boolean failIfNotFound) { diff --git a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorTest.java b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorTest.java index da628020d0f5..4f5984303802 100644 --- a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorTest.java +++ b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorTest.java @@ -137,6 +137,36 @@ public void testIsRootCausePacketTooBigException() ); } + @Test + public void testIsUniqueConstraintViolation() + { + MySQLConnector connector = new MySQLConnector( + CONNECTOR_CONFIG_SUPPLIER, + TABLES_CONFIG_SUPPLIER, + new MySQLConnectorSslConfig(), + MYSQL_DRIVER_CONFIG, + centralizedDatasourceSchemaConfig + ); + + // MySQL integrity_constraint_violation SQL state (23000) + Assert.assertTrue(connector.isUniqueConstraintViolation( + new SQLException("Duplicate entry 'value' for key 'PRIMARY'", "23000") + )); + + // Different SQL state should return false + Assert.assertFalse(connector.isUniqueConstraintViolation( + new SQLException("some other error", "42S02") + )); + + // SQLException wrapped in another exception (tests cause chain traversal) + Assert.assertTrue(connector.isUniqueConstraintViolation( + new RuntimeException(new SQLException("Duplicate entry", "23000")) + )); + + // Non-SQLException exception + Assert.assertFalse(connector.isUniqueConstraintViolation(new Exception("not a SQLException"))); + } + @Test public void testLimitClause() { diff --git a/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/postgresql/PostgreSQLConnector.java b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/postgresql/PostgreSQLConnector.java index 765e12aa39a8..48e0bd4e8c39 100644 --- a/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/postgresql/PostgreSQLConnector.java +++ b/extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/postgresql/PostgreSQLConnector.java @@ -308,4 +308,23 @@ public Set getIndexOnTable(String tableName) { return super.getIndexOnTable(StringUtils.toLowerCase(tableName)); } + + @Override + public boolean isUniqueConstraintViolation(Throwable t) + { + Throwable cause = t; + while (cause != null) { + if (cause instanceof SQLException) { + SQLException sqlException = (SQLException) cause; + String sqlState = sqlException.getSQLState(); + + // SQL standard unique constraint violation code is 23505 for PostgreSQL + if ("23505".equals(sqlState)) { + return true; + } + } + cause = cause.getCause(); + } + return false; + } } diff --git a/extensions-core/postgresql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/postgresql/PostgreSQLConnectorTest.java b/extensions-core/postgresql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/postgresql/PostgreSQLConnectorTest.java index bffe53b693b6..3fdf1c5e0693 100644 --- a/extensions-core/postgresql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/postgresql/PostgreSQLConnectorTest.java +++ b/extensions-core/postgresql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/postgresql/PostgreSQLConnectorTest.java @@ -69,6 +69,36 @@ public void testIsTransientException() Assert.assertFalse(connector.isTransientException(new Throwable("I give up"))); } + @Test + public void testIsUniqueConstraintViolation() + { + PostgreSQLConnector connector = new PostgreSQLConnector( + Suppliers.ofInstance(new MetadataStorageConnectorConfig()), + Suppliers.ofInstance(MetadataStorageTablesConfig.fromBase(null)), + new PostgreSQLConnectorConfig(), + new PostgreSQLTablesConfig(), + centralizedDatasourceSchemaConfig + ); + + // PostgreSQL unique_violation SQL state (23505) + Assert.assertTrue(connector.isUniqueConstraintViolation( + new SQLException("duplicate key value violates unique constraint", "23505") + )); + + // Different SQL state should return false + Assert.assertFalse(connector.isUniqueConstraintViolation( + new SQLException("some other error", "42P01") + )); + + // SQLException wrapped in another exception (tests cause chain traversal) + Assert.assertTrue(connector.isUniqueConstraintViolation( + new RuntimeException(new SQLException("duplicate key", "23505")) + )); + + // Non-SQLException exception + Assert.assertFalse(connector.isUniqueConstraintViolation(new Exception("not a SQLException"))); + } + @Test public void testLimitClause() { diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java index dfbdc3e4e85f..105c4ceb9aff 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/updater/MetadataStorageUpdaterJobSpec.java @@ -99,6 +99,7 @@ public MetadataStorageTablesConfig getMetadataStorageTablesConfig() null, null, null, + null, null ); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java index ccdc99dcd059..702f67b3bd12 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/AbstractBatchIndexTask.java @@ -641,6 +641,25 @@ public static boolean isGuaranteedRollup( return tuningConfig.isForceGuaranteedRollup(); } + /** + * Returns a function that adds the given indexing state fingerprint to all segments. + * If the fingerprint is null, returns an identity function that leaves segments unchanged. + */ + public static Function, Set> addIndexingStateFingerprintToSegments( + String indexingStateFingerprint + ) + { + if (indexingStateFingerprint != null) { + return segments -> segments.stream() + .map( + segment -> segment.withIndexingStateFingerprint(indexingStateFingerprint) + ) + .collect(Collectors.toSet()); + } else { + return Function.identity(); + } + } + public static Function, Set> addCompactionStateToSegments( boolean storeCompactionState, TaskToolbox toolbox, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java index f87ac965ce98..637a3f28c431 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java @@ -902,11 +902,19 @@ private TaskStatus generateAndPublishSegments( Tasks.STORE_COMPACTION_STATE_KEY, Tasks.DEFAULT_STORE_COMPACTION_STATE ); + + final String indexingStateFingerprint = getContextValue( + Tasks.INDEXING_STATE_FINGERPRINT_KEY, + null + ); + final Function, Set> annotateFunction = addCompactionStateToSegments( storeCompactionState, toolbox, ingestionSchema + ).andThen( + addIndexingStateFingerprintToSegments(indexingStateFingerprint) ); Set tombStones = Collections.emptySet(); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Tasks.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Tasks.java index b45eb45dc041..7f64a6cbad0d 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Tasks.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/Tasks.java @@ -68,4 +68,13 @@ public class Tasks static { Verify.verify(STORE_COMPACTION_STATE_KEY.equals(CompactSegments.STORE_COMPACTION_STATE_KEY)); } + + /** + * Context k:v pair that holds the fingerprint of the indexing state to be stored with the segment + */ + public static final String INDEXING_STATE_FINGERPRINT_KEY = "indexingStateFingerprint"; + + static { + Verify.verify(INDEXING_STATE_FINGERPRINT_KEY.equals(CompactSegments.INDEXING_STATE_FINGERPRINT_KEY)); + } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java index 74678086d590..02f11e8f5e20 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java @@ -1162,12 +1162,20 @@ private void publishSegments( Tasks.STORE_COMPACTION_STATE_KEY, Tasks.DEFAULT_STORE_COMPACTION_STATE ); - final Function, Set> annotateFunction = addCompactionStateToSegments( - storeCompactionState, - toolbox, - ingestionSchema + final String indexingStateFingerprint = getContextValue( + Tasks.INDEXING_STATE_FINGERPRINT_KEY, + null ); + final Function, Set> annotateFunction = + addCompactionStateToSegments( + storeCompactionState, + toolbox, + ingestionSchema + ).andThen( + addIndexingStateFingerprintToSegments(indexingStateFingerprint) + ); + Set tombStones = Collections.emptySet(); if (getIngestionMode() == IngestionMode.REPLACE) { TombstoneHelper tombstoneHelper = new TombstoneHelper(toolbox.getTaskActionClient()); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java index 6b984a4b6c03..24b6e1f6af40 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java @@ -31,6 +31,7 @@ import org.apache.druid.server.compaction.NewestSegmentFirstPolicy; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.server.coordinator.duty.CompactSegments; +import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.SegmentTimeline; import org.joda.time.Interval; @@ -70,17 +71,31 @@ public List createCompactionJobs( final List jobs = new ArrayList<>(); + CompactionState compactionState = config.toCompactionState(); + + String indexingStateFingerprint = params.getFingerprintMapper().generateFingerprint( + config.getDataSource(), + compactionState + ); + // Create a job for each CompactionCandidate while (segmentIterator.hasNext()) { final CompactionCandidate candidate = segmentIterator.next(); - ClientCompactionTaskQuery taskPayload - = CompactSegments.createCompactionTask(candidate, config, params.getClusterCompactionConfig().getEngine()); + ClientCompactionTaskQuery taskPayload = CompactSegments.createCompactionTask( + candidate, + config, + params.getClusterCompactionConfig().getEngine(), + indexingStateFingerprint, + params.getClusterCompactionConfig().isStoreCompactionStatePerSegment() + ); jobs.add( new CompactionJob( taskPayload, candidate, - CompactionSlotManager.computeSlotsRequiredForTask(taskPayload) + CompactionSlotManager.computeSlotsRequiredForTask(taskPayload), + indexingStateFingerprint, + compactionState ) ); } @@ -120,7 +135,8 @@ DataSourceCompactibleSegmentIterator getCompactibleCandidates( Intervals.complementOf(searchInterval), // This policy is used only while creating jobs // The actual order of jobs is determined by the policy used in CompactionJobQueue - new NewestSegmentFirstPolicy(null) + new NewestSegmentFirstPolicy(null), + params.getFingerprintMapper() ); // Collect stats for segments that are already compacted diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJob.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJob.java index 7a7e7fdc1eab..7611485bd6d8 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJob.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJob.java @@ -23,6 +23,7 @@ import org.apache.druid.indexing.template.BatchIndexingJob; import org.apache.druid.query.http.ClientSqlQuery; import org.apache.druid.server.compaction.CompactionCandidate; +import org.apache.druid.timeline.CompactionState; /** * {@link BatchIndexingJob} to compact an interval of a datasource. @@ -31,27 +32,37 @@ public class CompactionJob extends BatchIndexingJob { private final CompactionCandidate candidate; private final int maxRequiredTaskSlots; + private final String targetIndexingStateFingerprint; + private final CompactionState targetIndexingState; public CompactionJob( ClientCompactionTaskQuery task, CompactionCandidate candidate, - int maxRequiredTaskSlots + int maxRequiredTaskSlots, + String targetIndexingStateFingerprint, + CompactionState targetIndexingState ) { super(task, null); this.candidate = candidate; this.maxRequiredTaskSlots = maxRequiredTaskSlots; + this.targetIndexingStateFingerprint = targetIndexingStateFingerprint; + this.targetIndexingState = targetIndexingState; } public CompactionJob( ClientSqlQuery msqQuery, CompactionCandidate candidate, - int maxRequiredTaskSlots + int maxRequiredTaskSlots, + String targetIndexingStateFingerprint, + CompactionState targetIndexingState ) { super(null, msqQuery); this.candidate = candidate; this.maxRequiredTaskSlots = maxRequiredTaskSlots; + this.targetIndexingStateFingerprint = targetIndexingStateFingerprint; + this.targetIndexingState = targetIndexingState; } public String getDataSource() @@ -69,6 +80,16 @@ public int getMaxRequiredTaskSlots() return maxRequiredTaskSlots; } + public String getTargetIndexingStateFingerprint() + { + return targetIndexingStateFingerprint; + } + + public CompactionState getTargetIndexingState() + { + return targetIndexingState; + } + @Override public String toString() { @@ -76,6 +97,8 @@ public String toString() super.toString() + ", candidate=" + candidate + ", maxRequiredTaskSlots=" + maxRequiredTaskSlots + + ", targetIndexingStateFingerprint='" + targetIndexingStateFingerprint + '\'' + + ", targetIndexingState=" + targetIndexingState + '}'; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobParams.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobParams.java index 0113f1b78bac..36cd075922d5 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobParams.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobParams.java @@ -19,6 +19,7 @@ package org.apache.druid.indexing.compact; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; import org.apache.druid.server.compaction.CompactionSnapshotBuilder; import org.apache.druid.server.coordinator.ClusterCompactionConfig; import org.apache.druid.timeline.SegmentTimeline; @@ -33,18 +34,21 @@ public class CompactionJobParams private final TimelineProvider timelineProvider; private final ClusterCompactionConfig clusterCompactionConfig; private final CompactionSnapshotBuilder snapshotBuilder; + private final IndexingStateFingerprintMapper fingerprintMapper; public CompactionJobParams( DateTime scheduleStartTime, ClusterCompactionConfig clusterCompactionConfig, TimelineProvider timelineProvider, - CompactionSnapshotBuilder snapshotBuilder + CompactionSnapshotBuilder snapshotBuilder, + IndexingStateFingerprintMapper indexingStateFingerprintMapper ) { this.scheduleStartTime = scheduleStartTime; this.clusterCompactionConfig = clusterCompactionConfig; this.timelineProvider = timelineProvider; this.snapshotBuilder = snapshotBuilder; + this.fingerprintMapper = indexingStateFingerprintMapper; } /** @@ -88,6 +92,11 @@ public CompactionSnapshotBuilder getSnapshotBuilder() return snapshotBuilder; } + public IndexingStateFingerprintMapper getFingerprintMapper() + { + return fingerprintMapper; + } + @FunctionalInterface public interface TimelineProvider { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java index 77886af1a017..9446ac664f29 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java @@ -36,6 +36,9 @@ import org.apache.druid.java.util.common.Stopwatch; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.rpc.indexing.OverlordClient; +import org.apache.druid.segment.metadata.DefaultIndexingStateFingerprintMapper; +import org.apache.druid.segment.metadata.IndexingStateCache; +import org.apache.druid.segment.metadata.IndexingStateStorage; import org.apache.druid.server.compaction.CompactionCandidate; import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy; import org.apache.druid.server.compaction.CompactionSlotManager; @@ -96,6 +99,9 @@ public class CompactionJobQueue private final Set activeSupervisors; private final Map submittedTaskIdToJob; + private final IndexingStateStorage indexingStateStorage; + private final IndexingStateCache indexingStateCache; + public CompactionJobQueue( DataSourcesSnapshot dataSourcesSnapshot, ClusterCompactionConfig clusterCompactionConfig, @@ -104,7 +110,9 @@ public CompactionJobQueue( GlobalTaskLockbox taskLockbox, OverlordClient overlordClient, BrokerClient brokerClient, - ObjectMapper objectMapper + ObjectMapper objectMapper, + IndexingStateStorage indexingStateStorage, + IndexingStateCache indexingStateCache ) { this.runStats = new CoordinatorRunStats(); @@ -120,9 +128,13 @@ public CompactionJobQueue( DateTimes.nowUtc(), clusterCompactionConfig, dataSourcesSnapshot.getUsedSegmentsTimelinesPerDataSource()::get, - snapshotBuilder + snapshotBuilder, + new DefaultIndexingStateFingerprintMapper(indexingStateCache, objectMapper) ); + this.indexingStateStorage = indexingStateStorage; + this.indexingStateCache = indexingStateCache; + this.taskActionClientFactory = taskActionClientFactory; this.overlordClient = overlordClient; this.brokerClient = brokerClient; @@ -315,6 +327,7 @@ private String startTaskIfReady(CompactionJob job) // Assume MSQ jobs to be always ready if (job.isMsq()) { try { + persistPendingIndexingState(job); return FutureUtils.getUnchecked(brokerClient.submitSqlTask(job.getNonNullMsqQuery()), true) .getTaskId(); } @@ -333,6 +346,7 @@ private String startTaskIfReady(CompactionJob job) try { taskLockbox.add(task); if (task.isReady(taskActionClientFactory.create(task))) { + persistPendingIndexingState(job); // Hold the locks acquired by task.isReady() as we will reacquire them anyway FutureUtils.getUnchecked(overlordClient.runTask(task.getId(), task), true); return task.getId(); @@ -348,6 +362,22 @@ private String startTaskIfReady(CompactionJob job) } } + /** + * Persist the indexing state associated with the given job with {@link IndexingStateStorage}. + */ + private void persistPendingIndexingState(CompactionJob job) + { + if (job.getTargetIndexingState() != null && job.getTargetIndexingStateFingerprint() != null) { + indexingStateStorage.upsertIndexingState( + job.getDataSource(), + job.getTargetIndexingStateFingerprint(), + job.getTargetIndexingState(), + DateTimes.nowUtc() + ); + indexingStateCache.addIndexingState(job.getTargetIndexingStateFingerprint(), job.getTargetIndexingState()); + } + } + public CompactionStatus getCurrentStatusForJob(CompactionJob job, CompactionCandidateSearchPolicy policy) { final CompactionStatus compactionStatus = statusTracker.computeCompactionStatus(job.getCandidate(), policy); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java index c3bce6a09de1..11709e616c71 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java @@ -26,6 +26,7 @@ import org.apache.druid.client.DataSourcesSnapshot; import org.apache.druid.client.broker.BrokerClient; import org.apache.druid.client.indexing.ClientCompactionRunnerInfo; +import org.apache.druid.error.DruidException; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.actions.TaskActionClientFactory; @@ -44,6 +45,8 @@ import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.metadata.SegmentsMetadataManagerConfig; +import org.apache.druid.segment.metadata.IndexingStateCache; +import org.apache.druid.segment.metadata.IndexingStateStorage; import org.apache.druid.server.compaction.CompactionRunSimulator; import org.apache.druid.server.compaction.CompactionSimulateResult; import org.apache.druid.server.compaction.CompactionStatusTracker; @@ -139,6 +142,9 @@ public class OverlordCompactionScheduler implements CompactionScheduler private final boolean shouldPollSegments; private final long schedulePeriodMillis; + private final IndexingStateStorage indexingStateStorage; + private final IndexingStateCache indexingStateCache; + @Inject public OverlordCompactionScheduler( TaskMaster taskMaster, @@ -154,7 +160,9 @@ public OverlordCompactionScheduler( ScheduledExecutorFactory executorFactory, BrokerClient brokerClient, ServiceEmitter emitter, - ObjectMapper objectMapper + ObjectMapper objectMapper, + IndexingStateStorage indexingStateStorage, + IndexingStateCache indexingStateCache ) { final long segmentPollPeriodMillis = @@ -180,6 +188,8 @@ public OverlordCompactionScheduler( this.taskActionClientFactory = taskActionClientFactory; this.druidInputSourceFactory = druidInputSourceFactory; + this.indexingStateStorage = indexingStateStorage; + this.indexingStateCache = indexingStateCache; this.taskRunnerListener = new TaskRunnerListener() { @Override @@ -208,7 +218,18 @@ public void statusChanged(String taskId, TaskStatus status) @LifecycleStart public synchronized void start() { - // Do nothing + // Validate that if compaction supervisors are enabled, the segment metadata incremental cache must be enabled + if (compactionConfigSupplier.get().isUseSupervisors()) { + if (segmentManager != null && !segmentManager.isPollingDatabasePeriodically()) { + throw DruidException + .forPersona(DruidException.Persona.OPERATOR) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build( + "Compaction supervisors require segment metadata cache to be enabled. " + + "Set 'druid.manager.segments.useIncrementalCache=always' or 'ifSynced'" + ); + } + } } @LifecycleStop @@ -366,7 +387,9 @@ private synchronized void resetCompactionJobQueue() taskLockbox, overlordClient, brokerClient, - objectMapper + objectMapper, + indexingStateStorage, + indexingStateCache ); latestJobQueue.set(queue); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/IndexingStateCleanupConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/IndexingStateCleanupConfig.java new file mode 100644 index 000000000000..fad8ea101f49 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/IndexingStateCleanupConfig.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.config; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.common.config.Configs; +import org.apache.druid.server.coordinator.config.MetadataCleanupConfig; +import org.joda.time.Duration; + +import java.util.Objects; + +/** + * Configuration for cleaning up indexing state metadata. + *

+ * Extends {@link MetadataCleanupConfig} to add support for pending state retention. + */ +public class IndexingStateCleanupConfig extends MetadataCleanupConfig +{ + public static final IndexingStateCleanupConfig DEFAULT = new IndexingStateCleanupConfig(null, null, null, null); + + @JsonProperty("pendingDurationToRetain") + private final Duration pendingDurationToRetain; + + @JsonCreator + public IndexingStateCleanupConfig( + @JsonProperty("on") Boolean cleanupEnabled, + @JsonProperty("period") Duration cleanupPeriod, + @JsonProperty("durationToRetain") Duration durationToRetain, + @JsonProperty("pendingDurationToRetain") Duration pendingDurationToRetain + ) + { + super(cleanupEnabled, cleanupPeriod, durationToRetain); + this.pendingDurationToRetain = Configs.valueOrDefault(pendingDurationToRetain, Duration.standardDays(7)); + } + + public Duration getPendingDurationToRetain() + { + return pendingDurationToRetain; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + IndexingStateCleanupConfig that = (IndexingStateCleanupConfig) o; + return Objects.equals(pendingDurationToRetain, that.pendingDurationToRetain); + } + + @Override + public int hashCode() + { + return Objects.hash(super.hashCode(), pendingDurationToRetain); + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/OverlordKillConfigs.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/OverlordKillConfigs.java new file mode 100644 index 000000000000..3e77a8f1eae7 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/OverlordKillConfigs.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.config; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.common.config.Configs; + +public class OverlordKillConfigs +{ + public static OverlordKillConfigs DEFAULT = new OverlordKillConfigs(null); + + @JsonProperty("indexingStates") + private final IndexingStateCleanupConfig indexingStates; + + public OverlordKillConfigs( + @JsonProperty("indexingStates") IndexingStateCleanupConfig indexingStates + ) + { + this.indexingStates = Configs.valueOrDefault(indexingStates, IndexingStateCleanupConfig.DEFAULT); + } + + public IndexingStateCleanupConfig indexingStates() + { + return indexingStates; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/KillUnreferencedIndexingState.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/KillUnreferencedIndexingState.java new file mode 100644 index 000000000000..a00688b6ea22 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/KillUnreferencedIndexingState.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.duty; + +import org.apache.druid.indexing.overlord.config.IndexingStateCleanupConfig; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.segment.metadata.IndexingStateStorage; +import org.joda.time.DateTime; + +import javax.inject.Inject; +import java.util.List; + +/** + * Duty that cleans up unreferenced indexing states from the indexing state storage. + *

+ * The cleanup process involves: + *

    + *
  1. Marking unreferenced indexing states as unused.
    2. + *
  2. Repairing any unused states that are still referenced by segments.
    3. + *
  3. Deleting unused indexing states older than the configured retention duration.
    4. + *
  4. Deleting any pending indexing states that are older than the configured retention duration.
    5. + *
+ */ +public class KillUnreferencedIndexingState extends OverlordMetadataCleanupDuty +{ + private static final Logger log = new Logger(KillUnreferencedIndexingState.class); + private final IndexingStateStorage indexingStateStorage; + private final IndexingStateCleanupConfig config; + + @Inject + public KillUnreferencedIndexingState( + IndexingStateCleanupConfig config, + IndexingStateStorage indexingStateStorage + ) + { + super("indexingStates", config); + this.config = config; + this.indexingStateStorage = indexingStateStorage; + } + + @Override + public void run() + { + if (!config.isCleanupEnabled()) { + return; + } + + final DateTime now = getCurrentTime(); + + if (getLastCleanupTime().plus(config.getCleanupPeriod()).isBefore(now)) { + try { + // Pending cleanup (specific to indexing states) + DateTime pendingMinCreatedTime = now.minus(config.getPendingDurationToRetain()); + int deletedPendingEntries = indexingStateStorage.deletePendingIndexingStatesOlderThan( + pendingMinCreatedTime.getMillis() + ); + if (deletedPendingEntries > 0) { + log.info( + "Removed [%,d] pending [%s] created before [%s].", + deletedPendingEntries, + getEntryType(), + pendingMinCreatedTime + ); + } + } + catch (Exception e) { + log.error(e, "Failed to perform pending cleanup of [%s]", getEntryType()); + } + + // Delegate to parent for the non-specialized cleanup + super.run(); + } + } + + /** + * Cleans up unreferenced indexing states created before the specified time. + *

+ * Before deletion, it executes the following steps to ensure data integrity: + *

    + *
  1. Marks unreferenced indexing states as unused.
    2. + *
  2. Finds any unused indexing states that are still referenced by used segments and marks them as used to avoid unwanted deletion.
    3. + *
+ * @param minCreatedTime the minimum creation time for indexing states to be considered for deletion + * @return the number of indexing states deleted + */ + @Override + protected int cleanupEntriesCreatedBefore(DateTime minCreatedTime) + { + int unused = indexingStateStorage.markUnreferencedIndexingStatesAsUnused(); + log.info("Marked [%s] unreferenced indexing states as unused.", unused); + + List stateFingerprints = indexingStateStorage.findReferencedIndexingStateMarkedAsUnused(); + if (!stateFingerprints.isEmpty()) { + int numUpdated = indexingStateStorage.markIndexingStatesAsUsed(stateFingerprints); + log.info("Marked [%s] unused indexing states referenced by used segments as used.", numUpdated); + } + + return indexingStateStorage.deleteUnusedIndexingStatesOlderThan(minCreatedTime.getMillis()); + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/OverlordMetadataCleanupDuty.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/OverlordMetadataCleanupDuty.java new file mode 100644 index 000000000000..34d0fe55f1ab --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/OverlordMetadataCleanupDuty.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.duty; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.server.coordinator.config.MetadataCleanupConfig; +import org.joda.time.DateTime; + +/** + * Performs cleanup of stale metadata entries created before a configured retain duration. + *

+ * In every invocation of {@link #run}, the duty checks if the {@code cleanupPeriod} + * has elapsed since the {@link #lastCleanupTime}. If it has, then the method + * {@link #cleanupEntriesCreatedBefore(DateTime)} is invoked. Otherwise, the duty + * completes immediately without making any changes. + */ +public abstract class OverlordMetadataCleanupDuty implements OverlordDuty +{ + private static final Logger log = new Logger(OverlordMetadataCleanupDuty.class); + + private final String entryType; + private final MetadataCleanupConfig cleanupConfig; + + private DateTime lastCleanupTime = DateTimes.utc(0); + + protected OverlordMetadataCleanupDuty(String entryType, MetadataCleanupConfig cleanupConfig) + { + this.entryType = entryType; + this.cleanupConfig = cleanupConfig; + + if (cleanupConfig.isCleanupEnabled()) { + log.debug( + "Enabled cleanup of [%s] with period [%s] and durationToRetain [%s].", + entryType, cleanupConfig.getCleanupPeriod(), cleanupConfig.getDurationToRetain() + ); + } + } + + @Override + public void run() + { + if (!cleanupConfig.isCleanupEnabled()) { + return; + } + + final DateTime now = getCurrentTime(); + + if (lastCleanupTime.plus(cleanupConfig.getCleanupPeriod()).isBefore(now)) { + setLastCleanupTime(now); + + try { + DateTime minCreatedTime = now.minus(cleanupConfig.getDurationToRetain()); + int deletedEntries = cleanupEntriesCreatedBefore(minCreatedTime); + if (deletedEntries > 0) { + log.info("Removed [%,d] [%s] created before [%s].", deletedEntries, entryType, minCreatedTime); + } + } + catch (Exception e) { + log.error(e, "Failed to perform cleanup of [%s]", entryType); + } + } + } + + protected String getEntryType() + { + return entryType; + } + + protected DateTime getLastCleanupTime() + { + return lastCleanupTime; + } + + protected void setLastCleanupTime(DateTime time) + { + lastCleanupTime = time; + } + + @Override + public boolean isEnabled() + { + return cleanupConfig.isCleanupEnabled(); + } + + @Override + public DutySchedule getSchedule() + { + if (isEnabled()) { + return new DutySchedule(cleanupConfig.getCleanupPeriod().getMillis(), 0); + } else { + return new DutySchedule(0, 0); + } + } + + /** + * Cleans up metadata entries created before the {@code minCreatedTime}. + *

+ * This method is not invoked if the {@code cleanupPeriod} has not elapsed since the {@link #lastCleanupTime}. + * + * @return Number of deleted metadata entries + */ + protected abstract int cleanupEntriesCreatedBefore(DateTime minCreatedTime); + + /** + * Returns the current time. + *

+ * Exists so testing can spoof the current time to validate behavior Duty behavior. + */ + @VisibleForTesting + protected DateTime getCurrentTime() + { + return DateTimes.nowUtc(); + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java index b7c47a60a7dd..cb78e3bb33b5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java @@ -41,6 +41,8 @@ import org.apache.druid.metadata.segment.cache.HeapMemorySegmentMetadataCache; import org.apache.druid.metadata.segment.cache.SegmentMetadataCache; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; +import org.apache.druid.segment.metadata.IndexingStateCache; import org.apache.druid.segment.metadata.NoopSegmentSchemaCache; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.server.coordinator.simulate.BlockingExecutorService; @@ -134,7 +136,8 @@ public void before() metadataStorageTablesConfig, testDerbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); taskLockbox = new GlobalTaskLockbox(taskStorage, metadataStorageCoordinator); taskLockbox.syncFromStorage(); @@ -176,6 +179,7 @@ public boolean isBatchAllocationReduceMetadataIO() testDerbyConnector.createConfigTable(); testDerbyConnector.createTaskTables(); testDerbyConnector.createAuditTable(); + testDerbyConnector.createIndexingStatesTable(); segmentMetadataCache.start(); segmentMetadataCache.becomeLeader(); @@ -198,6 +202,7 @@ private SqlSegmentMetadataTransactionFactory setupTransactionFactory( Suppliers.ofInstance(new SegmentsMetadataManagerConfig(Period.seconds(1), cacheMode, null)), Suppliers.ofInstance(metadataStorageTablesConfig), new NoopSegmentSchemaCache(), + new IndexingStateCache(), testDerbyConnector, (poolSize, name) -> new WrappingScheduledExecutorService(name, metadataCachePollExec, false), emitter diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java index 6d0171bd3877..318631b81e9b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java @@ -87,6 +87,8 @@ import org.apache.druid.segment.loading.LocalDataSegmentPusherConfig; import org.apache.druid.segment.loading.SegmentCacheManager; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; +import org.apache.druid.segment.metadata.IndexingStateCache; import org.apache.druid.segment.metadata.SegmentSchemaCache; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.segment.realtime.NoopChatHandlerProvider; @@ -139,6 +141,7 @@ public abstract class IngestionTestBase extends InitializedNullHandlingTest private TestDataSegmentKiller dataSegmentKiller; private SegmentMetadataCache segmentMetadataCache; private SegmentSchemaCache segmentSchemaCache; + private IndexingStateCache indexingStateCache; protected File reportsFile; protected IngestionTestBase() @@ -164,6 +167,7 @@ public void setUpIngestionTestBase() throws IOException connector.createSegmentSchemasTable(); connector.createSegmentTable(); connector.createPendingSegmentsTable(); + connector.createIndexingStatesTable(); taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null)); SegmentSchemaManager segmentSchemaManager = new SegmentSchemaManager( derbyConnectorRule.metadataTablesConfigSupplier().get(), @@ -172,13 +176,15 @@ public void setUpIngestionTestBase() throws IOException ); segmentSchemaCache = new SegmentSchemaCache(); + indexingStateCache = new IndexingStateCache(); storageCoordinator = new IndexerSQLMetadataStorageCoordinator( createTransactionFactory(), objectMapper, derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnectorRule.getConnector(), segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); segmentsMetadataManager = new SqlSegmentsMetadataManagerV2( segmentMetadataCache, @@ -337,6 +343,7 @@ private SqlSegmentMetadataTransactionFactory createTransactionFactory() Suppliers.ofInstance(new SegmentsMetadataManagerConfig(Period.millis(10), cacheMode, null)), derbyConnectorRule.metadataTablesConfigSupplier(), segmentSchemaCache, + indexingStateCache, derbyConnectorRule.getConnector(), ScheduledExecutors::fixed, NoopServiceEmitter.instance() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java index b15f2f2b9b69..4c98b52f48c1 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java @@ -64,6 +64,8 @@ import org.apache.druid.query.http.ClientSqlQuery; import org.apache.druid.query.http.SqlTaskStatus; import org.apache.druid.segment.TestIndex; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; +import org.apache.druid.segment.metadata.IndexingStateCache; import org.apache.druid.server.compaction.CompactionSimulateResult; import org.apache.druid.server.compaction.CompactionStatistics; import org.apache.druid.server.compaction.CompactionStatus; @@ -173,7 +175,7 @@ public void setUp() segmentStorage = new TestIndexerMetadataStorageCoordinator(); segmentsMetadataManager = segmentStorage.getManager(); - compactionConfig = new AtomicReference<>(new ClusterCompactionConfig(1.0, 100, null, true, null)); + compactionConfig = new AtomicReference<>(new ClusterCompactionConfig(1.0, 100, null, true, null, null)); coordinatorOverlordServiceConfig = new CoordinatorOverlordServiceConfig(false, null); taskActionClientFactory = task -> new TaskActionClient() @@ -231,7 +233,9 @@ private void initScheduler() (nameFormat, numThreads) -> new WrappingScheduledExecutorService("test", executor, false), brokerClient, serviceEmitter, - OBJECT_MAPPER + OBJECT_MAPPER, + new HeapMemoryIndexingStateStorage(), + new IndexingStateCache() ); } @@ -444,7 +448,7 @@ public void test_simulateRunWithConfigUpdate() scheduler.startCompaction(dataSource, createSupervisorWithInlineSpec()); final CompactionSimulateResult simulateResult = scheduler.simulateRunWithConfigUpdate( - new ClusterCompactionConfig(null, null, null, null, null) + new ClusterCompactionConfig(null, null, null, null, null, null) ); Assert.assertEquals(1, simulateResult.getCompactionStates().size()); final Table pendingCompactionTable = simulateResult.getCompactionStates().get(CompactionStatus.State.PENDING); @@ -469,7 +473,7 @@ public void test_simulateRunWithConfigUpdate() scheduler.stopCompaction(dataSource); final CompactionSimulateResult simulateResultWhenDisabled = scheduler.simulateRunWithConfigUpdate( - new ClusterCompactionConfig(null, null, null, null, null) + new ClusterCompactionConfig(null, null, null, null, null, null) ); Assert.assertTrue(simulateResultWhenDisabled.getCompactionStates().isEmpty()); @@ -536,12 +540,12 @@ private void runCompactionTask(String taskId, Interval compactionInterval, Granu private void disableScheduler() { - compactionConfig.set(new ClusterCompactionConfig(null, null, null, false, null)); + compactionConfig.set(new ClusterCompactionConfig(null, null, null, false, null, null)); } private void enableScheduler() { - compactionConfig.set(new ClusterCompactionConfig(null, null, null, true, null)); + compactionConfig.set(new ClusterCompactionConfig(null, null, null, true, null, null)); } private void runScheduledJob() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/GlobalTaskLockboxTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/GlobalTaskLockboxTest.java index 2c5390e568b9..42e6e365b306 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/GlobalTaskLockboxTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/GlobalTaskLockboxTest.java @@ -60,6 +60,7 @@ import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector; @@ -145,7 +146,8 @@ public void setup() tablesConfig, derbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); lockbox = new GlobalTaskLockbox(taskStorage, metadataStorageCoordinator); @@ -491,7 +493,8 @@ public void testSyncWithUnknownTaskTypesFromModuleNotLoaded() derby.metadataTablesConfigSupplier().get(), derbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); GlobalTaskLockbox theBox = new GlobalTaskLockbox(taskStorage, metadataStorageCoordinator); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java index 723af0e721d0..3515790e4a2b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java @@ -41,6 +41,7 @@ import org.apache.druid.metadata.segment.SqlSegmentMetadataTransactionFactory; import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -106,7 +107,8 @@ public void setup() derby.metadataTablesConfigSupplier().get(), derbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ) ); lockbox.syncFromStorage(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java index 447a51f44b11..190d78bb33c9 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java @@ -51,6 +51,7 @@ import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -118,7 +119,8 @@ public void setUp() derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnectorRule.getConnector(), segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); final TaskActionClientFactory unsupportedTaskActionFactory = diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/duty/KillUnreferencedIndexingStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/duty/KillUnreferencedIndexingStateTest.java new file mode 100644 index 000000000000..cbb77f624fae --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/duty/KillUnreferencedIndexingStateTest.java @@ -0,0 +1,379 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.duty; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.indexing.overlord.config.IndexingStateCleanupConfig; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.metadata.MetadataStorageTablesConfig; +import org.apache.druid.metadata.TestDerbyConnector; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.metadata.IndexingStateStorage; +import org.apache.druid.segment.metadata.SqlIndexingStateStorage; +import org.apache.druid.timeline.CompactionState; +import org.joda.time.DateTime; +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class KillUnreferencedIndexingStateTest +{ + @Rule + public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = + new TestDerbyConnector.DerbyConnectorRule(); + + private final ObjectMapper jsonMapper = new DefaultObjectMapper(); + + private TestDerbyConnector derbyConnector; + private MetadataStorageTablesConfig tablesConfig; + private SqlIndexingStateStorage compactionStateStorage; + + @Before + public void setUp() + { + derbyConnector = derbyConnectorRule.getConnector(); + tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); + + derbyConnector.createIndexingStatesTable(); + derbyConnector.createSegmentTable(); + + compactionStateStorage = new SqlIndexingStateStorage(tablesConfig, jsonMapper, derbyConnector); + } + + @Test + public void test_killUnreferencedCompactionState_validateLifecycleOfActiveCompactionState() + { + // Setup time progression: now, +1hr, +7hrs + List dateTimes = new ArrayList<>(); + DateTime now = DateTimes.nowUtc(); + dateTimes.add(now); + dateTimes.add(now); + dateTimes.add(now.plusMinutes(61)); + dateTimes.add(now.plusMinutes(61)); + dateTimes.add(now.plusMinutes(6 * 60 + 1)); + dateTimes.add(now.plusMinutes(6 * 60 + 1)); + + IndexingStateCleanupConfig cleanupConfig = new IndexingStateCleanupConfig( + true, + Period.parse("PT1H").toStandardDuration(), + Period.parse("PT6H").toStandardDuration(), // Unused and over 6 hours old should be deleted + Period.parse("P8D").toStandardDuration() + ); + + KillUnreferencedIndexingState duty = + new TestKillUnreferencedIndexingState(cleanupConfig, compactionStateStorage, dateTimes); + + String fingerprint = "test_fingerprint"; + CompactionState state = createTestCompactionState(); + + compactionStateStorage.upsertIndexingState("test-ds", fingerprint, state, DateTimes.nowUtc()); + compactionStateStorage.markIndexingStatesAsActive(List.of(fingerprint)); + + Assert.assertEquals(Boolean.TRUE, getCompactionStateUsedStatus(fingerprint)); + + // Run 1: Should mark as unused (no segments reference it) + duty.run(); + Assert.assertEquals(Boolean.FALSE, getCompactionStateUsedStatus(fingerprint)); + + // Run 2: Still unused, but within retention period - should not delete + duty.run(); + Assert.assertNotNull(getCompactionStateUsedStatus(fingerprint)); + + // Run 3: Past retention period - should delete + duty.run(); + Assert.assertNull(getCompactionStateUsedStatus(fingerprint)); + } + + @Test + public void test_killUnreferencedCompactionState_validateRepair() + { + List dateTimes = new ArrayList<>(); + DateTime now = DateTimes.nowUtc(); + dateTimes.add(now); + dateTimes.add(now); + dateTimes.add(now.plusMinutes(61)); + dateTimes.add(now.plusMinutes(61)); + + IndexingStateCleanupConfig cleanupConfig = new IndexingStateCleanupConfig( + true, + Period.parse("PT1H").toStandardDuration(), + Period.parse("PT6H").toStandardDuration(), + Period.parse("P8D").toStandardDuration() + ); + + KillUnreferencedIndexingState duty = + new TestKillUnreferencedIndexingState(cleanupConfig, compactionStateStorage, dateTimes); + + // Insert compaction state + String fingerprint = "repair_fingerprint"; + CompactionState state = createTestCompactionState(); + + compactionStateStorage.upsertIndexingState("test-ds", fingerprint, state, DateTimes.nowUtc()); + compactionStateStorage.markIndexingStatesAsActive(List.of(fingerprint)); + + Assert.assertEquals(Boolean.TRUE, getCompactionStateUsedStatus(fingerprint)); + duty.run(); + Assert.assertEquals(Boolean.FALSE, getCompactionStateUsedStatus(fingerprint)); + + // Now insert a used segment that references this fingerprint + derbyConnector.retryWithHandle(handle -> { + handle.createStatement( + "INSERT INTO " + tablesConfig.getSegmentsTable() + " " + + "(id, dataSource, created_date, start, \"end\", partitioned, version, used, payload, " + + "used_status_last_updated, indexing_state_fingerprint) " + + "VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload, " + + ":used_status_last_updated, :indexing_state_fingerprint)" + ) + .bind("id", "testSegment_2024-01-01_2024-01-02_v1_0") + .bind("dataSource", "test-ds") + .bind("created_date", DateTimes.nowUtc().toString()) + .bind("start", "2024-01-01T00:00:00.000Z") + .bind("end", "2024-01-02T00:00:00.000Z") + .bind("partitioned", 0) + .bind("version", "v1") + .bind("used", true) + .bind("payload", new byte[]{}) + .bind("used_status_last_updated", DateTimes.nowUtc().toString()) + .bind("indexing_state_fingerprint", fingerprint) + .execute(); + return null; + }); + + // Confirm that the state is "repaired" now that it is referenced + duty.run(); + Assert.assertEquals(Boolean.TRUE, getCompactionStateUsedStatus(fingerprint)); + } + + @Test + public void test_killUnreferencedCompactionState_disabled() + { + IndexingStateCleanupConfig cleanupConfig = new IndexingStateCleanupConfig( + false, // cleanup disabled + Period.parse("PT1H").toStandardDuration(), + Period.parse("PT6H").toStandardDuration(), + Period.parse("P8D").toStandardDuration() + ); + + KillUnreferencedIndexingState duty = + new KillUnreferencedIndexingState(cleanupConfig, compactionStateStorage); + + // Insert compaction state + String fingerprint = "disabled_fingerprint"; + compactionStateStorage.upsertIndexingState("test-ds", fingerprint, createTestCompactionState(), DateTimes.nowUtc()); + compactionStateStorage.markIndexingStatesAsActive(List.of(fingerprint)); + + // Run duty - should do nothing + duty.run(); + + // Should still be used (not marked as unused since cleanup is disabled) + Assert.assertEquals(Boolean.TRUE, getCompactionStateUsedStatus(fingerprint)); + } + + @Test + public void test_killUnreferencedCompactionState_validateLifecycleOfPendingCompactionState() + { + List dateTimes = new ArrayList<>(); + DateTime now = DateTimes.nowUtc(); + dateTimes.add(now.plusDays(8)); + dateTimes.add(now.plusDays(8)); + dateTimes.add(now.plusDays(15)); + dateTimes.add(now.plusDays(15)); + + IndexingStateCleanupConfig cleanupConfig = new IndexingStateCleanupConfig( + true, + Period.parse("PT1H").toStandardDuration(), + Period.parse("P7D").toStandardDuration(), + Period.parse("P10D").toStandardDuration() // Pending states older than 10 days should be deleted + ); + + KillUnreferencedIndexingState duty = + new TestKillUnreferencedIndexingState(cleanupConfig, compactionStateStorage, dateTimes); + + String fingerprint = "pending_fingerprint"; + CompactionState state = createTestCompactionState(); + compactionStateStorage.upsertIndexingState("test-ds", fingerprint, state, DateTimes.nowUtc()); + + Assert.assertEquals(Boolean.TRUE, compactionStateStorage.isIndexingStatePending(fingerprint)); + + duty.run(); + Assert.assertNotNull(compactionStateStorage.isIndexingStatePending(fingerprint)); + + duty.run(); + Assert.assertNull(compactionStateStorage.isIndexingStatePending(fingerprint)); + } + + /** + * Validate multiple states cleaned up as per their individual retention policies. + */ + @Test + public void test_killUnreferencedCompactionState_validateMixedPendingAndActiveCompactionStateCleanup() + { + List dateTimes = new ArrayList<>(); + DateTime now = DateTimes.nowUtc(); + dateTimes.add(now.plusDays(8)); + dateTimes.add(now.plusDays(8)); + dateTimes.add(now.plusDays(31)); + dateTimes.add(now.plusDays(31)); + + IndexingStateCleanupConfig cleanupConfig = new IndexingStateCleanupConfig( + true, + Period.parse("PT1H").toStandardDuration(), + Period.parse("P7D").toStandardDuration(), + Period.parse("P30D").toStandardDuration() + ); + + KillUnreferencedIndexingState duty = + new TestKillUnreferencedIndexingState(cleanupConfig, compactionStateStorage, dateTimes); + + String pendingFingerprint = "pending_fp"; + String nonPendingFingerprint = "non_pending_fp"; + CompactionState state = createTestCompactionState(); + + compactionStateStorage.upsertIndexingState("test-ds", pendingFingerprint, state, DateTimes.nowUtc()); + compactionStateStorage.upsertIndexingState("test-ds", nonPendingFingerprint, state, DateTimes.nowUtc()); + compactionStateStorage.markIndexingStatesAsActive(List.of(nonPendingFingerprint)); + + Assert.assertEquals(Boolean.TRUE, compactionStateStorage.isIndexingStatePending(pendingFingerprint)); + Assert.assertNotNull(getCompactionStateUsedStatus(nonPendingFingerprint)); + + duty.run(); + Assert.assertNotNull(compactionStateStorage.isIndexingStatePending(pendingFingerprint)); + Assert.assertNull(getCompactionStateUsedStatus(nonPendingFingerprint)); + + duty.run(); + Assert.assertNull(getCompactionStateUsedStatus(nonPendingFingerprint)); + Assert.assertNull(compactionStateStorage.isIndexingStatePending(pendingFingerprint)); + } + + @Test + public void test_killUnreferencedCompactionState_pendingStateMarkedActiveNotDeleted() + { + List dateTimes = new ArrayList<>(); + DateTime now = DateTimes.nowUtc(); + dateTimes.add(now.plusDays(31)); // The state would be removed if it was still pending + dateTimes.add(now.plusDays(31)); // The state would be removed if it was still pending + + IndexingStateCleanupConfig cleanupConfig = new IndexingStateCleanupConfig( + true, + Period.parse("PT1H").toStandardDuration(), + Period.parse("P7D").toStandardDuration(), + Period.parse("P30D").toStandardDuration() + ); + + KillUnreferencedIndexingState duty = + new TestKillUnreferencedIndexingState(cleanupConfig, compactionStateStorage, dateTimes); + + String fingerprint = "pending_marked_active_fp"; + CompactionState state = createTestCompactionState(); + + compactionStateStorage.upsertIndexingState("test-ds", fingerprint, state, DateTimes.nowUtc()); + Assert.assertEquals(Boolean.TRUE, compactionStateStorage.isIndexingStatePending(fingerprint)); + + // Now insert a used segment that references this fingerprint + derbyConnector.retryWithHandle(handle -> { + handle.createStatement( + "INSERT INTO " + tablesConfig.getSegmentsTable() + " " + + "(id, dataSource, created_date, start, \"end\", partitioned, version, used, payload, " + + "used_status_last_updated, indexing_state_fingerprint) " + + "VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload, " + + ":used_status_last_updated, :indexing_state_fingerprint)" + ) + .bind("id", "testSegment_2024-01-01_2024-01-02_v1_0") + .bind("dataSource", "test-ds") + .bind("created_date", DateTimes.nowUtc().toString()) + .bind("start", "2024-01-01T00:00:00.000Z") + .bind("end", "2024-01-02T00:00:00.000Z") + .bind("partitioned", 0) + .bind("version", "v1") + .bind("used", true) + .bind("payload", new byte[]{}) + .bind("used_status_last_updated", DateTimes.nowUtc().toString()) + .bind("indexing_state_fingerprint", fingerprint) + .execute(); + return null; + }); + + compactionStateStorage.markIndexingStatesAsActive(List.of(fingerprint)); + Assert.assertNotEquals(Boolean.TRUE, compactionStateStorage.isIndexingStatePending(fingerprint)); + + duty.run(); + Assert.assertNotNull(compactionStateStorage.isIndexingStatePending(fingerprint)); + } + + private Boolean getCompactionStateUsedStatus(String fingerprint) + { + List usedStatus = derbyConnector.retryWithHandle( + handle -> handle.createQuery( + "SELECT used FROM " + tablesConfig.getIndexingStatesTable() + + " WHERE fingerprint = :fp" + ) + .bind("fp", fingerprint) + .mapTo(Boolean.class) + .list() + ); + + return usedStatus.isEmpty() ? null : usedStatus.get(0); + } + + /** + * Extension of KillUnreferencedIndexingState that allows controlling the reference time used for cleanup decisions. + *

+ * Allowing time control enables realistic testing of time-based retention logic. + */ + private static class TestKillUnreferencedIndexingState extends KillUnreferencedIndexingState + { + private final List dateTimes; + private int index = -1; + + public TestKillUnreferencedIndexingState( + IndexingStateCleanupConfig config, + IndexingStateStorage indexingStateStorage, + List dateTimes + ) + { + super(config, indexingStateStorage); + this.dateTimes = dateTimes; + } + + @Override + protected DateTime getCurrentTime() + { + index++; + return dateTimes.get(index); + } + } + + private CompactionState createTestCompactionState() + { + return new CompactionState( + new DynamicPartitionsSpec(100, null), + null, null, null, + IndexSpec.getDefault(), + null, null + ); + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java index a54459ad8891..3518e1dea409 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordCompactionResourceTest.java @@ -159,7 +159,7 @@ public void test_updateClusterConfig() replayAll(); Response response = compactionResource.updateClusterCompactionConfig( - new ClusterCompactionConfig(0.5, 10, null, true, CompactionEngine.MSQ), + new ClusterCompactionConfig(0.5, 10, null, true, CompactionEngine.MSQ, true), httpRequest ); Assert.assertEquals(200, response.getStatus()); @@ -170,7 +170,7 @@ public void test_updateClusterConfig() public void test_getClusterConfig() { final ClusterCompactionConfig clusterConfig = - new ClusterCompactionConfig(0.4, 100, null, true, CompactionEngine.MSQ); + new ClusterCompactionConfig(0.4, 100, null, true, CompactionEngine.MSQ, true); EasyMock.expect(configManager.getClusterCompactionConfig()) .andReturn(clusterConfig) .once(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java index 35cb09b3fe52..81fe5584b19d 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java @@ -120,6 +120,7 @@ import org.apache.druid.segment.loading.LocalDataSegmentPusher; import org.apache.druid.segment.loading.LocalDataSegmentPusherConfig; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.segment.realtime.NoopChatHandlerProvider; import org.apache.druid.segment.realtime.SegmentGenerationMetrics; @@ -636,7 +637,8 @@ protected void makeToolboxFactory(TestUtils testUtils, ServiceEmitter emitter, b derby.metadataTablesConfigSupplier().get(), derbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); taskLockbox = new GlobalTaskLockbox(taskStorage, metadataStorageCoordinator); final TaskActionToolbox taskActionToolbox = new TaskActionToolbox( diff --git a/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java index bf02ebebd82b..879f81778683 100644 --- a/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java +++ b/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java @@ -74,6 +74,7 @@ import org.apache.druid.indexing.common.actions.SegmentTransactionalReplaceAction; import org.apache.druid.indexing.common.actions.TaskAction; import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.task.AbstractBatchIndexTask; import org.apache.druid.indexing.common.task.Tasks; import org.apache.druid.indexing.common.task.batch.TooManyBucketsException; import org.apache.druid.indexing.common.task.batch.parallel.TombstoneHelper; @@ -1698,6 +1699,8 @@ private void handleQueryResults( Tasks.DEFAULT_STORE_COMPACTION_STATE ); + final String indexingStateFingerprint = querySpec.getContext().getString(Tasks.INDEXING_STATE_FINGERPRINT_KEY); + if (storeCompactionState) { DataSourceMSQDestination destination = (DataSourceMSQDestination) querySpec.getDestination(); if (!destination.isReplaceTimeChunks()) { @@ -1723,6 +1726,10 @@ private void handleQueryResults( ); } } + if (indexingStateFingerprint != null) { + compactionStateAnnotateFunction = compactionStateAnnotateFunction.andThen( + AbstractBatchIndexTask.addIndexingStateFingerprintToSegments(indexingStateFingerprint)); + } log.info("Query [%s] publishing %d segments.", queryDef.getQueryId(), segments.size()); publishAllSegments(segments, compactionStateAnnotateFunction); } else if (MSQControllerTask.isExport(querySpec.getDestination())) { diff --git a/multi-stage-query/src/main/java/org/apache/druid/msq/input/table/DataSegmentWithLocation.java b/multi-stage-query/src/main/java/org/apache/druid/msq/input/table/DataSegmentWithLocation.java index 10a0e257bccb..72e765d620dc 100644 --- a/multi-stage-query/src/main/java/org/apache/druid/msq/input/table/DataSegmentWithLocation.java +++ b/multi-stage-query/src/main/java/org/apache/druid/msq/input/table/DataSegmentWithLocation.java @@ -62,6 +62,7 @@ private DataSegmentWithLocation( @JsonProperty("size") long size, @JsonProperty("totalRows") Integer totalRows, @JsonProperty("servers") Set servers, + @JsonProperty("indexingStateFingerprint") String indexingStateFingerprint, @JacksonInject PruneSpecsHolder pruneSpecsHolder ) { @@ -78,6 +79,7 @@ private DataSegmentWithLocation( binaryVersion, size, totalRows, + indexingStateFingerprint, pruneSpecsHolder ); this.servers = Preconditions.checkNotNull(servers, "servers"); @@ -101,6 +103,7 @@ public DataSegmentWithLocation( dataSegment.getBinaryVersion(), dataSegment.getSize(), dataSegment.getTotalRows(), + dataSegment.getIndexingStateFingerprint(), PruneSpecsHolder.DEFAULT ); this.servers = servers; diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java index fc8e9e654bc3..b8179e463c18 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java @@ -200,7 +200,7 @@ public DimensionHandler getDimensionHandler() /** * Computes the 'effective' {@link DimensionSchema}, allowing columns which provide mechanisms for customizing storage - * format to fill in values from the segment level {@link IndexSpec} defaults. This is useful for comparising the + * format to fill in values from the segment level {@link IndexSpec} defaults. This is useful for comparing the * operator explicitly defined schema with the 'effective' schema that was written to the segments for things like * comparing compaction state. */ diff --git a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageConnector.java b/processing/src/main/java/org/apache/druid/metadata/MetadataStorageConnector.java index 1c185f38575b..f40b36efa171 100644 --- a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageConnector.java +++ b/processing/src/main/java/org/apache/druid/metadata/MetadataStorageConnector.java @@ -95,4 +95,12 @@ default void exportTable( * SegmentSchema table is created only when CentralizedDatasourceSchema feature is enabled. */ void createSegmentSchemasTable(); + + /** + * This table stores {@link org.apache.druid.timeline.CompactionState} objects. + *

+ * Multiple segments can refer to the same compaction state via its unique fingerprint + *

+ */ + void createIndexingStatesTable(); } diff --git a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java b/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java index 35915b52b70c..819b876154e0 100644 --- a/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java +++ b/processing/src/main/java/org/apache/druid/metadata/MetadataStorageTablesConfig.java @@ -32,7 +32,7 @@ public class MetadataStorageTablesConfig public static MetadataStorageTablesConfig fromBase(String base) { - return new MetadataStorageTablesConfig(base, null, null, null, null, null, null, null, null, null, null, null, null); + return new MetadataStorageTablesConfig(base, null, null, null, null, null, null, null, null, null, null, null, null, null); } private static final String DEFAULT_BASE = "druid"; @@ -76,6 +76,9 @@ public static MetadataStorageTablesConfig fromBase(String base) @JsonProperty("useShortIndexNames") private final boolean useShortIndexNames; + @JsonProperty("indexingStates") + private final String indexingStatesTable; + @JsonCreator public MetadataStorageTablesConfig( @JsonProperty("base") String base, @@ -90,7 +93,8 @@ public MetadataStorageTablesConfig( @JsonProperty("supervisors") String supervisorTable, @JsonProperty("upgradeSegments") String upgradeSegmentsTable, @JsonProperty("segmentSchemas") String segmentSchemasTable, - @JsonProperty("useShortIndexNames") Boolean useShortIndexNames + @JsonProperty("useShortIndexNames") Boolean useShortIndexNames, + @JsonProperty("indexingStates") String indexingStatesTable ) { this.base = (base == null) ? DEFAULT_BASE : base; @@ -107,6 +111,7 @@ public MetadataStorageTablesConfig( this.supervisorTable = makeTableName(supervisorTable, "supervisors"); this.segmentSchemasTable = makeTableName(segmentSchemasTable, "segmentSchemas"); this.useShortIndexNames = Configs.valueOrDefault(useShortIndexNames, false); + this.indexingStatesTable = makeTableName(indexingStatesTable, "indexingStates"); } private String makeTableName(String explicitTableName, String defaultSuffix) @@ -181,6 +186,11 @@ public String getSegmentSchemasTable() return segmentSchemasTable; } + public String getIndexingStatesTable() + { + return indexingStatesTable; + } + /** * If enabled, this causes table indices to be created with short, unique SHA-based identifiers. */ diff --git a/processing/src/main/java/org/apache/druid/timeline/DataSegment.java b/processing/src/main/java/org/apache/druid/timeline/DataSegment.java index 261fd8f23331..ad6406a1340e 100644 --- a/processing/src/main/java/org/apache/druid/timeline/DataSegment.java +++ b/processing/src/main/java/org/apache/druid/timeline/DataSegment.java @@ -115,6 +115,16 @@ public static class PruneSpecsHolder private final long size; private final Integer totalRows; + /** + * SHA-256 fingerprint representation of the CompactionState. + *

+ * A null fingerprint indicates that this segment either has not been compacted, or was compacted before indexing + * state fingerprinting existed. In the latter case, the segment would have a non-null {@link #lastCompactionState}. + *

+ */ + @Nullable + private final String indexingStateFingerprint; + /** * @deprecated use {@link #builder(SegmentId)} or {@link #builder(DataSegment)} instead. */ @@ -144,6 +154,7 @@ public DataSegment( binaryVersion, size, null, + null, PruneSpecsHolder.DEFAULT ); } @@ -178,6 +189,7 @@ public DataSegment( binaryVersion, size, null, + null, PruneSpecsHolder.DEFAULT ); } @@ -200,6 +212,7 @@ private DataSegment( @JsonProperty("binaryVersion") Integer binaryVersion, @JsonProperty("size") long size, @JsonProperty("totalRows") Integer totalRows, + @JsonProperty("indexingStateFingerprint") @Nullable String indexingStateFingerprint, @JacksonInject PruneSpecsHolder pruneSpecsHolder ) { @@ -216,6 +229,7 @@ private DataSegment( binaryVersion, size, totalRows, + indexingStateFingerprint, pruneSpecsHolder ); } @@ -233,6 +247,7 @@ public DataSegment( Integer binaryVersion, long size, Integer totalRows, + String indexingStateFingerprint, PruneSpecsHolder pruneSpecsHolder ) { @@ -252,6 +267,9 @@ public DataSegment( Preconditions.checkArgument(size >= 0); this.size = size; this.totalRows = totalRows; + this.indexingStateFingerprint = indexingStateFingerprint == null ? + null : + STRING_INTERNER.intern(indexingStateFingerprint); } /** @@ -354,6 +372,21 @@ public boolean isTombstone() return getShardSpec().getType().equals(ShardSpec.Type.TOMBSTONE); } + /** + * Get the inexing state fingerprint associated with this segment. + *

+ * A null fingerprint indicates that this segment either has not been compacted, or was compacted before compaction + * fingerprinting existed. In the latter case, the segment would have a non-null {@link #lastCompactionState}. + *

+ */ + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public String getIndexingStateFingerprint() + { + return indexingStateFingerprint; + } + @Override public boolean overshadows(DataSegment other) { @@ -448,6 +481,11 @@ public DataSegment withLastCompactionState(CompactionState compactionState) return builder(this).lastCompactionState(compactionState).build(); } + public DataSegment withIndexingStateFingerprint(String indexingStateFingerprint) + { + return builder(this).indexingStateFingerprint(indexingStateFingerprint).build(); + } + public DataSegment.Builder toBuilder() { return builder(this); @@ -488,6 +526,7 @@ public String toString() ", lastCompactionState=" + lastCompactionState + ", size=" + size + ", totalRows=" + totalRows + + ", indexingStateFingerprint=" + indexingStateFingerprint + '}'; } @@ -567,6 +606,7 @@ public static class Builder private Integer binaryVersion; private long size; private Integer totalRows; + private String indexingStateFingerprint; /** * @deprecated use {@link #Builder(SegmentId)} or {@link #Builder(DataSegment)} instead. @@ -594,6 +634,7 @@ private Builder(SegmentId segmentId) this.size = 0; this.totalRows = null; this.lastCompactionState = null; + this.indexingStateFingerprint = null; } private Builder(DataSegment segment) @@ -610,6 +651,7 @@ private Builder(DataSegment segment) this.binaryVersion = segment.getBinaryVersion(); this.size = segment.getSize(); this.totalRows = segment.getTotalRows(); + this.indexingStateFingerprint = segment.getIndexingStateFingerprint(); } private Builder(DataSegment.Builder segmentBuilder) @@ -626,6 +668,7 @@ private Builder(DataSegment.Builder segmentBuilder) this.binaryVersion = segmentBuilder.binaryVersion; this.size = segmentBuilder.size; this.totalRows = segmentBuilder.totalRows; + this.indexingStateFingerprint = segmentBuilder.indexingStateFingerprint; } public Builder dataSource(String dataSource) @@ -700,6 +743,12 @@ public Builder totalRows(Integer totalRows) return this; } + public Builder indexingStateFingerprint(String indexingStateFingerprint) + { + this.indexingStateFingerprint = indexingStateFingerprint; + return this; + } + public DataSegment build() { // Check stuff that goes into the id, at least. @@ -721,6 +770,7 @@ public DataSegment build() binaryVersion, size, totalRows, + indexingStateFingerprint, PruneSpecsHolder.DEFAULT ); } diff --git a/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageConnector.java b/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageConnector.java index d8722a2719f0..28291821e086 100644 --- a/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageConnector.java +++ b/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageConnector.java @@ -101,4 +101,10 @@ public void createSegmentSchemasTable() { throw new UnsupportedOperationException(); } + + @Override + public void createIndexingStatesTable() + { + throw new UnsupportedOperationException(); + } } diff --git a/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageTablesConfig.java b/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageTablesConfig.java index 784b7e2cad69..bbe82add29c9 100644 --- a/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageTablesConfig.java +++ b/processing/src/test/java/org/apache/druid/metadata/TestMetadataStorageTablesConfig.java @@ -39,6 +39,7 @@ public TestMetadataStorageTablesConfig() null, null, null, + null, null ); } diff --git a/processing/src/test/java/org/apache/druid/timeline/DataSegmentTest.java b/processing/src/test/java/org/apache/druid/timeline/DataSegmentTest.java index 756b743a7b9e..23e88564c8f3 100644 --- a/processing/src/test/java/org/apache/druid/timeline/DataSegmentTest.java +++ b/processing/src/test/java/org/apache/druid/timeline/DataSegmentTest.java @@ -462,5 +462,111 @@ public void testTombstoneType() Assert.assertFalse(segment3.isTombstone()); Assert.assertTrue(segment3.hasData()); + + } + + @Test + public void testSerializationWithIndexingStateFingerprint() throws Exception + { + final Interval interval = Intervals.of("2011-10-01/2011-10-02"); + final ImmutableMap loadSpec = ImmutableMap.of("something", "or_other"); + final String fingerprint = "abc123def456"; + final SegmentId segmentId = SegmentId.of("something", interval, "1", new NumberedShardSpec(3, 0)); + + DataSegment segment = DataSegment.builder(segmentId) + .loadSpec(loadSpec) + .dimensions(Arrays.asList("dim1", "dim2")) + .metrics(Arrays.asList("met1", "met2")) + .indexingStateFingerprint(fingerprint) + .binaryVersion(TEST_VERSION) + .size(1) + .build(); + + // Verify fingerprint is present in serialized JSON + final Map objectMap = MAPPER.readValue( + MAPPER.writeValueAsString(segment), + JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT + ); + Assert.assertEquals(fingerprint, objectMap.get("indexingStateFingerprint")); + + // Verify deserialization preserves fingerprint + DataSegment deserializedSegment = MAPPER.readValue(MAPPER.writeValueAsString(segment), DataSegment.class); + Assert.assertEquals(fingerprint, deserializedSegment.getIndexingStateFingerprint()); + Assert.assertEquals(segment.hashCode(), deserializedSegment.hashCode()); + } + + @Test + public void testSerializationWithNullIndexingStateFingerprint() throws Exception + { + final Interval interval = Intervals.of("2011-10-01/2011-10-02"); + final ImmutableMap loadSpec = ImmutableMap.of("something", "or_other"); + final SegmentId segmentId = SegmentId.of("something", interval, "1", new NumberedShardSpec(3, 0)); + + DataSegment segment = DataSegment.builder(segmentId) + .loadSpec(loadSpec) + .dimensions(Arrays.asList("dim1", "dim2")) + .metrics(Arrays.asList("met1", "met2")) + .indexingStateFingerprint(null) + .binaryVersion(TEST_VERSION) + .size(1) + .build(); + + // Verify fingerprint is NOT present in serialized JSON (due to @JsonInclude(NON_NULL)) + final Map objectMap = MAPPER.readValue( + MAPPER.writeValueAsString(segment), + JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT + ); + Assert.assertFalse("indexingStateFingerprint should not be in JSON when null", + objectMap.containsKey("indexingStateFingerprint")); + + // Verify deserialization handles missing fingerprint + DataSegment deserializedSegment = MAPPER.readValue(MAPPER.writeValueAsString(segment), DataSegment.class); + Assert.assertNull(deserializedSegment.getIndexingStateFingerprint()); + Assert.assertEquals(segment.hashCode(), deserializedSegment.hashCode()); + } + + @Test + public void testDeserializationBackwardCompatibility_missingIndexingStateFingerprint() throws Exception + { + // Simulate JSON from old Druid version without indexingStateFingerprint field + String jsonWithoutFingerprint = "{" + + "\"dataSource\": \"something\"," + + "\"interval\": \"2011-10-01T00:00:00.000Z/2011-10-02T00:00:00.000Z\"," + + "\"version\": \"1\"," + + "\"loadSpec\": {\"something\": \"or_other\"}," + + "\"dimensions\": \"dim1,dim2\"," + + "\"metrics\": \"met1,met2\"," + + "\"shardSpec\": {\"type\": \"numbered\", \"partitionNum\": 3, \"partitions\": 0}," + + "\"binaryVersion\": 9," + + "\"size\": 1" + + "}"; + + DataSegment deserializedSegment = MAPPER.readValue(jsonWithoutFingerprint, DataSegment.class); + Assert.assertNull("indexingStateFingerprint should be null for backward compatibility", + deserializedSegment.getIndexingStateFingerprint()); + Assert.assertEquals("something", deserializedSegment.getDataSource()); + Assert.assertEquals(Intervals.of("2011-10-01/2011-10-02"), deserializedSegment.getInterval()); + } + + @Test + public void testWithIndexingStateFingerprint() + { + final String fingerprint = "test_fingerprint_12345"; + final Interval interval = Intervals.of("2012-01-01/2012-01-02"); + final String version = DateTimes.of("2012-01-01T11:22:33.444Z").toString(); + final ShardSpec shardSpec = new NumberedShardSpec(7, 0); + final SegmentId segmentId = SegmentId.of("foo", interval, version, shardSpec); + + final DataSegment segment1 = DataSegment.builder(segmentId) + .size(0) + .indexingStateFingerprint(fingerprint) + .build(); + final DataSegment segment2 = DataSegment.builder(segmentId) + .size(0) + .build(); + + DataSegment withFingerprint = segment2.withIndexingStateFingerprint(fingerprint); + Assert.assertEquals(fingerprint, withFingerprint.getIndexingStateFingerprint()); + Assert.assertEquals(segment1, withFingerprint); } } diff --git a/processing/src/test/resources/test.runtime.properties b/processing/src/test/resources/test.runtime.properties index 4f713bc66ddf..3d6a07e1e78c 100644 --- a/processing/src/test/resources/test.runtime.properties +++ b/processing/src/test/resources/test.runtime.properties @@ -31,3 +31,4 @@ druid.metadata.storage.tables.upgradeSegments=jjj_upgradeSegments druid.query.segmentMetadata.defaultAnalysisTypes=["cardinality", "size"] druid.query.segmentMetadata.defaultHistory=P2W druid.metadata.storage.tables.segmentSchemas=kkk_segmentSchemas +druid.metadata.storage.tables.compactionStates=lll_compactionStates diff --git a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java index 4ff74ea1f448..4a72e01e7c46 100644 --- a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java +++ b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java @@ -56,6 +56,7 @@ import org.apache.druid.segment.SegmentSchemaMapping; import org.apache.druid.segment.SegmentUtils; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.IndexingStateStorage; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.server.http.DataSegmentPlus; @@ -111,6 +112,7 @@ public class IndexerSQLMetadataStorageCoordinator implements IndexerMetadataStor private final SegmentSchemaManager segmentSchemaManager; private final CentralizedDatasourceSchemaConfig centralizedDatasourceSchemaConfig; private final boolean schemaPersistEnabled; + private final IndexingStateStorage indexingStateStorage; private final SegmentMetadataTransactionFactory transactionFactory; @@ -121,7 +123,8 @@ public IndexerSQLMetadataStorageCoordinator( MetadataStorageTablesConfig dbTables, SQLMetadataConnector connector, SegmentSchemaManager segmentSchemaManager, - CentralizedDatasourceSchemaConfig centralizedDatasourceSchemaConfig + CentralizedDatasourceSchemaConfig centralizedDatasourceSchemaConfig, + IndexingStateStorage indexingStateStorage ) { this.transactionFactory = transactionFactory; @@ -133,6 +136,7 @@ public IndexerSQLMetadataStorageCoordinator( this.schemaPersistEnabled = centralizedDatasourceSchemaConfig.isEnabled() && !centralizedDatasourceSchemaConfig.isTaskSchemaPublishDisabled(); + this.indexingStateStorage = indexingStateStorage; } @LifecycleStart @@ -438,12 +442,12 @@ public SegmentPublishResult commitSegmentsAndMetadata( final String dataSource = segments.iterator().next().getDataSource(); try { - return inReadWriteDatasourceTransaction( + final SegmentPublishResult result = inReadWriteDatasourceTransaction( dataSource, transaction -> { // Try to update datasource metadata first if (startMetadata != null) { - final SegmentPublishResult result = updateDataSourceMetadataInTransaction( + final SegmentPublishResult metadataResult = updateDataSourceMetadataInTransaction( transaction, supervisorId, dataSource, @@ -452,8 +456,8 @@ public SegmentPublishResult commitSegmentsAndMetadata( ); // Do not proceed if the datasource metadata update failed - if (!result.isSuccess()) { - return result; + if (!metadataResult.isSuccess()) { + return metadataResult; } } @@ -462,6 +466,13 @@ public SegmentPublishResult commitSegmentsAndMetadata( ); } ); + + // Mark compaction state fingerprints as active after successful publish + if (result.isSuccess()) { + markIndexingStateFingerprintsAsActive(result.getSegments()); + } + + return result; } catch (CallbackFailedException e) { throw e; @@ -478,7 +489,7 @@ public SegmentPublishResult commitReplaceSegments( final String dataSource = verifySegmentsToCommit(replaceSegments); try { - return inReadWriteDatasourceTransaction( + final SegmentPublishResult result = inReadWriteDatasourceTransaction( dataSource, transaction -> { final Set segmentsToInsert = new HashSet<>(replaceSegments); @@ -520,6 +531,13 @@ public SegmentPublishResult commitReplaceSegments( ); } ); + + // Mark compaction state fingerprints as active after successful publish + if (result.isSuccess()) { + markIndexingStateFingerprintsAsActive(result.getSegments()); + } + + return result; } catch (CallbackFailedException e) { return SegmentPublishResult.fail(e.getMessage()); @@ -1213,7 +1231,7 @@ private SegmentPublishResult commitAppendSegmentsAndMetadataInTransaction( ); try { - return inReadWriteDatasourceTransaction( + final SegmentPublishResult result = inReadWriteDatasourceTransaction( dataSource, transaction -> { // Try to update datasource metadata first @@ -1254,6 +1272,13 @@ private SegmentPublishResult commitAppendSegmentsAndMetadataInTransaction( ); } ); + + // Mark compaction state fingerprints as active after successful publish + if (result.isSuccess()) { + markIndexingStateFingerprintsAsActive(result.getSegments()); + } + + return result; } catch (CallbackFailedException e) { throw e; @@ -1807,7 +1832,8 @@ protected Set insertSegments( usedSegments.contains(segment), segmentMetadata == null ? null : segmentMetadata.getSchemaFingerprint(), segmentMetadata == null ? null : segmentMetadata.getNumRows(), - null + null, + segment.getIndexingStateFingerprint() ); }).collect(Collectors.toSet()); @@ -1929,7 +1955,8 @@ private Set createNewIdsOfAppendSegmentsAfterReplace( null, oldSegmentMetadata.getSchemaFingerprint(), oldSegmentMetadata.getNumRows(), - upgradedFromSegmentId + upgradedFromSegmentId, + oldSegmentMetadata.getIndexingStateFingerprint() ) ); } @@ -2021,7 +2048,8 @@ private Set insertSegments( true, segmentMetadata == null ? null : segmentMetadata.getSchemaFingerprint(), segmentMetadata == null ? null : segmentMetadata.getNumRows(), - upgradedFromSegmentIdMap.get(segment.getId().toString()) + upgradedFromSegmentIdMap.get(segment.getId().toString()), + segment.getIndexingStateFingerprint() ); }).collect(Collectors.toSet()); @@ -2685,6 +2713,41 @@ public Map> retrieveUpgradedToSegmentIds( return upgradedToSegmentIds; } + /** + * Marks indexing state fingerprints as active (non-pending) for successfully published segments. + *

+ * Extracts unique indexing state fingerprints from the given segments and marks them as active + * in the inexing state storage. This is called after successful segment publishing to indicate + * that the indexing state is no longer pending and can be retained with the regular grace period. + * + * @param segments The segments that were successfully published + */ + private void markIndexingStateFingerprintsAsActive(Set segments) + { + if (segments == null || segments.isEmpty()) { + return; + } + + // Collect unique non-null indexing state fingerprints + final List fingerprints = segments.stream() + .map(DataSegment::getIndexingStateFingerprint) + .filter(fp -> fp != null && !fp.isEmpty()) + .distinct() + .collect(Collectors.toList()); + + try { + int rowsUpdated = indexingStateStorage.markIndexingStatesAsActive(fingerprints); + if (rowsUpdated > 0) { + log.info("Marked indexing states active for the following fingerprints: %s", fingerprints); + } + } + catch (Exception e) { + // Log but don't fail the overall operation - the fingerprint will stay pending + // and be cleaned up by the pending grace period + log.warn(e, "Failed to mark indexing states for the following fingerprints as active (Future segments publishes may remediate): %s", fingerprints); + } + } + /** * Performs a read-write transaction using the {@link SegmentMetadataTransactionFactory}, * which may use the segment metadata cache, if enabled and ready. diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java index 99fcfe0bc393..74829cddbfbf 100644 --- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java +++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataConnector.java @@ -229,6 +229,13 @@ protected boolean isRootCausePacketTooBigException(Throwable t) return false; } + /** + * Checks if the root cause of the given exception is a unique constraint violation. + * + * @return true if t is a unique constraint violation, false otherwise + */ + public abstract boolean isUniqueConstraintViolation(Throwable t); + /** * Creates the given table and indexes if the table doesn't already exist. */ @@ -355,6 +362,8 @@ public void createSegmentTable(final String tableName) columns.add("used BOOLEAN NOT NULL"); columns.add("payload %2$s NOT NULL"); columns.add("used_status_last_updated VARCHAR(255) NOT NULL"); + columns.add("indexing_state_fingerprint VARCHAR(255)"); + columns.add("upgraded_from_segment_id VARCHAR(255)"); if (centralizedDatasourceSchemaConfig.isEnabled()) { columns.add("schema_fingerprint VARCHAR(255)"); @@ -614,6 +623,8 @@ protected void alterSegmentTable() columnNameTypes.put("upgraded_from_segment_id", "VARCHAR(255)"); + columnNameTypes.put("indexing_state_fingerprint", "VARCHAR(255)"); + if (centralizedDatasourceSchemaConfig.isEnabled()) { columnNameTypes.put("schema_fingerprint", "VARCHAR(255)"); columnNameTypes.put("num_rows", "BIGINT"); @@ -1097,6 +1108,48 @@ public void createSegmentSchemasTable() } } + /** + * Creates the indexing states table for storing fingerprinted indexing states + *

+ * This table stores unique indexing states that are referenced by + * segments via fingerprints. + */ + public void createIndexingStatesTable(final String tableName) + { + createTable( + tableName, + ImmutableList.of( + StringUtils.format( + "CREATE TABLE %1$s (\n" + + " created_date VARCHAR(255) NOT NULL,\n" + + " dataSource VARCHAR(255) NOT NULL,\n" + + " fingerprint VARCHAR(255) NOT NULL,\n" + + " payload %2$s NOT NULL,\n" + + " used BOOLEAN NOT NULL,\n" + + " pending BOOLEAN NOT NULL,\n" + + " used_status_last_updated VARCHAR(255) NOT NULL,\n" + + " PRIMARY KEY (fingerprint)\n" + + ")", + tableName, getPayloadType() + ) + ) + ); + + createIndex( + tableName, + "IDX_%s_USED", + List.of("used", "used_status_last_updated") + ); + } + + @Override + public void createIndexingStatesTable() + { + if (config.get().isCreateTables()) { + createIndexingStatesTable(tablesConfigSupplier.get().getIndexingStatesTable()); + } + } + /** * Get the Set of the index on given table * @@ -1243,15 +1296,27 @@ private void validateSegmentsTable() (tableHasColumn(segmentsTables, "schema_fingerprint") && tableHasColumn(segmentsTables, "num_rows")); - if (tableHasColumn(segmentsTables, "used_status_last_updated") && schemaPersistenceRequirementMet) { - // do nothing - } else { + StringBuilder missingColumns = new StringBuilder(); + if (!tableHasColumn(segmentsTables, "used_status_last_updated")) { + missingColumns.append("used_status_last_updated, "); + } + if (!schemaPersistenceRequirementMet) { + missingColumns.append("schema_fingerprint, num_rows, "); + } + if (!tableHasColumn(segmentsTables, "indexing_state_fingerprint")) { + missingColumns.append("indexing_state_fingerprint, "); + } + + if (missingColumns.length() > 0) { throw new ISE( "Cannot start Druid as table[%s] has an incompatible schema." - + " Reason: One or all of these columns [used_status_last_updated, schema_fingerprint, num_rows] does not exist in table." + + " Reason: The following columns do not exist in the table: [%s]" + " See https://druid.apache.org/docs/latest/operations/upgrade-prep.html for more info on remediation.", - tablesConfigSupplier.get().getSegmentsTable() + tablesConfigSupplier.get().getSegmentsTable(), + missingColumns.substring(0, missingColumns.length() - 2) ); + } else { + // do nothing } } diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProvider.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProvider.java index 8491c5242267..f3542cdc6d0f 100644 --- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProvider.java +++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProvider.java @@ -77,6 +77,7 @@ public void start() connector.createSegmentSchemasTable(); connector.createSegmentTable(); connector.createUpgradeSegmentsTable(); + connector.createIndexingStatesTable(); } @Override diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java index 0eba529c7b4d..bf4e20c86a63 100644 --- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java +++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java @@ -40,11 +40,13 @@ import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.metadata.segment.cache.IndexingStateRecord; import org.apache.druid.metadata.segment.cache.SegmentSchemaRecord; import org.apache.druid.segment.SchemaPayload; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.server.http.DataSegmentPlus; +import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentTimeline; @@ -585,7 +587,7 @@ private CloseableIterator retrieveSegmentBatchById( final Query> query = handle.createQuery( StringUtils.format( "SELECT payload, used, schema_fingerprint, num_rows," - + " upgraded_from_segment_id, used_status_last_updated" + + " upgraded_from_segment_id, used_status_last_updated, indexing_state_fingerprint" + " FROM %s WHERE dataSource = :dataSource %s", dbTables.getSegmentsTable(), getParameterizedInConditionForColumn("id", segmentIds) ) @@ -607,7 +609,8 @@ private CloseableIterator retrieveSegmentBatchById( r.getBoolean(2), schemaFingerprint, numRows, - r.getString(5) + r.getString(5), + r.getString(7) ); } ) @@ -615,7 +618,7 @@ private CloseableIterator retrieveSegmentBatchById( } else { final Query> query = handle.createQuery( StringUtils.format( - "SELECT payload, used, upgraded_from_segment_id, used_status_last_updated, created_date" + "SELECT payload, used, upgraded_from_segment_id, used_status_last_updated, created_date, indexing_state_fingerprint" + " FROM %s WHERE dataSource = :dataSource %s", dbTables.getSegmentsTable(), getParameterizedInConditionForColumn("id", segmentIds) ) @@ -634,7 +637,8 @@ private CloseableIterator retrieveSegmentBatchById( r.getBoolean(2), null, null, - r.getString(3) + r.getString(3), + r.getString(6) ) ) .iterator(); @@ -1701,6 +1705,124 @@ private SegmentSchemaRecord mapToSchemaRecord(ResultSet resultSet) } } + /** + * Retrieves all unique indexing state fingerprints currently marked as used. + * This is used for delta syncs to determine which fingerprints are still active. + * + * @return Set of indexing state fingerprints + */ + public Set retrieveAllUsedIndexingStateFingerprints() + { + final String sql = StringUtils.format( + "SELECT fingerprint FROM %s WHERE used = true", + dbTables.getIndexingStatesTable() + ); + + return Set.copyOf( + handle.createQuery(sql) + .setFetchSize(connector.getStreamingFetchSize()) + .mapTo(String.class) + .list() + ); + } + + /** + * Retrieves all indexing states marked as used (full sync). + * + * @return List of IndexingStateRecord objects + */ + public List retrieveAllUsedIndexingStates() + { + final String sql = StringUtils.format( + "SELECT fingerprint, payload FROM %s WHERE used = true", + dbTables.getIndexingStatesTable() + ); + + return retrieveValidIndexingStateRecordsWithQuery(handle.createQuery(sql)); + } + + /** + * Retrieves indexing states for specific fingerprints (delta sync). + * Used to fetch only newly added indexing states. + * + * @param fingerprints Set of fingerprints to retrieve + * @return List of IndexingStateRecord objects + */ + public List retrieveIndexingStatesForFingerprints( + Set fingerprints + ) + { + final List> fingerprintBatches = Lists.partition( + List.copyOf(fingerprints), + MAX_INTERVALS_PER_BATCH + ); + + final List records = new ArrayList<>(); + for (List fingerprintBatch : fingerprintBatches) { + records.addAll( + retrieveBatchOfIndexingStates(fingerprintBatch) + ); + } + + return records; + } + + /** + * Retrieves a batch of indexing state records for the given fingerprints. + */ + private List retrieveBatchOfIndexingStates(List fingerprints) + { + final String sql = StringUtils.format( + "SELECT fingerprint, payload FROM %s" + + " WHERE used = true" + + " %s", + dbTables.getIndexingStatesTable(), + getParameterizedInConditionForColumn("fingerprint", fingerprints) + ); + + final Query> query = handle.createQuery(sql); + bindColumnValuesToQueryWithInCondition("fingerprint", fingerprints, query); + + return retrieveValidIndexingStateRecordsWithQuery(query); + } + + /** + * Executes the given query and maps results to valid IndexingStateRecord objects. + * Records that fail to parse are filtered out. + */ + private List retrieveValidIndexingStateRecordsWithQuery( + Query> query + ) + { + return query.setFetchSize(connector.getStreamingFetchSize()) + .map((index, r, ctx) -> mapToIndexingStateRecord(r)) + .list() + .stream() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + /** + * Tries to parse the fields of the result set into a {@link IndexingStateRecord}. + * + * @return null if an error occurred while parsing the result + */ + private IndexingStateRecord mapToIndexingStateRecord(ResultSet resultSet) + { + String fingerprint = null; + try { + fingerprint = resultSet.getString("fingerprint"); + return new IndexingStateRecord( + fingerprint, + jsonMapper.readValue(resultSet.getBytes("payload"), CompactionState.class) + ); + } + catch (Throwable t) { + log.error(t, "Could not read indexing state with fingerprint[%s]", fingerprint); + return null; + } + } + private ResultIterator getDataSegmentResultIterator(Query> sql) { return sql.map((index, r, ctx) -> JacksonUtils.readValue(jsonMapper, r.getBytes(2), DataSegment.class)) @@ -1722,6 +1844,7 @@ private ResultIterator getDataSegmentPlusResultIterator( used, null, null, + null, null ); } diff --git a/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransaction.java b/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransaction.java index 5c9938f8c5a3..4e144433260f 100644 --- a/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransaction.java +++ b/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransaction.java @@ -245,10 +245,10 @@ public int insertSegments(Set segments) segments, "INSERT INTO %1$s " + "(id, dataSource, created_date, start, %2$send%2$s, partitioned, " - + "version, used, payload, used_status_last_updated, upgraded_from_segment_id) " + + "version, used, payload, used_status_last_updated, upgraded_from_segment_id, indexing_state_fingerprint) " + "VALUES " + "(:id, :dataSource, :created_date, :start, :end, :partitioned, " - + ":version, :used, :payload, :used_status_last_updated, :upgraded_from_segment_id)" + + ":version, :used, :payload, :used_status_last_updated, :upgraded_from_segment_id, :indexing_state_fingerprint)" ); } @@ -261,11 +261,11 @@ public int insertSegmentsWithMetadata(Set segments) "INSERT INTO %1$s " + "(id, dataSource, created_date, start, %2$send%2$s, partitioned, " + "version, used, payload, used_status_last_updated, upgraded_from_segment_id, " - + "schema_fingerprint, num_rows) " + + "schema_fingerprint, num_rows, indexing_state_fingerprint) " + "VALUES " + "(:id, :dataSource, :created_date, :start, :end, :partitioned, " + ":version, :used, :payload, :used_status_last_updated, :upgraded_from_segment_id, " - + ":schema_fingerprint, :num_rows)" + + ":schema_fingerprint, :num_rows, :indexing_state_fingerprint)" ); } @@ -532,7 +532,8 @@ private int insertSegmentsInBatches( .bind("used", Boolean.TRUE.equals(segmentPlus.getUsed())) .bind("payload", getJsonBytes(segment)) .bind("used_status_last_updated", toNonNullString(segmentPlus.getUsedStatusLastUpdatedDate())) - .bind("upgraded_from_segment_id", segmentPlus.getUpgradedFromSegmentId()); + .bind("upgraded_from_segment_id", segmentPlus.getUpgradedFromSegmentId()) + .bind("indexing_state_fingerprint", segmentPlus.getIndexingStateFingerprint()); if (persistAdditionalMetadata) { preparedBatchPart diff --git a/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java b/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java index 2a1cf50133e4..0b85243c8f25 100644 --- a/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java @@ -51,8 +51,10 @@ import org.apache.druid.query.DruidMetrics; import org.apache.druid.segment.SchemaPayload; import org.apache.druid.segment.SegmentMetadata; +import org.apache.druid.segment.metadata.IndexingStateCache; import org.apache.druid.segment.metadata.SegmentSchemaCache; import org.apache.druid.server.http.DataSegmentPlus; +import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.joda.time.DateTime; @@ -137,6 +139,9 @@ private enum CacheState private final boolean useSchemaCache; private final SegmentSchemaCache segmentSchemaCache; + private final boolean useIndexingStateCache; + private final IndexingStateCache indexingStateCache; + private final ListeningScheduledExecutorService pollExecutor; private final ServiceEmitter emitter; @@ -168,6 +173,7 @@ public HeapMemorySegmentMetadataCache( Supplier config, Supplier tablesConfig, SegmentSchemaCache segmentSchemaCache, + IndexingStateCache indexingStateCache, SQLMetadataConnector connector, ScheduledExecutorFactory executorFactory, ServiceEmitter emitter @@ -179,6 +185,8 @@ public HeapMemorySegmentMetadataCache( this.tablesConfig = tablesConfig.get(); this.useSchemaCache = segmentSchemaCache.isEnabled(); this.segmentSchemaCache = segmentSchemaCache; + this.useIndexingStateCache = indexingStateCache.isEnabled(); + this.indexingStateCache = indexingStateCache; this.connector = connector; this.pollExecutor = isEnabled() ? MoreExecutors.listeningDecorator(executorFactory.create(1, "SegmentMetadataCache-%s")) @@ -232,6 +240,9 @@ public void stop() datasourceToSegmentCache.forEach((datasource, cache) -> cache.stop()); datasourceToSegmentCache.clear(); datasourcesSnapshot.set(null); + if (useIndexingStateCache) { + indexingStateCache.clear(); + } syncFinishTime.set(null); updateCacheState(CacheState.STOPPED, "Stopped sync with metadata store"); @@ -576,6 +587,10 @@ private long syncWithMetadataStore() retrieveAndResetUsedSegmentSchemas(datasourceToSummary); } + if (useIndexingStateCache) { + retrieveAndResetUsedIndexingStates(); + } + markCacheSynced(syncStartTime); syncFinishTime.set(DateTimes.nowUtc()); @@ -785,13 +800,13 @@ private void retrieveAllUsedSegments( final String sql; if (useSchemaCache) { sql = StringUtils.format( - "SELECT id, payload, created_date, used_status_last_updated, schema_fingerprint, num_rows" + "SELECT id, payload, created_date, used_status_last_updated, indexing_state_fingerprint, schema_fingerprint, num_rows" + " FROM %s WHERE used = true", tablesConfig.getSegmentsTable() ); } else { sql = StringUtils.format( - "SELECT id, payload, created_date, used_status_last_updated" + "SELECT id, payload, created_date, used_status_last_updated, indexing_state_fingerprint" + " FROM %s WHERE used = true", tablesConfig.getSegmentsTable() ); @@ -1071,9 +1086,10 @@ private DataSegmentPlus mapToSegmentPlus(ResultSet resultSet) DateTimes.of(resultSet.getString(3)), SqlSegmentsMetadataQuery.nullAndEmptySafeDate(resultSet.getString(4)), true, - useSchemaCache ? resultSet.getString(5) : null, - useSchemaCache ? (Long) resultSet.getObject(6) : null, - null + useSchemaCache ? resultSet.getString(6) : null, + useSchemaCache ? (Long) resultSet.getObject(7) : null, + null, + resultSet.getString(5) ); } catch (Throwable t) { @@ -1106,6 +1122,89 @@ private void emitMetric(String datasource, String metric, long value) ); } + /** + * Retrieves required used indexing states from the metadata store and resets + * them in the {@link IndexingStateCache}. If this is the first sync, all used + * indexing states are retrieved from the metadata store. If this is a delta sync, + * first only the fingerprints of all used indexing states are retrieved. Payloads are + * then fetched for only the fingerprints which are not present in the cache. + */ + private void retrieveAndResetUsedIndexingStates() + { + final Stopwatch indexingStateSyncDuration = Stopwatch.createStarted(); + + // Reset the IndexingStateCache with latest indexing states + final Map fingerprintToStateMap; + if (syncFinishTime.get() == null) { + fingerprintToStateMap = buildIndexingStateFingerprintToStateMapForFullSync(); + } else { + fingerprintToStateMap = buildIndexingStateFingerprintToStateMapForDeltaSync(); + } + + indexingStateCache.resetIndexingStatesForPublishedSegments(fingerprintToStateMap); + + // Emit metrics for the current contents of the cache + indexingStateCache.getAndResetStats().forEach(this::emitMetric); + emitMetric(Metric.RETRIEVE_INDEXING_STATES_DURATION_MILLIS, indexingStateSyncDuration.millisElapsed()); + } + + /** + * Retrieves all used indexing states from the metadata store and builds a + * fresh map from indexing state fingerprint to state. + */ + private Map buildIndexingStateFingerprintToStateMapForFullSync() + { + final List records = query( + SqlSegmentsMetadataQuery::retrieveAllUsedIndexingStates + ); + + return records.stream().collect( + Collectors.toMap( + IndexingStateRecord::getFingerprint, + IndexingStateRecord::getState + ) + ); + } + + /** + * Retrieves indexing states from the metadata store if they are not present + * in the cache or have been recently updated in the metadata store. These + * indexing states along with those already present in the cache are used to + * build a complete updated map from indexing state fingerprint to state. + * + * @return Complete updated map from indexing state fingerprint to state for all + * used indexing states currently persisted in the metadata store. + */ + private Map buildIndexingStateFingerprintToStateMapForDeltaSync() + { + // Identify fingerprints in the cache and in the metadata store + final Map fingerprintToStateMap = new HashMap<>( + indexingStateCache.getPublishedIndexingStateMap() + ); + final Set cachedFingerprints = Set.copyOf(fingerprintToStateMap.keySet()); + final Set persistedFingerprints = query( + SqlSegmentsMetadataQuery::retrieveAllUsedIndexingStateFingerprints + ); + + // Remove entry for indexing states that have been deleted from the metadata store + final Set deletedFingerprints = Sets.difference(cachedFingerprints, persistedFingerprints); + deletedFingerprints.forEach(fingerprintToStateMap::remove); + emitMetric(Metric.DELETED_INDEXING_STATES, deletedFingerprints.size()); + + // Retrieve and add entry for indexing states that have been added to the metadata store + final Set addedFingerprints = Sets.difference(persistedFingerprints, cachedFingerprints); + final List addedIndexingStateRecords = query( + sql -> sql.retrieveIndexingStatesForFingerprints(addedFingerprints) + ); + + addedIndexingStateRecords.forEach( + record -> fingerprintToStateMap.put(record.getFingerprint(), record.getState()) + ); + emitMetric(Metric.ADDED_INDEXING_STATES, addedIndexingStateRecords.size()); + + return fingerprintToStateMap; + } + /** * Summary of segments currently present in the metadata store for a single * datasource. diff --git a/server/src/main/java/org/apache/druid/metadata/segment/cache/IndexingStateRecord.java b/server/src/main/java/org/apache/druid/metadata/segment/cache/IndexingStateRecord.java new file mode 100644 index 000000000000..b04484d87d58 --- /dev/null +++ b/server/src/main/java/org/apache/druid/metadata/segment/cache/IndexingStateRecord.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.metadata.segment.cache; + +import org.apache.druid.timeline.CompactionState; + +/** + * Represents a single record in the druid_indexingStates table. + */ +public class IndexingStateRecord +{ + private final String fingerprint; + private final CompactionState state; + + public IndexingStateRecord(String fingerprint, CompactionState state) + { + this.fingerprint = fingerprint; + this.state = state; + } + + public String getFingerprint() + { + return fingerprint; + } + + public CompactionState getState() + { + return state; + } +} diff --git a/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java b/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java index c8e87ca4d55d..7ea59b9bf695 100644 --- a/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java +++ b/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java @@ -110,6 +110,11 @@ private Metric() */ public static final String RETRIEVE_SEGMENT_SCHEMAS_DURATION_MILLIS = METRIC_NAME_PREFIX + "fetchSchemas/time"; + /** + * Time taken in milliseconds to fetch all indexing states from the metadata store. + */ + public static final String RETRIEVE_INDEXING_STATES_DURATION_MILLIS = METRIC_NAME_PREFIX + "fetchIndexingStates/time"; + /** * Time taken to update the datasource snapshot in the cache. */ @@ -158,6 +163,16 @@ private Metric() */ public static final String SKIPPED_SEGMENT_SCHEMAS = METRIC_NAME_PREFIX + "schema/skipped"; + /** + * Number of indexing states added to the cache in the latest sync. + */ + public static final String ADDED_INDEXING_STATES = METRIC_NAME_PREFIX + "indexingState/added"; + + /** + * Number of indexing states deleted from the cache in the latest sync. + */ + public static final String DELETED_INDEXING_STATES = METRIC_NAME_PREFIX + "indexingState/deleted"; + /** * Number of unparseable pending segment records skipped while refreshing the cache. */ diff --git a/server/src/main/java/org/apache/druid/metadata/storage/derby/DerbyConnector.java b/server/src/main/java/org/apache/druid/metadata/storage/derby/DerbyConnector.java index 97081d47ba3e..cb4815d2313e 100644 --- a/server/src/main/java/org/apache/druid/metadata/storage/derby/DerbyConnector.java +++ b/server/src/main/java/org/apache/druid/metadata/storage/derby/DerbyConnector.java @@ -212,6 +212,25 @@ public Boolean withHandle(Handle handle) ); } + @Override + public boolean isUniqueConstraintViolation(Throwable t) + { + Throwable cause = t; + while (cause != null) { + if (cause instanceof SQLException) { + SQLException sqlException = (SQLException) cause; + String sqlState = sqlException.getSQLState(); + + // SQL standard unique constraint violation code is 23505 for Derby + if ("23505".equals(sqlState)) { + return true; + } + } + cause = cause.getCause(); + } + return false; + } + @LifecycleStart public void start() { diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DefaultIndexingStateFingerprintMapper.java b/server/src/main/java/org/apache/druid/segment/metadata/DefaultIndexingStateFingerprintMapper.java new file mode 100644 index 000000000000..8b81b0e5cade --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/DefaultIndexingStateFingerprintMapper.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.timeline.CompactionState; + +import java.util.Optional; + +/** + * Default implementation of {@link IndexingStateFingerprintMapper} that delegates to + * {@link IndexingStateStorage} for fingerprint generation and {@link IndexingStateCache} + * for state lookups. + */ +public class DefaultIndexingStateFingerprintMapper implements IndexingStateFingerprintMapper +{ + private final IndexingStateCache indexingStateCache; + private final ObjectMapper deterministicMapper; + + public DefaultIndexingStateFingerprintMapper( + IndexingStateCache indexingStateCache, + ObjectMapper jsonMapper + ) + { + this.indexingStateCache = indexingStateCache; + this.deterministicMapper = createDeterministicMapper(jsonMapper); + } + + @SuppressWarnings("UnstableApiUsage") + @Override + public String generateFingerprint(String dataSource, CompactionState indexingState) + { + final Hasher hasher = Hashing.sha256().newHasher(); + + hasher.putBytes(StringUtils.toUtf8(dataSource)); + hasher.putByte((byte) 0xff); + + try { + hasher.putBytes(deterministicMapper.writeValueAsBytes(indexingState)); + } + catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize CompactionState object for fingerprinting", e); + } + hasher.putByte((byte) 0xff); + + return BaseEncoding.base16().encode(hasher.hash().asBytes()); + } + + @Override + public Optional getStateForFingerprint(String fingerprint) + { + return indexingStateCache.getIndexingStateByFingerprint(fingerprint); + } + + /** + * Decorate the provided {@link ObjectMapper} to ensure deterministic serialization of IndexingState objects. + */ + private static ObjectMapper createDeterministicMapper(ObjectMapper baseMapper) + { + ObjectMapper sortedMapper = baseMapper.copy(); + sortedMapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + sortedMapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); + return sortedMapper; + } +} diff --git a/server/src/main/java/org/apache/druid/segment/metadata/IndexingStateCache.java b/server/src/main/java/org/apache/druid/segment/metadata/IndexingStateCache.java new file mode 100644 index 000000000000..900276e91fa8 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/IndexingStateCache.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.timeline.CompactionState; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * In-memory cache of indexing states used by {@link org.apache.druid.metadata.segment.cache.HeapMemorySegmentMetadataCache}. + *

+ * This cache stores indexing states for published segments polled from the metadata store. + * It is the primary way to read indexing states in production. + *

+ * The cache is populated during segment metadata cache sync operations and provides fast lookups + * without hitting the database. + */ +@LazySingleton +public class IndexingStateCache +{ + private static final Logger log = new Logger(IndexingStateCache.class); + + /** + * Atomically updated reference to published indexing states. + */ + private final AtomicReference publishedIndexingStates + = new AtomicReference<>(PublishedIndexingStates.EMPTY); + + private final AtomicInteger cacheMissCount = new AtomicInteger(0); + private final AtomicInteger cacheHitCount = new AtomicInteger(0); + + public boolean isEnabled() + { + // Always enabled when this implementation is bound + return true; + } + + /** + * Resets the cache with indexing states polled from the metadata store. + * Called after each successful poll in HeapMemorySegmentMetadataCache. + * + * @param fingerprintToStateMap Complete fp:state map of all active indexing state fingerprints + */ + public void resetIndexingStatesForPublishedSegments( + Map fingerprintToStateMap + ) + { + this.publishedIndexingStates.set( + new PublishedIndexingStates(fingerprintToStateMap) + ); + log.debug("Reset indexing state cache with [%d] fingerprints", fingerprintToStateMap.size()); + } + + /** + * Retrieves an indexing state by its fingerprint. + * This is the indexing method for reading indexing states. + * + * @param fingerprint The fingerprint to look up + * @return The cached indexing state, or Optional.empty() if not cached + */ + public Optional getIndexingStateByFingerprint(String fingerprint) + { + if (fingerprint == null) { + return Optional.empty(); + } + + CompactionState state = publishedIndexingStates.get() + .fingerprintToStateMap + .get(fingerprint); + if (state == null) { + cacheMissCount.incrementAndGet(); + return Optional.empty(); + } else { + cacheHitCount.incrementAndGet(); + return Optional.of(state); + } + } + + /** + * Adds or updates a single indexing state in the cache. + *

+ * This is called when a new indexing state is persisted to the database via upsertIndexingState + * to ensure the cache is immediately consistent without waiting for the next sync. + *

+ * This method checks if the state is already cached before performing the atomic update. + * + * @param fingerprint The fingerprint key + * @param state The indexing state to cache + */ + public void addIndexingState(String fingerprint, CompactionState state) + { + if (fingerprint == null || state == null) { + return; + } + + // Check if the state is already cached - avoid expensive update if not needed + CompactionState existing = publishedIndexingStates.get() + .fingerprintToStateMap + .get(fingerprint); + if (state.equals(existing)) { + log.debug("Indexing state for fingerprint[%s] already cached, skipping update", fingerprint); + return; + } + + // State is not cached or different - perform atomic update + publishedIndexingStates.updateAndGet(current -> { + // Double-check in case another thread updated between our check and now + if (state.equals(current.fingerprintToStateMap.get(fingerprint))) { + return current; + } + + Map newMap = new HashMap<>(current.fingerprintToStateMap); + newMap.put(fingerprint, state); + return new PublishedIndexingStates(newMap); + }); + + log.debug("Added indexing state to cache for fingerprint[%s]", fingerprint); + } + + /** + * Gets the full cached map (immutable copy). + * Used by HeapMemorySegmentMetadataCache for delta sync calculations. + */ + public Map getPublishedIndexingStateMap() + { + return publishedIndexingStates.get().fingerprintToStateMap; + } + + /** + * Clears the cache. Called when node stops being leader. + */ + public void clear() + { + publishedIndexingStates.set(PublishedIndexingStates.EMPTY); + resetStats(); + } + + /** + * @return Summary stats for metric emission + */ + public Map getAndResetStats() + { + return Map.of( + Metric.INDEXING_STATE_CACHE_HITS, cacheHitCount.getAndSet(0), + Metric.INDEXING_STATE_CACHE_MISSES, cacheMissCount.getAndSet(0), + Metric.INDEXING_STATE_CACHE_FINGERPRINTS, + publishedIndexingStates.get().fingerprintToStateMap.size() + ); + } + + /** + * Resets hit/miss stats. + */ + private void resetStats() + { + cacheHitCount.set(0); + cacheMissCount.set(0); + } + + /** + * Immutable snapshot of indexing states polled from DB. + */ + private static class PublishedIndexingStates + { + private static final PublishedIndexingStates EMPTY = + new PublishedIndexingStates(Map.of()); + + private final Map fingerprintToStateMap; + + private PublishedIndexingStates(Map fingerprintToStateMap) + { + this.fingerprintToStateMap = Map.copyOf(fingerprintToStateMap); + } + } +} diff --git a/server/src/main/java/org/apache/druid/segment/metadata/IndexingStateFingerprintMapper.java b/server/src/main/java/org/apache/druid/segment/metadata/IndexingStateFingerprintMapper.java new file mode 100644 index 000000000000..dc8760010b25 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/IndexingStateFingerprintMapper.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import org.apache.druid.timeline.CompactionState; + +import java.util.Optional; + +/** + * Provides operations for mapping between indexing state fingerprints and their corresponding states. + *

+ * This interface abstracts the fingerprint generation and lookup operations, simplifying + * dependencies and improving testability for classes that need both operations. + */ +public interface IndexingStateFingerprintMapper +{ + /** + * Generates a deterministic fingerprint for the given indexing state and datasource. + *

+ * The fingerprint is a SHA-256 hash of the datasource name and serialized indexing state that is globally unique in + * the segment space. + * + * @param dataSource The datasource name + * @param indexingState The compaction configuration to fingerprint + * @return A hex-encoded SHA-256 fingerprint string + */ + String generateFingerprint(String dataSource, CompactionState indexingState); + + /** + * Retrieves a compaction state by its fingerprint. + * + * @param fingerprint The fingerprint to look up + * @return The compaction state, or Optional.empty() if not found + */ + Optional getStateForFingerprint(String fingerprint); +} diff --git a/server/src/main/java/org/apache/druid/segment/metadata/IndexingStateStorage.java b/server/src/main/java/org/apache/druid/segment/metadata/IndexingStateStorage.java new file mode 100644 index 000000000000..926553563858 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/IndexingStateStorage.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import org.apache.druid.timeline.CompactionState; +import org.joda.time.DateTime; + +import java.util.List; + +/** + * Manages indexing state persistence and fingerprint generation. + *

+ * Implementations may be backed by a database (like {@link SqlIndexingStateStorage}) or + * use in-memory storage (like {@link HeapMemoryIndexingStateStorage}). + */ +public interface IndexingStateStorage +{ + /** + * Upserts an indexing state to storage. + *

+ * If a fingerprint already exists, update to reflect proper used state and timestamp. + * If a fingerprint doesn't exist, inserts a new row with the full state payload. + * + * @param dataSource The datasource name + * @param fingerprint The fingerprint of the indexing state + * @param indexingState The indexing state to upsert + * @param updateTime The timestamp for this update + */ + + void upsertIndexingState( + String dataSource, + String fingerprint, + CompactionState indexingState, + DateTime updateTime + ); + + /** + * Marks indexing states as unused if they are not referenced by any used segments. + *

+ * This is used for cleanup operations. + * + * @return Number of rows updated, or 0 if not applicable + */ + int markUnreferencedIndexingStatesAsUnused(); + + /** + * Finds all indexing state fingerprints which have been marked as unused but are + * still referenced by some used segments. This is used for validation/reconciliation. + * Implementations may return an empty list if not applicable. + * + * @return List of fingerprints, or empty list + */ + List findReferencedIndexingStateMarkedAsUnused(); + + /** + * Marks indexing states as used. + *

+ * This is used for reconciliation operations to avoid deleting states that are still in use. + * + * @param stateFingerprints List of fingerprints to mark as used + * @return Number of rows updated, or 0 if not applicable + */ + int markIndexingStatesAsUsed(List stateFingerprints); + + /** + * Marks indexing states as active + * + * @param stateFingerprints List of fingerprints to mark as active + * @return Number of rows updated, or 0 if not applicable + */ + int markIndexingStatesAsActive(List stateFingerprints); + + /** + * Deletes pending indexing states older than the given timestamp. + * @param timestamp The cutoff timestamp in milliseconds + * @return Number of rows deleted, or 0 if not applicable + */ + int deletePendingIndexingStatesOlderThan(long timestamp); + + /** + * Deletes unused indexing states older than the given timestamp. + *

+ * This is used for cleanup operations. + * + * @param timestamp The cutoff timestamp in milliseconds + * @return Number of rows deleted, or 0 if not applicable + */ + int deleteUnusedIndexingStatesOlderThan(long timestamp); +} diff --git a/server/src/main/java/org/apache/druid/segment/metadata/Metric.java b/server/src/main/java/org/apache/druid/segment/metadata/Metric.java index 3f1babbbde4a..a29f4a7eafff 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/Metric.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/Metric.java @@ -20,11 +20,12 @@ package org.apache.druid.segment.metadata; /** - * Metrics related to {@link SegmentSchemaCache} and {@link SegmentSchemaManager}. + * Metrics related to {@link SegmentSchemaCache}, {@link SegmentSchemaManager}, and {@link IndexingStateCache}. */ public class Metric { private static final String PREFIX = "segment/schemaCache/"; + private static final String INDEXING_STATE_PREFIX = "segment/indexingStateCache/"; public static final String CACHE_MISSES = "miss/count"; @@ -57,4 +58,9 @@ public class Metric * Number of used cold segments in the metadata store. */ public static final String USED_COLD_SEGMENTS = "segment/used/deepStorageOnly/count"; + + // Compaction state cache metrics + public static final String INDEXING_STATE_CACHE_HITS = INDEXING_STATE_PREFIX + "hit/count"; + public static final String INDEXING_STATE_CACHE_MISSES = INDEXING_STATE_PREFIX + "miss/count"; + public static final String INDEXING_STATE_CACHE_FINGERPRINTS = INDEXING_STATE_PREFIX + "fingerprint/count"; } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/NoopIndexingStateCache.java b/server/src/main/java/org/apache/druid/segment/metadata/NoopIndexingStateCache.java new file mode 100644 index 000000000000..36bf8b2f260c --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/NoopIndexingStateCache.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import org.apache.druid.timeline.CompactionState; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +/** + * No-op implementation of {@link IndexingStateCache} + */ +public class NoopIndexingStateCache extends IndexingStateCache +{ + @Override + public boolean isEnabled() + { + return false; + } + + @Override + public void resetIndexingStatesForPublishedSegments(Map fingerprintToStateMap) + { + // No-op + } + + @Override + public Optional getIndexingStateByFingerprint(String fingerprint) + { + return Optional.empty(); + } + + @Override + public Map getPublishedIndexingStateMap() + { + return Collections.emptyMap(); + } + + @Override + public void clear() + { + // No-op + } + + @Override + public Map getAndResetStats() + { + return Collections.emptyMap(); + } +} diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SqlIndexingStateStorage.java b/server/src/main/java/org/apache/druid/segment/metadata/SqlIndexingStateStorage.java new file mode 100644 index 000000000000..4232d6b67455 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/SqlIndexingStateStorage.java @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.InternalServerError; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.guice.annotations.Json; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.metadata.MetadataStorageTablesConfig; +import org.apache.druid.metadata.SQLMetadataConnector; +import org.apache.druid.timeline.CompactionState; +import org.joda.time.DateTime; +import org.skife.jdbi.v2.Handle; +import org.skife.jdbi.v2.SQLStatement; +import org.skife.jdbi.v2.Update; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.validation.constraints.NotEmpty; +import java.util.ArrayList; +import java.util.List; + +/** + * Database-backed implementation of {@link IndexingStateStorage}. + *

+ * Manages the persistence and retrieval of {@link CompactionState} (AKA IndexingState) objects in the metadata storage. + * Indexing states are uniquely identified by their fingerprints, which are SHA-256 hashes of their content. + *

+ *

+ * This implementation is designed to be called from a single thread and relies on + * database constraints and the retry mechanism to handle any conflicts. Operations are idempotent - concurrent + * upserts for the same fingerprint will either succeed or fail with a constraint violation that is safely ignored. + *

+ */ +@LazySingleton +public class SqlIndexingStateStorage implements IndexingStateStorage +{ + private static final EmittingLogger log = new EmittingLogger(SqlIndexingStateStorage.class); + + private final MetadataStorageTablesConfig dbTables; + private final ObjectMapper jsonMapper; + private final SQLMetadataConnector connector; + + @Inject + public SqlIndexingStateStorage( + MetadataStorageTablesConfig dbTables, + @Json ObjectMapper jsonMapper, + SQLMetadataConnector connector + ) + { + this.dbTables = dbTables; + this.jsonMapper = jsonMapper; + this.connector = connector; + } + + @Override + public void upsertIndexingState( + @NotEmpty final String dataSource, + @NotEmpty final String fingerprint, + @Nonnull final CompactionState indexingState, + @Nonnull final DateTime updateTime + ) + { + // Strictly sanitize inputs to avoid writing junk data to the rdbms + StringBuilder errors = new StringBuilder(); + if (dataSource == null || dataSource.isEmpty()) { + errors.append("dataSource cannot be empty; "); + } + if (fingerprint == null || fingerprint.isEmpty()) { + errors.append("fingerprint cannot be empty; "); + } + if (indexingState == null) { + errors.append("indexingState cannot be null; "); + } + if (updateTime == null) { + errors.append("updateTime cannot be null; "); + } + if (errors.length() > 0) { + throw DruidException.forPersona(DruidException.Persona.DEVELOPER) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build(errors.toString().trim()); + } + + try { + connector.retryWithHandle(handle -> { + // Check if the fingerprint already exists and its used status + final FingerprintState state = getFingerprintState(handle, fingerprint); + final String now = updateTime.toString(); + + switch (state) { + case EXISTS_AND_USED: + // Fingerprint exists and is already marked as used - no operation needed + log.debug( + "Indexing state for fingerprint[%s] in dataSource[%s] already exists and is marked as used. Skipping update.", + fingerprint, + dataSource + ); + break; + + case EXISTS_AND_UNUSED: + updateExistingUnusedState(handle, fingerprint, dataSource, now); + break; + + case DOES_NOT_EXIST: + insertNewState(handle, fingerprint, dataSource, indexingState, now); + break; + + default: + throw new IllegalStateException("Unknown fingerprint state: " + state); + } + return null; + }); + } + catch (Throwable e) { + if (connector.isUniqueConstraintViolation(e)) { + // Swallow exception - another thread already persisted the same data + log.info( + "Fingerprints already exist for datasource[%s] (likely concurrent insert). " + + "Treating as success since operation is idempotent.", + dataSource + ); + } else { + // For other exceptions, let them propagate + throw e; + } + } + } + + @Override + public int markUnreferencedIndexingStatesAsUnused() + { + return connector.retryWithHandle( + handle -> + handle.createStatement( + StringUtils.format( + "UPDATE %s SET used = false, used_status_last_updated = :now WHERE used = true AND pending = false " + + "AND fingerprint NOT IN (SELECT DISTINCT indexing_state_fingerprint FROM %s WHERE used = true AND indexing_state_fingerprint IS NOT NULL)", + dbTables.getIndexingStatesTable(), + dbTables.getSegmentsTable() + ) + ) + .bind("now", DateTimes.nowUtc().toString()) + .execute()); + } + + @Override + public List findReferencedIndexingStateMarkedAsUnused() + { + return connector.retryWithHandle( + handle -> + handle.createQuery( + StringUtils.format( + "SELECT DISTINCT indexing_state_fingerprint FROM %s WHERE used = true AND indexing_state_fingerprint IN (SELECT fingerprint FROM %s WHERE used = false)", + dbTables.getSegmentsTable(), + dbTables.getIndexingStatesTable() + )) + .mapTo(String.class) + .list() + ); + } + + @Override + public int markIndexingStatesAsUsed(List stateFingerprints) + { + if (stateFingerprints.isEmpty()) { + return 0; + } + + return connector.retryWithHandle( + handle -> { + Update statement = handle.createStatement( + StringUtils.format( + "UPDATE %s SET used = true, pending = false, used_status_last_updated = :now" + + " WHERE fingerprint IN (%s)", + dbTables.getIndexingStatesTable(), + buildParameterizedInClause("fp", stateFingerprints.size()) + ) + ).bind("now", DateTimes.nowUtc().toString()); + + bindValuesToInClause(stateFingerprints, "fp", statement); + + return statement.execute(); + } + ); + } + + @Override + public int markIndexingStatesAsActive(List stateFingerprints) + { + if (stateFingerprints.isEmpty()) { + return 0; + } + + return connector.retryWithHandle( + handle -> { + Update statement = handle.createStatement( + StringUtils.format( + "UPDATE %s SET pending = false" + + " WHERE pending = true AND fingerprint IN (%s)", + dbTables.getIndexingStatesTable(), + buildParameterizedInClause("fp", stateFingerprints.size()) + ) + ); + + bindValuesToInClause(new ArrayList<>(stateFingerprints), "fp", statement); + + return statement.execute(); + } + ); + } + + @Override + public int deleteUnusedIndexingStatesOlderThan(long timestamp) + { + return connector.retryWithHandle( + handle -> handle.createStatement( + StringUtils.format( + "DELETE FROM %s WHERE used = false AND pending = false AND used_status_last_updated < :maxUpdateTime", + dbTables.getIndexingStatesTable() + )) + .bind("maxUpdateTime", DateTimes.utc(timestamp).toString()) + .execute()); + } + + @Override + public int deletePendingIndexingStatesOlderThan(long timestamp) + { + return connector.retryWithHandle( + handle -> handle.createStatement( + StringUtils.format( + "DELETE FROM %s WHERE pending = true AND used_status_last_updated < :maxUpdateTime", + dbTables.getIndexingStatesTable() + )) + .bind("maxUpdateTime", DateTimes.utc(timestamp).toString()) + .execute()); + } + + /** + * Checks if the indexing state for the given fingerprint is pending. + *

+ * Useful for testing purposes to verify the pending status of an indexing state. + *

+ */ + @Nullable + @VisibleForTesting + public Boolean isIndexingStatePending(final String fingerprint) + { + return connector.retryWithHandle( + handle -> { + String sql = StringUtils.format( + "SELECT pending FROM %s WHERE fingerprint = :fingerprint", + dbTables.getIndexingStatesTable() + ); + + return handle.createQuery(sql) + .bind("fingerprint", fingerprint) + .mapTo(Boolean.class) + .first(); + } + ); + } + + /** + * Represents the state of an indexing state fingerprint in the database. + *

+ * Intent is to help upsert logic decide whether to insert, update, or skip operations. + */ + private enum FingerprintState + { + /** Fingerprint does not exist in the database */ + DOES_NOT_EXIST, + /** Fingerprint exists and is marked as used */ + EXISTS_AND_USED, + /** Fingerprint exists but is marked as unused */ + EXISTS_AND_UNUSED + } + + /** + * Updates an existing unused indexing state to mark it as used. + */ + private void updateExistingUnusedState( + Handle handle, + String fingerprint, + String dataSource, + String updateTime + ) + { + log.info( + "Found existing indexing state in DB for fingerprint[%s] in dataSource[%s]. Marking as used.", + fingerprint, + dataSource + ); + + String updateSql = StringUtils.format( + "UPDATE %s SET used = :used, used_status_last_updated = :used_status_last_updated " + + "WHERE fingerprint = :fingerprint", + dbTables.getIndexingStatesTable() + ); + handle.createStatement(updateSql) + .bind("used", true) + .bind("used_status_last_updated", updateTime) + .bind("fingerprint", fingerprint) + .execute(); + + log.info("Updated existing indexing state for datasource[%s].", dataSource); + } + + /** + * Inserts a new indexing state into the database. + */ + private void insertNewState( + Handle handle, + String fingerprint, + String dataSource, + CompactionState indexingState, + String updateTime + ) + { + log.info("Inserting new indexing state for fingerprint[%s] in dataSource[%s].", fingerprint, dataSource); + + String insertSql = StringUtils.format( + "INSERT INTO %s (created_date, dataSource, fingerprint, payload, used, pending, used_status_last_updated) " + + "VALUES (:created_date, :dataSource, :fingerprint, :payload, :used, :pending, :used_status_last_updated)", + dbTables.getIndexingStatesTable() + ); + + try { + handle.createStatement(insertSql) + .bind("created_date", updateTime) + .bind("dataSource", dataSource) + .bind("fingerprint", fingerprint) + .bind("payload", jsonMapper.writeValueAsBytes(indexingState)) + .bind("used", true) + .bind("pending", true) + .bind("used_status_last_updated", updateTime) + .execute(); + + log.info( + "Published indexing state for fingerprint[%s] to DB for datasource[%s].", + fingerprint, + dataSource + ); + } + catch (JsonProcessingException e) { + throw InternalServerError.exception( + e, + "Failed to serialize indexing state for fingerprint[%s]", + fingerprint + ); + } + } + + /** + * Checks the state of a fingerprint in the metadata DB. + * + * @param handle Database handle + * @param fingerprintToCheck The fingerprint to check + * @return The state of the fingerprint (exists and used, exists and unused, or does not exist) + */ + private FingerprintState getFingerprintState( + final Handle handle, + @Nonnull final String fingerprintToCheck + ) + { + if (fingerprintToCheck.isEmpty()) { + return FingerprintState.DOES_NOT_EXIST; + } + + String sql = StringUtils.format( + "SELECT used FROM %s WHERE fingerprint = :fingerprint", + dbTables.getIndexingStatesTable() + ); + + Boolean used = handle.createQuery(sql) + .bind("fingerprint", fingerprintToCheck) + .mapTo(Boolean.class) + .first(); + + if (used == null) { + return FingerprintState.DOES_NOT_EXIST; + } + + return used ? FingerprintState.EXISTS_AND_USED : FingerprintState.EXISTS_AND_UNUSED; + } + + /** + * Builds a parameterized IN clause for the specified column with placeholders. + * Must be followed by a call to {@link #bindValuesToInClause(List, String, SQLStatement)}. + * + * @param parameterPrefix prefix for parameter names (e.g., "fingerprint") + * @param valueCount number of values in the IN clause + * @return parameterized IN clause like "(?, ?, ?)" but with named parameters + */ + private static String buildParameterizedInClause(String parameterPrefix, int valueCount) + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < valueCount; i++) { + sb.append(":").append(parameterPrefix).append(i); + if (i != valueCount - 1) { + sb.append(","); + } + } + return sb.toString(); + } + + /** + * Binds values to a parameterized IN clause in a SQL query. + * + * @param values list of values to bind + * @param parameterPrefix prefix used when building the IN clause + * @param query the SQL statement to bind values to + */ + private static void bindValuesToInClause( + List values, + String parameterPrefix, + SQLStatement query + ) + { + for (int i = 0; i < values.size(); i++) { + query.bind(parameterPrefix + i, values.get(i)); + } + } +} diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java index 58fe28e6ea96..bab65f90bf94 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java @@ -138,7 +138,8 @@ public void onTaskSubmitted(String taskId, CompactionCandidate candidateSegments Integer.MAX_VALUE, clusterConfig.getCompactionPolicy(), clusterConfig.isUseSupervisors(), - clusterConfig.getEngine() + clusterConfig.getEngine(), + clusterConfig.isStoreCompactionStatePerSegment() ); final CoordinatorRunStats stats = new CoordinatorRunStats(); diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java index cc52513b16c5..99e1eef21465 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java @@ -30,8 +30,10 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.granularity.GranularityType; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; import org.apache.druid.segment.transform.CompactionTransformSpec; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; @@ -55,6 +57,8 @@ */ public class CompactionStatus { + private static final Logger log = new Logger(CompactionStatus.class); + private static final CompactionStatus COMPLETE = new CompactionStatus(State.COMPLETE, null, null, null); public enum State @@ -62,13 +66,19 @@ public enum State COMPLETE, PENDING, RUNNING, SKIPPED } + /** + * List of checks performed to determine if compaction is already complete based on indexing state fingerprints. + */ + private static final List> FINGERPRINT_CHECKS = List.of( + Evaluator::allFingerprintedCandidatesHaveExpectedFingerprint + ); + /** * List of checks performed to determine if compaction is already complete. *

* The order of the checks must be honored while evaluating them. */ private static final List> CHECKS = Arrays.asList( - Evaluator::segmentsHaveBeenCompactedAtLeastOnce, Evaluator::partitionsSpecIsUpToDate, Evaluator::indexSpecIsUpToDate, Evaluator::segmentGranularityIsUpToDate, @@ -249,14 +259,25 @@ public static CompactionStatus running(String message) */ static CompactionStatus compute( CompactionCandidate candidateSegments, - DataSourceCompactionConfig config + DataSourceCompactionConfig config, + @Nullable IndexingStateFingerprintMapper fingerprintMapper ) { - return new Evaluator(candidateSegments, config).evaluate(); + final CompactionState expectedState = config.toCompactionState(); + String expectedFingerprint; + if (fingerprintMapper == null) { + expectedFingerprint = null; + } else { + expectedFingerprint = fingerprintMapper.generateFingerprint( + config.getDataSource(), + expectedState + ); + } + return new Evaluator(candidateSegments, config, expectedFingerprint, fingerprintMapper).evaluate(); } @Nullable - static PartitionsSpec findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig tuningConfig) + public static PartitionsSpec findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig tuningConfig) { final PartitionsSpec partitionsSpecFromTuningConfig = tuningConfig.getPartitionsSpec(); if (partitionsSpecFromTuningConfig == null) { @@ -330,18 +351,28 @@ private static class Evaluator private final ClientCompactionTaskQueryTuningConfig tuningConfig; private final UserCompactionTaskGranularityConfig configuredGranularitySpec; + private final List fingerprintedSegments = new ArrayList<>(); + private final List compactedSegments = new ArrayList<>(); private final List uncompactedSegments = new ArrayList<>(); private final Map> unknownStateToSegments = new HashMap<>(); + @Nullable + private final String targetFingerprint; + private final IndexingStateFingerprintMapper fingerprintMapper; + private Evaluator( CompactionCandidate candidateSegments, - DataSourceCompactionConfig compactionConfig + DataSourceCompactionConfig compactionConfig, + @Nullable String targetFingerprint, + @Nullable IndexingStateFingerprintMapper fingerprintMapper ) { this.candidateSegments = candidateSegments; this.compactionConfig = compactionConfig; this.tuningConfig = ClientCompactionTaskQueryTuningConfig.from(compactionConfig); this.configuredGranularitySpec = compactionConfig.getGranularitySpec(); + this.targetFingerprint = targetFingerprint; + this.fingerprintMapper = fingerprintMapper; } private CompactionStatus evaluate() @@ -351,37 +382,137 @@ private CompactionStatus evaluate() return inputBytesCheck; } - final List reasonsForCompaction = - CHECKS.stream() - .map(f -> f.apply(this)) - .filter(status -> !status.isComplete()) - .map(CompactionStatus::getReason) - .collect(Collectors.toList()); + List reasonsForCompaction = new ArrayList<>(); + CompactionStatus compactedOnceCheck = segmentsHaveBeenCompactedAtLeastOnce(); + if (!compactedOnceCheck.isComplete()) { + reasonsForCompaction.add(compactedOnceCheck.getReason()); + } + + if (fingerprintMapper != null && targetFingerprint != null) { + // First try fingerprint-based evaluation (fast path) + CompactionStatus fingerprintStatus = FINGERPRINT_CHECKS.stream() + .map(f -> f.apply(this)) + .filter(status -> !status.isComplete()) + .findFirst().orElse(COMPLETE); - // Consider segments which have passed all checks to be compacted - final List compactedSegments = unknownStateToSegments - .values() - .stream() - .flatMap(List::stream) - .collect(Collectors.toList()); + if (!fingerprintStatus.isComplete()) { + reasonsForCompaction.add(fingerprintStatus.getReason()); + } + } + + if (!unknownStateToSegments.isEmpty()) { + // Run CHECKS against any states with uknown compaction status + reasonsForCompaction.addAll( + CHECKS.stream() + .map(f -> f.apply(this)) + .filter(status -> !status.isComplete()) + .map(CompactionStatus::getReason) + .collect(Collectors.toList()) + ); + + // Any segments left in unknownStateToSegments passed all checks and are considered compacted + this.compactedSegments.addAll( + unknownStateToSegments + .values() + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()) + ); + } if (reasonsForCompaction.isEmpty()) { return COMPLETE; } else { return CompactionStatus.pending( - createStats(compactedSegments), + createStats(this.compactedSegments), createStats(uncompactedSegments), reasonsForCompaction.get(0) ); } } + /** + * Evaluates the fingerprints of all fingerprinted candidate segments against the expected fingerprint. + *

+ * If all fingerprinted segments have the expected fingerprint, the check can quickly pass as COMPLETE. However, + * if any fingerprinted segment has a mismatched fingerprint, we need to investigate further by adding them to + * {@link #unknownStateToSegments} where their indexing states will be analyzed. + *

+ */ + private CompactionStatus allFingerprintedCandidatesHaveExpectedFingerprint() + { + Map> mismatchedFingerprintToSegmentMap = new HashMap<>(); + for (DataSegment segment : fingerprintedSegments) { + String fingerprint = segment.getIndexingStateFingerprint(); + if (fingerprint == null) { + // Should not happen since we are iterating over fingerprintedSegments + } else if (fingerprint.equals(targetFingerprint)) { + compactedSegments.add(segment); + } else { + mismatchedFingerprintToSegmentMap + .computeIfAbsent(fingerprint, k -> new ArrayList<>()) + .add(segment); + } + } + + if (mismatchedFingerprintToSegmentMap.isEmpty()) { + // All fingerprinted segments have the expected fingerprint - compaction is complete + return COMPLETE; + } + + if (fingerprintMapper == null) { + // Cannot evaluate further without a fingerprint mapper + uncompactedSegments.addAll( + mismatchedFingerprintToSegmentMap.values() + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()) + ); + return CompactionStatus.pending("Segments have a mismatched fingerprint and no fingerprint mapper is available"); + } + + boolean fingerprintedSegmentWithoutCachedStateFound = false; + + for (Map.Entry> e : mismatchedFingerprintToSegmentMap.entrySet()) { + String fingerprint = e.getKey(); + CompactionState stateToValidate = fingerprintMapper.getStateForFingerprint(fingerprint).orElse(null); + if (stateToValidate == null) { + log.warn("No indexing state found for fingerprint[%s]", fingerprint); + fingerprintedSegmentWithoutCachedStateFound = true; + uncompactedSegments.addAll(e.getValue()); + } else { + // Note that this does not mean we need compaction yet - we need to validate the state further to determine this + unknownStateToSegments.compute( + stateToValidate, + (state, segments) -> { + if (segments == null) { + segments = new ArrayList<>(); + } + segments.addAll(e.getValue()); + return segments; + }); + } + } + + if (fingerprintedSegmentWithoutCachedStateFound) { + return CompactionStatus.pending("One or more fingerprinted segments do not have a cached indexing state"); + } else { + return COMPLETE; + } + } + + /** + * Checks if all the segments have been compacted at least once and groups them into uncompacted, fingerprinted, or + * non-fingerprinted. + */ private CompactionStatus segmentsHaveBeenCompactedAtLeastOnce() { - // Identify the compaction states of all the segments for (DataSegment segment : candidateSegments.getSegments()) { + final String fingerprint = segment.getIndexingStateFingerprint(); final CompactionState segmentState = segment.getLastCompactionState(); - if (segmentState == null) { + if (fingerprint != null) { + fingerprintedSegments.add(segment); + } else if (segmentState == null) { uncompactedSegments.add(segment); } else { unknownStateToSegments.computeIfAbsent(segmentState, s -> new ArrayList<>()).add(segment); diff --git a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java index 9b87b0409f69..1994e87a6388 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java @@ -30,6 +30,7 @@ import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.guava.Comparators; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.Partitions; @@ -68,6 +69,7 @@ public class DataSourceCompactibleSegmentIterator implements CompactionSegmentIt private final String dataSource; private final DataSourceCompactionConfig config; + private final IndexingStateFingerprintMapper fingerprintMapper; private final List compactedSegments = new ArrayList<>(); private final List skippedSegments = new ArrayList<>(); @@ -84,12 +86,14 @@ public DataSourceCompactibleSegmentIterator( DataSourceCompactionConfig config, SegmentTimeline timeline, List skipIntervals, - CompactionCandidateSearchPolicy searchPolicy + CompactionCandidateSearchPolicy searchPolicy, + IndexingStateFingerprintMapper indexingStateFingerprintMapper ) { this.config = config; this.dataSource = config.getDataSource(); this.queue = new PriorityQueue<>(searchPolicy::compareCandidates); + this.fingerprintMapper = indexingStateFingerprintMapper; populateQueue(timeline, skipIntervals); } @@ -326,7 +330,7 @@ private void findAndEnqueueSegmentsToCompact(CompactibleSegmentIterator compacti } final CompactionCandidate candidates = CompactionCandidate.from(segments, config.getSegmentGranularity()); - final CompactionStatus compactionStatus = CompactionStatus.compute(candidates, config); + final CompactionStatus compactionStatus = CompactionStatus.compute(candidates, config, fingerprintMapper); final CompactionCandidate candidatesWithStatus = candidates.withCurrentStatus(compactionStatus); if (compactionStatus.isComplete()) { diff --git a/server/src/main/java/org/apache/druid/server/compaction/PriorityBasedCompactionSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/PriorityBasedCompactionSegmentIterator.java index 49d936fda0ac..43028fa37ff4 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/PriorityBasedCompactionSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/PriorityBasedCompactionSegmentIterator.java @@ -22,6 +22,7 @@ import com.google.common.collect.Maps; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.timeline.SegmentTimeline; import org.joda.time.Interval; @@ -48,7 +49,8 @@ public PriorityBasedCompactionSegmentIterator( CompactionCandidateSearchPolicy searchPolicy, Map compactionConfigs, Map datasourceToTimeline, - Map> skipIntervals + Map> skipIntervals, + IndexingStateFingerprintMapper indexingStateFingerprintMapper ) { this.queue = new PriorityQueue<>(searchPolicy::compareCandidates); @@ -69,7 +71,8 @@ public PriorityBasedCompactionSegmentIterator( compactionConfigs.get(datasource), timeline, skipIntervals.getOrDefault(datasource, Collections.emptyList()), - searchPolicy + searchPolicy, + indexingStateFingerprintMapper ) ); addNextItemForDatasourceToQueue(datasource); diff --git a/server/src/main/java/org/apache/druid/server/coordination/LoadableDataSegment.java b/server/src/main/java/org/apache/druid/server/coordination/LoadableDataSegment.java index 8658f97b530c..427c670cf4cf 100644 --- a/server/src/main/java/org/apache/druid/server/coordination/LoadableDataSegment.java +++ b/server/src/main/java/org/apache/druid/server/coordination/LoadableDataSegment.java @@ -54,7 +54,8 @@ private LoadableDataSegment( @JsonProperty("lastCompactionState") @Nullable CompactionState lastCompactionState, @JsonProperty("binaryVersion") Integer binaryVersion, @JsonProperty("size") long size, - @JsonProperty("totalRows") Integer totalRows + @JsonProperty("totalRows") Integer totalRows, + @JsonProperty("indexingStateFingerprint") String indexingStateFingerprint ) { super( @@ -70,6 +71,7 @@ private LoadableDataSegment( binaryVersion, size, totalRows, + indexingStateFingerprint, PruneSpecsHolder.DEFAULT ); } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/ClusterCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/ClusterCompactionConfig.java index 437849ad1bf2..08e543504a6a 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/ClusterCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/ClusterCompactionConfig.java @@ -45,6 +45,7 @@ public class ClusterCompactionConfig private final boolean useSupervisors; private final CompactionEngine engine; private final CompactionCandidateSearchPolicy compactionPolicy; + private final boolean storeCompactionStatePerSegment; @JsonCreator public ClusterCompactionConfig( @@ -52,7 +53,8 @@ public ClusterCompactionConfig( @JsonProperty("maxCompactionTaskSlots") @Nullable Integer maxCompactionTaskSlots, @JsonProperty("compactionPolicy") @Nullable CompactionCandidateSearchPolicy compactionPolicy, @JsonProperty("useSupervisors") @Nullable Boolean useSupervisors, - @JsonProperty("engine") @Nullable CompactionEngine engine + @JsonProperty("engine") @Nullable CompactionEngine engine, + @JsonProperty("storeCompactionStatePerSegment") @Nullable Boolean storeCompactionStatePerSegment ) { this.compactionTaskSlotRatio = Configs.valueOrDefault(compactionTaskSlotRatio, 0.1); @@ -60,6 +62,10 @@ public ClusterCompactionConfig( this.compactionPolicy = Configs.valueOrDefault(compactionPolicy, DEFAULT_COMPACTION_POLICY); this.engine = Configs.valueOrDefault(engine, CompactionEngine.NATIVE); this.useSupervisors = Configs.valueOrDefault(useSupervisors, false); + this.storeCompactionStatePerSegment = Configs.valueOrDefault( + storeCompactionStatePerSegment, + true + ); if (!this.useSupervisors && this.engine == CompactionEngine.MSQ) { throw InvalidInput.exception("MSQ Compaction engine can be used only with compaction supervisors."); @@ -96,6 +102,19 @@ public CompactionEngine getEngine() return engine; } + /** + * Whether to persist last compaction state directly in segments for backwards compatibility. + *

+ * In a future release this option will be removed and last compaction state will no longer be persisted in segments. + * Instead, it will only be stored in the metadata store with a fingerprint id that segments will reference. Some + * operators may want to disable this behavior early to begin saving space in segment metadatastore table entries. + */ + @JsonProperty + public boolean isStoreCompactionStatePerSegment() + { + return storeCompactionStatePerSegment; + } + @Override public boolean equals(Object o) { @@ -110,7 +129,8 @@ public boolean equals(Object o) && Objects.equals(maxCompactionTaskSlots, that.maxCompactionTaskSlots) && Objects.equals(compactionPolicy, that.compactionPolicy) && Objects.equals(useSupervisors, that.useSupervisors) - && Objects.equals(engine, that.engine); + && Objects.equals(engine, that.engine) + && Objects.equals(storeCompactionStatePerSegment, that.storeCompactionStatePerSegment); } @Override @@ -121,7 +141,8 @@ public int hashCode() maxCompactionTaskSlots, compactionPolicy, useSupervisors, - engine + engine, + storeCompactionStatePerSegment ); } @@ -134,6 +155,7 @@ public String toString() ", useSupervisors=" + useSupervisors + ", engine=" + engine + ", compactionPolicy=" + compactionPolicy + + ", storeCompactionStatePerSegment=" + storeCompactionStatePerSegment + '}'; } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorConfigManager.java b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorConfigManager.java index 5d435549bacd..a047bf99070d 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorConfigManager.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorConfigManager.java @@ -166,7 +166,8 @@ public boolean updateCompactionTaskSlots( Configs.valueOrDefault(maxCompactionTaskSlots, currentClusterConfig.getMaxCompactionTaskSlots()), currentClusterConfig.getCompactionPolicy(), currentClusterConfig.isUseSupervisors(), - currentClusterConfig.getEngine() + currentClusterConfig.getEngine(), + currentClusterConfig.isStoreCompactionStatePerSegment() ); return current.withClusterConfig(updatedClusterConfig); diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java index 8d9b861b5713..39a24d3eb5e8 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java @@ -21,16 +21,26 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.indexer.CompactionEngine; +import org.apache.druid.indexer.granularity.GranularitySpec; +import org.apache.druid.indexer.granularity.UniformGranularitySpec; +import org.apache.druid.indexer.partitions.PartitionsSpec; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.transform.CompactionTransformSpec; +import org.apache.druid.server.compaction.CompactionStatus; +import org.apache.druid.timeline.CompactionState; import org.joda.time.Period; import javax.annotation.Nullable; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = InlineSchemaDataSourceCompactionConfig.class) @JsonSubTypes(value = { @@ -89,4 +99,63 @@ public interface DataSourceCompactionConfig @Nullable AggregatorFactory[] getMetricsSpec(); + + /** + * Converts this compaction config to a {@link CompactionState}. + *

+ * For IndexSpec and DimensionsSpec, we convert to their effective specs so that the fingerprint and associated state + * reflect the actual layout of the segments after compaction (with all missing defaults not included in the compaction + * config filled in). This is consistent with how {@link org.apache.druid.timeline.DataSegment#lastCompactionState } + * has been computed historically. + */ + default CompactionState toCompactionState() + { + ClientCompactionTaskQueryTuningConfig tuningConfig = ClientCompactionTaskQueryTuningConfig.from(this); + + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(tuningConfig); + + IndexSpec indexSpec = tuningConfig.getIndexSpec() == null + ? IndexSpec.getDefault().getEffectiveSpec() + : tuningConfig.getIndexSpec().getEffectiveSpec(); + + DimensionsSpec dimensionsSpec = null; + if (getDimensionsSpec() != null && getDimensionsSpec().getDimensions() != null) { + dimensionsSpec = DimensionsSpec.builder() + .setDimensions( + getDimensionsSpec().getDimensions() + .stream() + .map(dim -> dim.getEffectiveSchema(indexSpec)) + .collect(Collectors.toList()) + ).build(); + } + + List metricsSpec = getMetricsSpec() == null + ? null + : Arrays.asList(getMetricsSpec()); + + CompactionTransformSpec transformSpec = getTransformSpec(); + + GranularitySpec granularitySpec = null; + if (getGranularitySpec() != null) { + UserCompactionTaskGranularityConfig userGranularityConfig = getGranularitySpec(); + granularitySpec = new UniformGranularitySpec( + userGranularityConfig.getSegmentGranularity(), + userGranularityConfig.getQueryGranularity(), + userGranularityConfig.isRollup(), + null // intervals + ); + } + + List projections = getProjections(); + + return new CompactionState( + partitionsSpec, + dimensionsSpec, + metricsSpec, + transformSpec, + indexSpec, + granularitySpec, + projections + ); + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCompactionConfig.java index cc11d9fdf718..3af188d9a03a 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCompactionConfig.java @@ -40,7 +40,7 @@ public class DruidCompactionConfig public static final String CONFIG_KEY = "coordinator.compaction.config"; private static final DruidCompactionConfig EMPTY_INSTANCE - = new DruidCompactionConfig(List.of(), null, null, null, null, null); + = new DruidCompactionConfig(List.of(), null, null, null, null, null, null); private final List compactionConfigs; private final ClusterCompactionConfig clusterConfig; @@ -86,7 +86,8 @@ public DruidCompactionConfig( @JsonProperty("maxCompactionTaskSlots") @Nullable Integer maxCompactionTaskSlots, @JsonProperty("compactionPolicy") @Nullable CompactionCandidateSearchPolicy compactionPolicy, @JsonProperty("useSupervisors") @Nullable Boolean useSupervisors, - @JsonProperty("engine") @Nullable CompactionEngine engine + @JsonProperty("engine") @Nullable CompactionEngine engine, + @JsonProperty("storeCompactionStatePerSegment") @Nullable Boolean storeCompactionStatePerSegment ) { this( @@ -96,7 +97,8 @@ public DruidCompactionConfig( maxCompactionTaskSlots, compactionPolicy, useSupervisors, - engine + engine, + storeCompactionStatePerSegment ) ); } @@ -140,6 +142,12 @@ public CompactionEngine getEngine() return clusterConfig.getEngine(); } + @JsonProperty + public boolean isStoreCompactionStatePerSegment() + { + return clusterConfig.isStoreCompactionStatePerSegment(); + } + /** * Returns the cluster-level compaction config. Not used for serialization. */ diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 9947e521f657..060c16edfadb 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -76,7 +76,12 @@ public class CompactSegments implements CoordinatorCustomDuty * Must be the same as org.apache.druid.indexing.common.task.Tasks.STORE_COMPACTION_STATE_KEY */ public static final String STORE_COMPACTION_STATE_KEY = "storeCompactionState"; - public static final String COMPACTION_INTERVAL_KEY = "compactionInterval"; + + /** + * Must be the same as org.apache.druid.indexing.common.task.Tasks.INDEXING_STATE_FINGERPRINT_KEY + */ + public static final String INDEXING_STATE_FINGERPRINT_KEY = "indexingStateFingerprint"; + private static final String COMPACTION_REASON_KEY = "compactionReason"; private static final Logger LOG = new Logger(CompactSegments.class); @@ -177,7 +182,8 @@ public void run( policy, compactionConfigs, dataSources.getUsedSegmentsTimelinesPerDataSource(), - slotManager.getDatasourceIntervalsToSkipCompaction() + slotManager.getDatasourceIntervalsToSkipCompaction(), + null ); final CompactionSnapshotBuilder compactionSnapshotBuilder = new CompactionSnapshotBuilder(stats); @@ -254,7 +260,13 @@ private int submitCompactionTasks( snapshotBuilder.addToComplete(entry); } - final ClientCompactionTaskQuery taskPayload = createCompactionTask(entry, config, defaultEngine); + final ClientCompactionTaskQuery taskPayload = createCompactionTask( + entry, + config, + defaultEngine, + null, + true + ); final String taskId = taskPayload.getId(); FutureUtils.getUnchecked(overlordClient.runTask(taskId, taskPayload), true); @@ -280,7 +292,9 @@ private int submitCompactionTasks( public static ClientCompactionTaskQuery createCompactionTask( CompactionCandidate candidate, DataSourceCompactionConfig config, - CompactionEngine defaultEngine + CompactionEngine defaultEngine, + String indexingStateFingerprint, + boolean storeCompactionStatePerSegment ) { final List segmentsToCompact = candidate.getSegments(); @@ -358,6 +372,9 @@ public static ClientCompactionTaskQuery createCompactionTask( autoCompactionContext.put(COMPACTION_REASON_KEY, candidate.getCurrentStatus().getReason()); } + autoCompactionContext.put(STORE_COMPACTION_STATE_KEY, storeCompactionStatePerSegment); + autoCompactionContext.put(INDEXING_STATE_FINGERPRINT_KEY, indexingStateFingerprint); + return compactSegments( candidate, config.getTaskPriority(), diff --git a/server/src/main/java/org/apache/druid/server/http/DataSegmentPlus.java b/server/src/main/java/org/apache/druid/server/http/DataSegmentPlus.java index bfda5cbf3ad4..e06241f6aae7 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSegmentPlus.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSegmentPlus.java @@ -58,6 +58,9 @@ public class DataSegmentPlus @Nullable private final String upgradedFromSegmentId; + @Nullable + private final String indexingStateFingerprint; + @JsonCreator public DataSegmentPlus( @JsonProperty("dataSegment") final DataSegment dataSegment, @@ -66,7 +69,8 @@ public DataSegmentPlus( @JsonProperty("used") @Nullable final Boolean used, @JsonProperty("schemaFingerprint") @Nullable final String schemaFingerprint, @JsonProperty("numRows") @Nullable final Long numRows, - @JsonProperty("upgradedFromSegmentId") @Nullable final String upgradedFromSegmentId + @JsonProperty("upgradedFromSegmentId") @Nullable final String upgradedFromSegmentId, + @JsonProperty("indexingStateFingerprint") @Nullable String indexingStateFingerprint ) { this.dataSegment = dataSegment; @@ -76,6 +80,7 @@ public DataSegmentPlus( this.schemaFingerprint = schemaFingerprint; this.numRows = numRows; this.upgradedFromSegmentId = upgradedFromSegmentId; + this.indexingStateFingerprint = indexingStateFingerprint; } @Nullable @@ -126,6 +131,13 @@ public String getUpgradedFromSegmentId() return upgradedFromSegmentId; } + @Nullable + @JsonProperty + public String getIndexingStateFingerprint() + { + return indexingStateFingerprint; + } + @Override public boolean equals(Object o) { @@ -142,7 +154,8 @@ public boolean equals(Object o) && Objects.equals(used, that.getUsed()) && Objects.equals(schemaFingerprint, that.getSchemaFingerprint()) && Objects.equals(numRows, that.getNumRows()) - && Objects.equals(upgradedFromSegmentId, that.getUpgradedFromSegmentId()); + && Objects.equals(upgradedFromSegmentId, that.getUpgradedFromSegmentId()) + && Objects.equals(indexingStateFingerprint, that.getIndexingStateFingerprint()); } @Override @@ -155,7 +168,8 @@ public int hashCode() used, schemaFingerprint, numRows, - upgradedFromSegmentId + upgradedFromSegmentId, + indexingStateFingerprint ); } @@ -170,6 +184,7 @@ public String toString() ", schemaFingerprint=" + getSchemaFingerprint() + ", numRows=" + getNumRows() + ", upgradedFromSegmentId=" + getUpgradedFromSegmentId() + + ", indexingStateFingerprint=" + getIndexingStateFingerprint() + '}'; } } diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java index 9aa508cbba4f..b7264204bfe5 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java @@ -2766,6 +2766,7 @@ private MyDataSegment() -1, 0, 0, + null, PruneSpecsHolder.DEFAULT ); } diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorMarkUsedTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorMarkUsedTest.java index 2406af6f12b6..dadad5e4f1ba 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorMarkUsedTest.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorMarkUsedTest.java @@ -30,6 +30,7 @@ import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache; import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -95,7 +96,8 @@ public int getMaxRetries() derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector, null, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); derbyConnector.createSegmentTable(); diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorReadOnlyTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorReadOnlyTest.java index e0e6898df8b2..d68da314ad16 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorReadOnlyTest.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorReadOnlyTest.java @@ -36,6 +36,8 @@ import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; +import org.apache.druid.segment.metadata.NoopIndexingStateCache; import org.apache.druid.segment.metadata.NoopSegmentSchemaCache; import org.apache.druid.server.coordinator.simulate.BlockingExecutorService; import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector; @@ -104,6 +106,7 @@ public void setup() () -> new SegmentsMetadataManagerConfig(null, cacheMode, null), derbyConnectorRule.metadataTablesConfigSupplier(), new NoopSegmentSchemaCache(), + new NoopIndexingStateCache(), derbyConnector, (corePoolSize, nameFormat) -> new WrappingScheduledExecutorService( nameFormat, @@ -178,7 +181,8 @@ private IndexerSQLMetadataStorageCoordinator createStorageCoordinator( derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector, null, - CentralizedDatasourceSchemaConfig.enabled(false) + CentralizedDatasourceSchemaConfig.enabled(false), + new HeapMemoryIndexingStateStorage() ); } diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java index dd9508c3c19b..0beecb318956 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java @@ -27,6 +27,7 @@ import org.apache.druid.data.input.StringTuple; import org.apache.druid.error.DruidExceptionMatcher; import org.apache.druid.error.ExceptionMatcher; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; import org.apache.druid.indexing.overlord.DataSourceMetadata; import org.apache.druid.indexing.overlord.ObjectMetadata; import org.apache.druid.indexing.overlord.SegmentCreateRequest; @@ -42,19 +43,24 @@ import org.apache.druid.metadata.segment.cache.HeapMemorySegmentMetadataCache; import org.apache.druid.metadata.segment.cache.Metric; import org.apache.druid.metadata.segment.cache.SegmentMetadataCache; +import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.SegmentSchemaMapping; import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; import org.apache.druid.segment.metadata.FingerprintGenerator; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; +import org.apache.druid.segment.metadata.NoopIndexingStateCache; import org.apache.druid.segment.metadata.NoopSegmentSchemaCache; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.segment.metadata.SegmentSchemaTestUtils; +import org.apache.druid.segment.metadata.SqlIndexingStateStorage; import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.server.coordinator.simulate.BlockingExecutorService; import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector; import org.apache.druid.server.coordinator.simulate.WrappingScheduledExecutorService; import org.apache.druid.server.http.DataSegmentPlus; +import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentTimeline; @@ -112,6 +118,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata private StubServiceEmitter emitter; private SqlSegmentMetadataTransactionFactory transactionFactory; private BlockingExecutorService cachePollExecutor; + private SqlIndexingStateStorage indexingStateStorage; private final SegmentMetadataCache.UsageMode cacheMode; @@ -141,12 +148,18 @@ public void setUp() derbyConnector.createSegmentTable(); derbyConnector.createUpgradeSegmentsTable(); derbyConnector.createPendingSegmentsTable(); + derbyConnector.createIndexingStatesTable(); metadataUpdateCounter.set(0); segmentTableDropUpdateCounter.set(0); fingerprintGenerator = new FingerprintGenerator(mapper); segmentSchemaManager = new SegmentSchemaManager(derbyConnectorRule.metadataTablesConfigSupplier().get(), mapper, derbyConnector); segmentSchemaTestUtils = new SegmentSchemaTestUtils(derbyConnectorRule, derbyConnector, mapper); + indexingStateStorage = new SqlIndexingStateStorage( + derbyConnectorRule.metadataTablesConfigSupplier().get(), + mapper, + derbyConnector + ); emitter = new StubServiceEmitter(); leaderSelector = new TestDruidLeaderSelector(); @@ -158,6 +171,7 @@ public void setUp() () -> new SegmentsMetadataManagerConfig(null, cacheMode, null), derbyConnectorRule.metadataTablesConfigSupplier(), new NoopSegmentSchemaCache(), + new NoopIndexingStateCache(), derbyConnector, (corePoolSize, nameFormat) -> new WrappingScheduledExecutorService( nameFormat, @@ -198,7 +212,8 @@ public int getMaxRetries() derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + indexingStateStorage ) { @Override @@ -794,7 +809,8 @@ public void testTransactionalAnnounceRetryAndSuccess() throws IOException derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ) { @Override @@ -953,7 +969,8 @@ public void test_commitSegmentsAndMetadata_isAtomic() derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector, segmentSchemaManager, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ) { @Override @@ -4315,6 +4332,97 @@ public void testWriteOperation_alwaysUsesCache_inModeIfSynced() emitter.verifyValue(Metric.READ_WRITE_TRANSACTIONS, 1L); } + @Test + public void testCommitSegmentsAndMetadata_marksPendingIndexingStateAsActive() + { + String fingerprint = "vanillaFingerprint"; + CompactionState state = createTestIndexingState(); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, fingerprint, state, DateTimes.nowUtc()); + Assert.assertEquals(Boolean.TRUE, indexingStateStorage.isIndexingStatePending(fingerprint)); + + final DataSegment segment = CreateDataSegments.ofDatasource(TestDataSource.WIKI) + .startingAt("2023-01-01") + .withIndexingStateFingerprint(fingerprint) + .eachOfSizeInMb(500) + .get(0); + + coordinator.commitSegmentsAndMetadata( + ImmutableSet.of(segment), + SUPERVISOR_ID, + new ObjectMetadata(null), + new ObjectMetadata(ImmutableMap.of("foo", "bar")), + null + ); + + Assert.assertEquals(Boolean.FALSE, indexingStateStorage.isIndexingStatePending(fingerprint)); + } + + @Test + public void testCommitReplaceSegments_marksPendingIndexingStateAsActive() + { + String fingerprint = "replaceFingerprint"; + CompactionState state = createTestIndexingState(); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, fingerprint, state, DateTimes.nowUtc()); + Assert.assertEquals(Boolean.TRUE, indexingStateStorage.isIndexingStatePending(fingerprint)); + + final DataSegment segment = CreateDataSegments.ofDatasource(TestDataSource.WIKI) + .startingAt("2023-01-01") + .withIndexingStateFingerprint(fingerprint) + .eachOfSizeInMb(500) + .get(0); + + final String replaceTaskId = "replaceTask"; + final ReplaceTaskLock replaceLock = new ReplaceTaskLock( + replaceTaskId, + Intervals.of("2023-01-01/2023-01-02"), + "2024-01-01" + ); + + coordinator.commitReplaceSegments( + ImmutableSet.of(segment), + ImmutableSet.of(replaceLock), + null + ); + + Assert.assertEquals(Boolean.FALSE, indexingStateStorage.isIndexingStatePending(fingerprint)); + } + + @Test + public void testCommitAppendSegments_marksPendingIndexingStateAsActive() + { + String fingerprint = "appendFingerprint"; + CompactionState state = createTestIndexingState(); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, fingerprint, state, DateTimes.nowUtc()); + Assert.assertEquals(Boolean.TRUE, indexingStateStorage.isIndexingStatePending(fingerprint)); + + final DataSegment segment = CreateDataSegments.ofDatasource(TestDataSource.WIKI) + .startingAt("2023-01-01") + .withIndexingStateFingerprint(fingerprint) + .eachOfSizeInMb(500) + .get(0); + + final String taskAllocatorId = "appendTask"; + + coordinator.commitAppendSegments( + ImmutableSet.of(segment), + Map.of(), + taskAllocatorId, + null + ); + + Assert.assertEquals(Boolean.FALSE, indexingStateStorage.isIndexingStatePending(fingerprint)); + } + + private CompactionState createTestIndexingState() + { + return new CompactionState( + new DynamicPartitionsSpec(100, null), + null, null, null, + IndexSpec.getDefault(), + null, null + ); + } + private SegmentIdWithShardSpec allocatePendingSegment( String datasource, String sequenceName, diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorSchemaPersistenceTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorSchemaPersistenceTest.java index 642f14f9824a..ba64c709fb18 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorSchemaPersistenceTest.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorSchemaPersistenceTest.java @@ -39,6 +39,7 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; import org.apache.druid.segment.metadata.FingerprintGenerator; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.segment.metadata.SegmentSchemaManager; import org.apache.druid.segment.metadata.SegmentSchemaTestUtils; import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector; @@ -108,7 +109,8 @@ public void setUp() derbyConnectorRule.metadataTablesConfigSupplier().get(), derbyConnector, segmentSchemaManager, - centralizedDatasourceSchemaConfig + centralizedDatasourceSchemaConfig, + new HeapMemoryIndexingStateStorage() ) { @Override diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java index 36dd38a85c00..be0120162cf0 100644 --- a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java +++ b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java @@ -617,7 +617,8 @@ public static void insertUsedSegments( true, null, null, - upgradedFromSegmentIdMap.get(segment.getId().toString()) + upgradedFromSegmentIdMap.get(segment.getId().toString()), + null ) ); } diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java index f898d12000c7..3fef28a963a6 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataConnectorTest.java @@ -76,6 +76,7 @@ public void testCreateTables() tables.add(tablesConfig.getTasksTable()); tables.add(tablesConfig.getAuditTable()); tables.add(tablesConfig.getSupervisorTable()); + tables.add(tablesConfig.getIndexingStatesTable()); connector.createSegmentTable(); connector.createConfigTable(); @@ -83,6 +84,7 @@ public void testCreateTables() connector.createTaskTables(); connector.createAuditTable(); connector.createSupervisorsTable(); + connector.createIndexingStatesTable(); connector.getDBI().withHandle( handle -> { @@ -187,6 +189,22 @@ public void testAlterSegmentTableAddLastUsed() )); } + /** + * This is a test for the upgrade path where a cluster is upgrading from a version that did not have used_status_last_updated + * in the segments table. + */ + @Test + public void testAlterSegmentTableAddIndexingStateFingerprint() + { + connector.createSegmentTable(); + derbyConnectorRule.segments().update("ALTER TABLE %1$s DROP COLUMN INDEXING_STATE_FINGERPRINT"); + connector.alterSegmentTable(); + Assert.assertTrue(connector.tableHasColumn( + derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable(), + "INDEXING_STATE_FINGERPRINT" + )); + } + @Test public void testInsertOrUpdate() { @@ -309,7 +327,8 @@ public void test_useShortIndexNames_true_tableIndices_areNotAdded_ifExist() tablesConfig = new MetadataStorageTablesConfig( "druidTest", null, null, null, null, null, null, null, null, null, null, null, - true + true, + null ); connector = new TestDerbyConnector(new MetadataStorageConnectorConfig(), tablesConfig); @@ -343,7 +362,8 @@ public void test_useShortIndexNames_false_tableIndices_areNotAdded_ifExist() tablesConfig = new MetadataStorageTablesConfig( "druidTest", null, null, null, null, null, null, null, null, null, null, null, - false + false, + null ); connector = new TestDerbyConnector(new MetadataStorageConnectorConfig(), tablesConfig); @@ -377,7 +397,8 @@ public void test_useShortIndexNames_true_tableIndices_areAdded_IfNotExist() tablesConfig = new MetadataStorageTablesConfig( "druidTest", null, null, null, null, null, null, null, null, null, null, null, - true + true, + null ); connector = new TestDerbyConnector(new MetadataStorageConnectorConfig(), tablesConfig); @@ -403,7 +424,8 @@ public void test_useShortIndexNames_false_tableIndices_areAdded_IfNotExist() tablesConfig = new MetadataStorageTablesConfig( "druidTest", null, null, null, null, null, null, null, null, null, null, null, - false + false, + null ); connector = new TestDerbyConnector(new MetadataStorageConnectorConfig(), tablesConfig); final String segmentsTable = tablesConfig.getSegmentsTable(); @@ -470,6 +492,12 @@ public String limitClause(int limit) return ""; } + @Override + public boolean isUniqueConstraintViolation(Throwable t) + { + return false; + } + @Override public String getQuoteString() { diff --git a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataQueryTest.java b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataQueryTest.java index 1084c888e01e..8a8fed1c87c3 100644 --- a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataQueryTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataQueryTest.java @@ -19,15 +19,22 @@ package org.apache.druid.metadata; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.metadata.segment.cache.IndexingStateRecord; import org.apache.druid.metadata.storage.derby.DerbyConnector; +import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.metadata.SqlIndexingStateStorage; import org.apache.druid.server.coordinator.CreateDataSegments; +import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.joda.time.DateTime; @@ -38,6 +45,8 @@ import org.junit.Rule; import org.junit.Test; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -365,4 +374,355 @@ private static Set getIds(Set segments) { return segments.stream().map(DataSegment::getId).collect(Collectors.toSet()); } + + // ==================== Indexing State Tests ==================== + + @Test + public void test_retrieveAllUsedIndexingStateFingerprints_emptyDatabase() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + Set fingerprints = read(SqlSegmentsMetadataQuery::retrieveAllUsedIndexingStateFingerprints); + + Assert.assertTrue("Should return empty set when no segments have indexing states", fingerprints.isEmpty()); + } + + @Test + public void test_retrieveAllUsedIndexingStateFingerprints() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + Map indexingStates = new HashMap<>(); + indexingStates.put("fp1", createTestIndexingState()); + indexingStates.put("fp2", createTestIndexingState()); + indexingStates.put("fp3", createTestIndexingState()); + insertIndexingStates(indexingStates); + + insertSegmentWithIndexingState("seg1", "fp1", true); + insertSegmentWithIndexingState("seg2", "fp2", true); + insertSegmentWithIndexingState("seg3", "fp1", true); // Duplicate fingerprint + insertSegmentWithIndexingState("seg4", "fp3", false); // Unused segment + + Set fingerprints = read(SqlSegmentsMetadataQuery::retrieveAllUsedIndexingStateFingerprints); + + Assert.assertEquals("Should return all fingerprints in the cache", Set.of("fp1", "fp2", "fp3"), fingerprints); + } + + @Test + public void test_retrieveAllUsedIndexingStateFingerprints_ignoresNullFingerprints() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + Map indexingStates = new HashMap<>(); + indexingStates.put("fp1", createTestIndexingState()); + insertIndexingStates(indexingStates); + + insertSegmentWithIndexingState("seg1", "fp1", true); + insertSegmentWithIndexingState("seg2", null, true); // No indexing state + + Set fingerprints = read(SqlSegmentsMetadataQuery::retrieveAllUsedIndexingStateFingerprints); + + Assert.assertEquals("Should ignore segments without indexing states", Set.of("fp1"), fingerprints); + } + + @Test + public void test_retrieveAllUsedIndexingStates_emptyDatabase() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + List records = read(SqlSegmentsMetadataQuery::retrieveAllUsedIndexingStates); + + Assert.assertTrue("Should return empty list when no indexing states exist", records.isEmpty()); + } + + @Test + public void test_retrieveAllUsedIndexingStates_fullSync() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + CompactionState state1 = createTestIndexingState(); + CompactionState state2 = new CompactionState( + new DynamicPartitionsSpec(200, null), + DimensionsSpec.EMPTY, + null, + null, + IndexSpec.getDefault(), + null, + null + ); + CompactionState state3 = createTestIndexingState(); + + Map indexingStates = new HashMap<>(); + indexingStates.put("fp1", state1); + indexingStates.put("fp2", state2); + indexingStates.put("fp3", state3); // Unreferenced state + insertIndexingStates(indexingStates); + + // Only reference fp1 and fp2 + insertSegmentWithIndexingState("seg1", "fp1", true); + insertSegmentWithIndexingState("seg2", "fp2", true); + + List records = read(SqlSegmentsMetadataQuery::retrieveAllUsedIndexingStates); + + Assert.assertEquals("Should return all indexing states", 3, records.size()); + + Set retrievedFingerprints = records.stream() + .map(IndexingStateRecord::getFingerprint) + .collect(Collectors.toSet()); + Assert.assertEquals("Should contain all fps", Set.of("fp1", "fp2", "fp3"), retrievedFingerprints); + + // Verify payloads + Map retrievedStates = records.stream() + .collect(Collectors.toMap( + IndexingStateRecord::getFingerprint, + IndexingStateRecord::getState + )); + Assert.assertEquals("fp1 state should match", state1, retrievedStates.get("fp1")); + Assert.assertEquals("fp2 state should match", state2, retrievedStates.get("fp2")); + Assert.assertEquals("fp3 state should match", state3, retrievedStates.get("fp3")); + } + + @Test + public void test_retrieveAllUsedIndexingStates_onlyFromUsedSegments() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + Map indexingStates = new HashMap<>(); + indexingStates.put("fp1", createTestIndexingState()); + indexingStates.put("fp2", createTestIndexingState()); + insertIndexingStates(indexingStates); + + insertSegmentWithIndexingState("seg1", "fp1", true); // Used + insertSegmentWithIndexingState("seg2", "fp2", false); // Unused + + List records = read(SqlSegmentsMetadataQuery::retrieveAllUsedIndexingStates); + + Assert.assertEquals("Should only return all indexing states", 2, records.size()); + } + + @Test + public void test_retrieveAllUsedIndexingStates_ignoresUnusedIndexingStates() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + Map indexingStates = new HashMap<>(); + indexingStates.put("fp1", createTestIndexingState()); + insertIndexingStates(indexingStates); + + insertSegmentWithIndexingState("seg1", "fp1", true); + + markIndexingStateAsUnused("fp1"); + + List records = read(SqlSegmentsMetadataQuery::retrieveAllUsedIndexingStates); + + Assert.assertTrue("Should not return unused indexing states", records.isEmpty()); + } + + @Test + public void test_retrieveIndexingStatesForFingerprints_emptyInput() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + List records = read( + sql -> sql.retrieveIndexingStatesForFingerprints(Set.of()) + ); + + Assert.assertTrue("Should return empty list for empty input", records.isEmpty()); + } + + @Test + public void test_retrieveIndexingStatesForFingerprints_deltaSync() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + Map indexingStates = new HashMap<>(); + indexingStates.put("fp1", createTestIndexingState()); + indexingStates.put("fp2", createTestIndexingState()); + indexingStates.put("fp3", createTestIndexingState()); + insertIndexingStates(indexingStates); + + // Request specific fingerprints (delta sync scenario) + List records = read( + sql -> sql.retrieveIndexingStatesForFingerprints(Set.of("fp1", "fp3")) + ); + + Assert.assertEquals("Should return requested fingerprints", 2, records.size()); + + Set retrievedFingerprints = records.stream() + .map(IndexingStateRecord::getFingerprint) + .collect(Collectors.toSet()); + Assert.assertEquals("Should contain only requested fingerprints", Set.of("fp1", "fp3"), retrievedFingerprints); + } + + @Test + public void test_retrieveIndexingStatesForFingerprints_largeBatch() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + // Insert 150 indexing states (exceeds batching threshold of 100) + Map indexingStates = new HashMap<>(); + Set expectedFingerprints = new HashSet<>(); + for (int i = 0; i < 150; i++) { + String fingerprint = "fp" + i; + indexingStates.put(fingerprint, createTestIndexingState()); + expectedFingerprints.add(fingerprint); + } + insertIndexingStates(indexingStates); + + // Request all fingerprints + List records = read( + sql -> sql.retrieveIndexingStatesForFingerprints(expectedFingerprints) + ); + + Assert.assertEquals("Should return all fingerprints across multiple batches", 150, records.size()); + + Set retrievedFingerprints = records.stream() + .map(IndexingStateRecord::getFingerprint) + .collect(Collectors.toSet()); + Assert.assertEquals("Should contain all requested fingerprints", expectedFingerprints, retrievedFingerprints); + } + + @Test + public void test_retrieveIndexingStatesForFingerprints_nonexistentFingerprints() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + Map indexingStates = new HashMap<>(); + indexingStates.put("fp1", createTestIndexingState()); + insertIndexingStates(indexingStates); + + // Request fingerprints that don't exist + List records = read( + sql -> sql.retrieveIndexingStatesForFingerprints(Set.of("fp999", "fp888")) + ); + + Assert.assertTrue("Should return empty list when fingerprints don't exist", records.isEmpty()); + } + + @Test + public void test_retrieveIndexingStatesForFingerprints_mixedExistingAndNonexistent() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + Map indexingStates = new HashMap<>(); + indexingStates.put("fp1", createTestIndexingState()); + indexingStates.put("fp2", createTestIndexingState()); + insertIndexingStates(indexingStates); + + // Mix existing and non-existing fingerprints + List records = read( + sql -> sql.retrieveIndexingStatesForFingerprints(Set.of("fp1", "fp999", "fp2", "fp888")) + ); + + Assert.assertEquals("Should return only existing fingerprints", 2, records.size()); + + Set retrievedFingerprints = records.stream() + .map(IndexingStateRecord::getFingerprint) + .collect(Collectors.toSet()); + Assert.assertEquals("Should contain only existing fingerprints", Set.of("fp1", "fp2"), retrievedFingerprints); + } + + @Test + public void test_retrieveIndexingStatesForFingerprints_onlyReturnsUsedStates() + { + derbyConnectorRule.getConnector().createIndexingStatesTable(); + + Map indexingStates = new HashMap<>(); + indexingStates.put("fp1", createTestIndexingState()); + indexingStates.put("fp2", createTestIndexingState()); + insertIndexingStates(indexingStates); + + // Mark fp2 as unused + markIndexingStateAsUnused("fp2"); + + List records = read( + sql -> sql.retrieveIndexingStatesForFingerprints(Set.of("fp1", "fp2")) + ); + + Assert.assertEquals("Should only return used indexing states", 1, records.size()); + Assert.assertEquals("Should return fp1", "fp1", records.get(0).getFingerprint()); + } + + // ==================== Helper Methods for Indexing State Tests ==================== + + private CompactionState createTestIndexingState() + { + return new CompactionState( + new DynamicPartitionsSpec(100, null), + DimensionsSpec.EMPTY, + null, + null, + IndexSpec.getDefault(), + null, + null + ); + } + + private void insertIndexingStates(Map indexingStates) + { + ObjectMapper mapper = TestHelper.JSON_MAPPER; + MetadataStorageTablesConfig tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); + SqlIndexingStateStorage manager = new SqlIndexingStateStorage( + tablesConfig, + mapper, + derbyConnectorRule.getConnector() + ); + + derbyConnectorRule.getConnector().retryWithHandle(handle -> { + for (Map.Entry entry : indexingStates.entrySet()) { + manager.upsertIndexingState(TestDataSource.WIKI, entry.getKey(), entry.getValue(), DateTimes.nowUtc()); + } + return null; + }); + } + + private void insertSegmentWithIndexingState( + String segmentId, + String indexingStateFingerprint, + boolean used + ) + { + MetadataStorageTablesConfig tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); + DerbyConnector connector = derbyConnectorRule.getConnector(); + + connector.retryWithHandle(handle -> { + handle.createStatement( + "INSERT INTO " + tablesConfig.getSegmentsTable() + " " + + "(id, dataSource, created_date, start, \"end\", partitioned, version, used, payload, " + + "used_status_last_updated, indexing_state_fingerprint) " + + "VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload, " + + ":used_status_last_updated, :indexing_state_fingerprint)" + ) + .bind("id", segmentId) + .bind("dataSource", TestDataSource.WIKI) + .bind("created_date", DateTimes.nowUtc().toString()) + .bind("start", JAN_1.toString()) + .bind("end", JAN_1.plusDays(1).toString()) + .bind("partitioned", false) + .bind("version", V1) + .bind("used", used) + .bind("payload", TestHelper.JSON_MAPPER.writeValueAsBytes(WIKI_SEGMENTS_2X5D.get(0))) + .bind("used_status_last_updated", DateTimes.nowUtc().toString()) + .bind("indexing_state_fingerprint", indexingStateFingerprint) + .execute(); + return null; + }); + } + + private void markIndexingStateAsUnused(String fingerprint) + { + MetadataStorageTablesConfig tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get(); + DerbyConnector connector = derbyConnectorRule.getConnector(); + + connector.retryWithHandle(handle -> { + handle.createStatement( + "UPDATE " + tablesConfig.getIndexingStatesTable() + " " + + "SET used = false " + + "WHERE fingerprint = :fingerprint" + ) + .bind("fingerprint", fingerprint) + .execute(); + return null; + }); + } } diff --git a/server/src/test/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2Test.java b/server/src/test/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2Test.java index 08098decdab1..293b076b1115 100644 --- a/server/src/test/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2Test.java +++ b/server/src/test/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2Test.java @@ -33,6 +33,7 @@ import org.apache.druid.metadata.segment.cache.SegmentMetadataCache; import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.IndexingStateCache; import org.apache.druid.segment.metadata.NoopSegmentSchemaCache; import org.apache.druid.segment.metadata.SegmentSchemaCache; import org.apache.druid.server.coordinator.CreateDataSegments; @@ -74,6 +75,7 @@ public void setup() throws Exception { setUp(derbyConnectorRule); connector.createPendingSegmentsTable(); + connector.createIndexingStatesTable(); emitter = new StubServiceEmitter(); @@ -91,6 +93,7 @@ private void initManager( Suppliers.ofInstance(new SegmentsMetadataManagerConfig(Period.seconds(1), cacheMode, null)), Suppliers.ofInstance(storageConfig), useSchemaCache ? new SegmentSchemaCache() : new NoopSegmentSchemaCache(), + new IndexingStateCache(), connector, (poolSize, name) -> new WrappingScheduledExecutorService(name, segmentMetadataCacheExec, false), emitter diff --git a/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemoryDatasourceSegmentCacheTest.java b/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemoryDatasourceSegmentCacheTest.java index c78cf31bfdb4..ae3deda0eadd 100644 --- a/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemoryDatasourceSegmentCacheTest.java +++ b/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemoryDatasourceSegmentCacheTest.java @@ -220,6 +220,7 @@ public void testInsertSegments_canMarkItAsUnused() true, null, null, + null, null ); @@ -779,7 +780,8 @@ private static DataSegmentPlus updateSegment(DataSegmentPlus segment, DateTime n segment.getUsed(), segment.getSchemaFingerprint(), segment.getNumRows(), - segment.getUpgradedFromSegmentId() + segment.getUpgradedFromSegmentId(), + segment.getIndexingStateFingerprint() ); } diff --git a/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java index fa652a2cfde9..e739744480eb 100644 --- a/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java @@ -39,6 +39,7 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; import org.apache.druid.segment.metadata.FingerprintGenerator; +import org.apache.druid.segment.metadata.IndexingStateCache; import org.apache.druid.segment.metadata.NoopSegmentSchemaCache; import org.apache.druid.segment.metadata.SegmentSchemaCache; import org.apache.druid.segment.metadata.SegmentSchemaTestUtils; @@ -76,6 +77,7 @@ public class HeapMemorySegmentMetadataCacheTest private HeapMemorySegmentMetadataCache cache; private SegmentSchemaCache schemaCache; + private IndexingStateCache indexingStateCache; private SegmentSchemaTestUtils schemaTestUtils; @Before @@ -89,6 +91,7 @@ public void setup() derbyConnector.createSegmentTable(); derbyConnector.createSegmentSchemasTable(); derbyConnector.createPendingSegmentsTable(); + derbyConnector.createIndexingStatesTable(); schemaTestUtils = new SegmentSchemaTestUtils(derbyConnectorRule, derbyConnector, TestHelper.JSON_MAPPER); EmittingLogger.registerEmitter(serviceEmitter); @@ -119,11 +122,13 @@ private void setupTargetWithCaching(SegmentMetadataCache.UsageMode cacheMode, bo final SegmentsMetadataManagerConfig metadataManagerConfig = new SegmentsMetadataManagerConfig(null, cacheMode, null); schemaCache = useSchemaCache ? new SegmentSchemaCache() : new NoopSegmentSchemaCache(); + indexingStateCache = new IndexingStateCache(); cache = new HeapMemorySegmentMetadataCache( TestHelper.JSON_MAPPER, () -> metadataManagerConfig, derbyConnectorRule.metadataTablesConfigSupplier(), schemaCache, + indexingStateCache, derbyConnector, executorFactory, serviceEmitter @@ -512,6 +517,7 @@ public void testSync_updatesUsedSegment_ifCacheHasOlderEntry() true, null, null, + null, null ); updateSegmentInMetadataStore(updatedSegment); diff --git a/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java b/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java index 294025f25065..2a1af270abaa 100644 --- a/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java +++ b/server/src/test/java/org/apache/druid/rpc/indexing/OverlordClientImplTest.java @@ -504,7 +504,8 @@ public void test_getClusterCompactionConfig() 101, new NewestSegmentFirstPolicy(null), true, - CompactionEngine.MSQ + CompactionEngine.MSQ, + true ); serviceClient.expectAndRespond( new RequestBuilder(HttpMethod.GET, "/druid/indexer/v1/compaction/config/cluster"), @@ -523,7 +524,7 @@ public void test_getClusterCompactionConfig() public void test_updateClusterCompactionConfig() throws ExecutionException, InterruptedException, JsonProcessingException { - final ClusterCompactionConfig config = new ClusterCompactionConfig(null, null, null, null, null); + final ClusterCompactionConfig config = new ClusterCompactionConfig(null, null, null, null, null, null); serviceClient.expectAndRespond( new RequestBuilder(HttpMethod.POST, "/druid/indexer/v1/compaction/config/cluster") .jsonContent(jsonMapper, config), diff --git a/server/src/test/java/org/apache/druid/segment/metadata/HeapMemoryIndexingStateStorage.java b/server/src/test/java/org/apache/druid/segment/metadata/HeapMemoryIndexingStateStorage.java new file mode 100644 index 000000000000..e22c5430e52a --- /dev/null +++ b/server/src/test/java/org/apache/druid/segment/metadata/HeapMemoryIndexingStateStorage.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import org.apache.druid.timeline.CompactionState; +import org.joda.time.DateTime; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * In-memory implementation of {@link IndexingStateStorage} that stores + * indexing state fingerprints and states in heap memory without requiring a database. + *

+ * Useful for simulations and unit tests where database persistence is not needed. + * Database-specific operations (cleanup, unused marking) are no-ops in this implementation. + */ +public class HeapMemoryIndexingStateStorage implements IndexingStateStorage +{ + private final ConcurrentMap fingerprintToStateMap; + + /** + * Creates an in-memory indexing state manager with a default deterministic mapper. + * This is a convenience constructor for tests and simulations. + */ + public HeapMemoryIndexingStateStorage() + { + this.fingerprintToStateMap = new ConcurrentHashMap<>(); + } + + @Override + public void upsertIndexingState( + final String dataSource, + final String fingerprint, + final CompactionState indexingState, + final DateTime updateTime + ) + { + // Store in memory for lookup during simulations/tests + this.fingerprintToStateMap.put(fingerprint, indexingState); + } + + @Override + public int markUnreferencedIndexingStatesAsUnused() + { + return 0; + } + + @Override + public List findReferencedIndexingStateMarkedAsUnused() + { + return List.of(); + } + + @Override + public int markIndexingStatesAsUsed(List stateFingerprints) + { + return 0; + } + + @Override + public int markIndexingStatesAsActive(List stateFingerprints) + { + return 0; + } + + @Override + public int deletePendingIndexingStatesOlderThan(long timestamp) + { + return 0; + } + + @Override + public int deleteUnusedIndexingStatesOlderThan(long timestamp) + { + return 0; + } + + /** + * Gets all stored indexing states. For test verification only. + */ + public Map getAllStoredStates() + { + return Map.copyOf(fingerprintToStateMap); + } + + /** + * Clears all stored indexing states. Useful for test cleanup or resetting + * state between test runs. + */ + public void clear() + { + fingerprintToStateMap.clear(); + } + + /** + * Returns the number of stored indexing state fingerprints. + */ + public int size() + { + return fingerprintToStateMap.size(); + } + +} diff --git a/server/src/test/java/org/apache/druid/segment/metadata/IndexingStateCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/IndexingStateCacheTest.java new file mode 100644 index 000000000000..4027edaf5f08 --- /dev/null +++ b/server/src/test/java/org/apache/druid/segment/metadata/IndexingStateCacheTest.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.timeline.CompactionState; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class IndexingStateCacheTest +{ + private IndexingStateCache cache; + + @BeforeEach + public void setUp() + { + cache = new IndexingStateCache(); + } + + @Test + public void test_isEnabled_returnsTrue() + { + assertTrue(cache.isEnabled()); + } + + @Test + public void test_getIndexingStateByFingerprint_emptyCache_returnsEmpty() + { + Optional result = cache.getIndexingStateByFingerprint("nonexistent"); + assertFalse(result.isPresent()); + } + + @Test + public void test_getIndexingStateByFingerprint_nullFingerprint_returnsEmpty() + { + Optional result = cache.getIndexingStateByFingerprint(null); + assertFalse(result.isPresent()); + } + + @Test + public void test_resetIndexingStatesForPublishedSegments_andThen_getIndexingStateByFingerprint() + { + CompactionState state1 = createTestIndexingState(); + CompactionState state2 = createTestIndexingState(); + + Map stateMap = new HashMap<>(); + stateMap.put("fingerprint1", state1); + stateMap.put("fingerprint2", state2); + + cache.resetIndexingStatesForPublishedSegments(stateMap); + + Optional result1 = cache.getIndexingStateByFingerprint("fingerprint1"); + assertTrue(result1.isPresent()); + assertEquals(state1, result1.get()); + + Optional result2 = cache.getIndexingStateByFingerprint("fingerprint2"); + assertTrue(result2.isPresent()); + assertEquals(state2, result2.get()); + + Optional result3 = cache.getIndexingStateByFingerprint("nonexistent"); + assertFalse(result3.isPresent()); + } + + @Test + public void test_getPublishedIndexingStateMap_returnsImmutableSnapshot() + { + CompactionState state1 = createTestIndexingState(); + Map stateMap = new HashMap<>(); + stateMap.put("fingerprint1", state1); + + cache.resetIndexingStatesForPublishedSegments(stateMap); + + Map retrieved = cache.getPublishedIndexingStateMap(); + assertEquals(1, retrieved.size()); + assertEquals(state1, retrieved.get("fingerprint1")); + } + + @Test + public void test_clear_emptiesCache() + { + CompactionState state1 = createTestIndexingState(); + Map stateMap = new HashMap<>(); + stateMap.put("fingerprint1", state1); + + cache.resetIndexingStatesForPublishedSegments(stateMap); + + Optional beforeClear = cache.getIndexingStateByFingerprint("fingerprint1"); + assertTrue(beforeClear.isPresent()); + + cache.clear(); + + Optional afterClear = cache.getIndexingStateByFingerprint("fingerprint1"); + assertFalse(afterClear.isPresent()); + + Map mapAfterClear = cache.getPublishedIndexingStateMap(); + assertEquals(0, mapAfterClear.size()); + } + + @Test + public void test_stats_trackHitsAndMisses() + { + CompactionState state1 = createTestIndexingState(); + Map stateMap = new HashMap<>(); + stateMap.put("fingerprint1", state1); + + cache.resetIndexingStatesForPublishedSegments(stateMap); + + // Generate 3 hits + cache.getIndexingStateByFingerprint("fingerprint1"); + cache.getIndexingStateByFingerprint("fingerprint1"); + cache.getIndexingStateByFingerprint("fingerprint1"); + + // Generate 2 misses + cache.getIndexingStateByFingerprint("nonexistent1"); + cache.getIndexingStateByFingerprint("nonexistent2"); + + Map stats = cache.getAndResetStats(); + assertEquals(3, stats.get(Metric.INDEXING_STATE_CACHE_HITS)); + assertEquals(2, stats.get(Metric.INDEXING_STATE_CACHE_MISSES)); + assertEquals(1, stats.get(Metric.INDEXING_STATE_CACHE_FINGERPRINTS)); + } + + @Test + public void test_stats_resetAfterReading() + { + CompactionState state1 = createTestIndexingState(); + Map stateMap = new HashMap<>(); + stateMap.put("fingerprint1", state1); + + cache.resetIndexingStatesForPublishedSegments(stateMap); + + // Generate hits and misses + cache.getIndexingStateByFingerprint("fingerprint1"); + cache.getIndexingStateByFingerprint("nonexistent"); + + Map stats1 = cache.getAndResetStats(); + assertEquals(1, stats1.get(Metric.INDEXING_STATE_CACHE_HITS)); + assertEquals(1, stats1.get(Metric.INDEXING_STATE_CACHE_MISSES)); + + // Stats should be reset after reading + Map stats2 = cache.getAndResetStats(); + assertEquals(0, stats2.get(Metric.INDEXING_STATE_CACHE_HITS)); + assertEquals(0, stats2.get(Metric.INDEXING_STATE_CACHE_MISSES)); + assertEquals(1, stats2.get(Metric.INDEXING_STATE_CACHE_FINGERPRINTS)); // Fingerprints count doesn't reset + } + + @Test + public void test_multipleResets_replacesCache() + { + CompactionState state1 = createTestIndexingState(); + CompactionState state2 = createTestIndexingState(); + + // First reset + Map firstMap = new HashMap<>(); + firstMap.put("fingerprint1", state1); + cache.resetIndexingStatesForPublishedSegments(firstMap); + + Optional result1 = cache.getIndexingStateByFingerprint("fingerprint1"); + assertTrue(result1.isPresent()); + assertEquals(state1, result1.get()); + + // Second reset with different data + Map secondMap = new HashMap<>(); + secondMap.put("fingerprint2", state2); + cache.resetIndexingStatesForPublishedSegments(secondMap); + + // Old fingerprint should be gone + Optional oldResult = cache.getIndexingStateByFingerprint("fingerprint1"); + assertFalse(oldResult.isPresent()); + + // New fingerprint should exist + Optional newResult = cache.getIndexingStateByFingerprint("fingerprint2"); + assertTrue(newResult.isPresent()); + assertEquals(state2, newResult.get()); + } + + @Test + public void test_resetWithEmptyMap() + { + CompactionState state1 = createTestIndexingState(); + Map stateMap = new HashMap<>(); + stateMap.put("fingerprint1", state1); + + cache.resetIndexingStatesForPublishedSegments(stateMap); + + Optional beforeReset = cache.getIndexingStateByFingerprint("fingerprint1"); + assertTrue(beforeReset.isPresent()); + + // Reset with empty map + cache.resetIndexingStatesForPublishedSegments(Collections.emptyMap()); + + Optional afterReset = cache.getIndexingStateByFingerprint("fingerprint1"); + assertFalse(afterReset.isPresent()); + + Map stats = cache.getAndResetStats(); + assertEquals(0, stats.get(Metric.INDEXING_STATE_CACHE_FINGERPRINTS)); + } + + @Test + public void test_addIndexingState_addsNewStateToCache() + { + CompactionState state = createTestIndexingState(); + String fingerprint = "test_fingerprint_123"; + + // Initially, cache should not have the state + assertEquals(Optional.empty(), cache.getIndexingStateByFingerprint(fingerprint)); + + // Add the state to cache + cache.addIndexingState(fingerprint, state); + + // Now cache should have the state + assertEquals(Optional.of(state), cache.getIndexingStateByFingerprint(fingerprint)); + } + + @Test + public void test_addIndexingState_withDifferentStateForSameFingerprint_updatesCache() + { + CompactionState state1 = createTestIndexingState(); + CompactionState state2 = new CompactionState( + new DynamicPartitionsSpec(200, null), + DimensionsSpec.EMPTY, + null, + null, + IndexSpec.getDefault(), + null, + null + ); + String fingerprint = "same_fp"; + + // Add first state + cache.addIndexingState(fingerprint, state1); + assertEquals(Optional.of(state1), cache.getIndexingStateByFingerprint(fingerprint)); + + // Add different state with same fingerprint + cache.addIndexingState(fingerprint, state2); + + // Cache should now have the new state + assertEquals(Optional.of(state2), cache.getIndexingStateByFingerprint(fingerprint)); + } + + @Test + public void test_addIndexingState_withNullFingerprint_doesNothing() + { + CompactionState state = createTestIndexingState(); + + cache.addIndexingState(null, state); + + // Cache should remain empty + assertEquals(0, cache.getPublishedIndexingStateMap().size()); + } + + @Test + public void test_addIndexingState_withNullState_doesNothing() + { + cache.addIndexingState("some_fp", null); + + // Cache should remain empty + assertEquals(0, cache.getPublishedIndexingStateMap().size()); + } + + private CompactionState createTestIndexingState() + { + return new CompactionState( + new DynamicPartitionsSpec(100, null), + DimensionsSpec.EMPTY, + null, + null, + IndexSpec.getDefault(), + null, + null + ); + } +} diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SqlIndexingStateStorageTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SqlIndexingStateStorageTest.java new file mode 100644 index 000000000000..fca63f86131e --- /dev/null +++ b/server/src/test/java/org/apache/druid/segment/metadata/SqlIndexingStateStorageTest.java @@ -0,0 +1,613 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.indexer.partitions.HashedPartitionsSpec; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.metadata.MetadataStorageTablesConfig; +import org.apache.druid.metadata.TestDerbyConnector; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.timeline.CompactionState; +import org.joda.time.DateTime; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SqlIndexingStateStorageTest +{ + @RegisterExtension + public static final TestDerbyConnector.DerbyConnectorRule5 DERBY_CONNECTOR_RULE = + new TestDerbyConnector.DerbyConnectorRule5(); + + private final ObjectMapper jsonMapper = new DefaultObjectMapper(); + + private static TestDerbyConnector derbyConnector; + private static MetadataStorageTablesConfig tablesConfig; + private SqlIndexingStateStorage manager; + + private static DefaultIndexingStateFingerprintMapper fingerprintMapper; + + @BeforeAll + public static void setUpClass() + { + derbyConnector = DERBY_CONNECTOR_RULE.getConnector(); + tablesConfig = DERBY_CONNECTOR_RULE.metadataTablesConfigSupplier().get(); + derbyConnector.createIndexingStatesTable(); + derbyConnector.createSegmentTable(); + fingerprintMapper = new DefaultIndexingStateFingerprintMapper( + new NoopIndexingStateCache(), + new DefaultObjectMapper() + ); + } + + @BeforeEach + public void setUp() + { + derbyConnector.retryWithHandle(handle -> { + handle.createStatement("DELETE FROM " + tablesConfig.getSegmentsTable()).execute(); + handle.createStatement("DELETE FROM " + tablesConfig.getIndexingStatesTable()).execute(); + return null; + }); + + manager = new SqlIndexingStateStorage(tablesConfig, jsonMapper, derbyConnector); + } + + @Test + public void test_upsertIndexingState_successfullyInsertsIntoDatabase() + { + CompactionState state1 = createTestIndexingState(); + String fingerprint = "fingerprint_abc123"; + + manager.upsertIndexingState( + "testDatasource", + fingerprint, + state1, + DateTimes.nowUtc() + ); + + // Verify the state was inserted into database by checking count + Integer count = derbyConnector.retryWithHandle(handle -> + handle.createQuery( + "SELECT COUNT(*) FROM " + tablesConfig.getIndexingStatesTable() + + " WHERE fingerprint = :fp" + ).bind("fp", fingerprint) + .map((i, r, ctx) -> r.getInt(1)) + .first() + ); + assertEquals(1, count); + } + + @Test + public void test_upsertIndexingState_andThen_markUnreferencedIndexingStateAsUnused_andThen_markIndexingStatesAsUsed() + { + CompactionState state1 = createTestIndexingState(); + String fingerprint = "fingerprint_abc123"; + + manager.upsertIndexingState( + "testDatasource", + fingerprint, + state1, + DateTimes.nowUtc() + ); + manager.markIndexingStatesAsActive(List.of(fingerprint)); + + assertEquals(1, manager.markUnreferencedIndexingStatesAsUnused()); + assertEquals(1, manager.markIndexingStatesAsUsed(List.of(fingerprint))); + } + + @Test + public void test_findReferencedIndexingStateMarkedAsUnused() + { + CompactionState state1 = createTestIndexingState(); + String fingerprint = "fingerprint_abc123"; + + manager.upsertIndexingState( + "testDatasource", + fingerprint, + state1, + DateTimes.nowUtc() + ); + manager.markIndexingStatesAsActive(List.of(fingerprint)); + + manager.markUnreferencedIndexingStatesAsUnused(); + assertEquals(0, manager.findReferencedIndexingStateMarkedAsUnused().size()); + + derbyConnector.retryWithHandle(handle -> { + handle.createStatement( + "INSERT INTO " + tablesConfig.getSegmentsTable() + " " + + "(id, dataSource, created_date, start, \"end\", partitioned, version, used, payload, " + + "used_status_last_updated, indexing_state_fingerprint) " + + "VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload, " + + ":used_status_last_updated, :indexing_state_fingerprint)" + ) + .bind("id", "testSegment_2024-01-01_2024-01-02_v1_0") + .bind("dataSource", "testDatasource") + .bind("created_date", DateTimes.nowUtc().toString()) + .bind("start", "2024-01-01T00:00:00.000Z") + .bind("end", "2024-01-02T00:00:00.000Z") + .bind("partitioned", 0) + .bind("version", "v1") + .bind("used", true) + .bind("payload", new byte[]{}) // Empty payload is fine for this test + .bind("used_status_last_updated", DateTimes.nowUtc().toString()) + .bind("indexing_state_fingerprint", fingerprint) + .execute(); + return null; + }); + + List referenced = manager.findReferencedIndexingStateMarkedAsUnused(); + assertEquals(1, referenced.size()); + assertEquals(fingerprint, referenced.get(0)); + } + + @Test + public void test_deleteIndexingStatesOlderThan_deletesOnlyOldUnusedStates() + { + DateTime now = DateTimes.nowUtc(); + DateTime oldTime = now.minusDays(60); + DateTime recentTime = now.minusDays(15); + DateTime cutoffTime = now.minusDays(30); + + String oldFingerprint = "old_fp_should_delete"; + String recentFingerprint = "recent_fp_should_keep"; + + CompactionState oldState = createTestIndexingState(); + CompactionState recentState = createTestIndexingState(); + + // Insert old unused state (60 days old) + derbyConnector.retryWithHandle(handle -> { + handle.createStatement( + "INSERT INTO " + tablesConfig.getIndexingStatesTable() + " " + + "(created_date, dataSource, fingerprint, payload, used, pending, used_status_last_updated) " + + "VALUES (:cd, :ds, :fp, :pl, :used, :pending, :updated)" + ) + .bind("cd", oldTime.toString()) + .bind("ds", "testDatasource") + .bind("fp", oldFingerprint) + .bind("pl", jsonMapper.writeValueAsBytes(oldState)) + .bind("used", false) + .bind("pending", false) + .bind("updated", oldTime.toString()) + .execute(); + return null; + }); + + // Insert recent unused state (15 days old) + derbyConnector.retryWithHandle(handle -> { + handle.createStatement( + "INSERT INTO " + tablesConfig.getIndexingStatesTable() + " " + + "(created_date, dataSource, fingerprint, payload, used, pending, used_status_last_updated) " + + "VALUES (:cd, :ds, :fp, :pl, :used, :pending, :updated)" + ) + .bind("cd", recentTime.toString()) + .bind("ds", "testDatasource") + .bind("fp", recentFingerprint) + .bind("pl", jsonMapper.writeValueAsBytes(recentState)) + .bind("used", false) + .bind("pending", false) + .bind("updated", recentTime.toString()) + .execute(); + return null; + }); + + // Delete states older than 30 days + int deleted = manager.deleteUnusedIndexingStatesOlderThan(cutoffTime.getMillis()); + assertEquals(1, deleted); + + // Verify only 1 state remains in the table + Integer count = derbyConnector.retryWithHandle(handle -> + handle.createQuery("SELECT COUNT(*) FROM " + tablesConfig.getIndexingStatesTable()) + .map((i, r, ctx) -> r.getInt(1)) + .first() + ); + assertEquals(1, count); + } + + @Test + public void test_upsertIndexingState_withNullState_throwsException() + { + Exception exception = assertThrows( + Exception.class, + () -> manager.upsertIndexingState("ds", "somePrint", null, DateTimes.nowUtc()) + ); + + assertTrue( + exception.getMessage().contains("indexingState cannot be null"), + "Exception message should contain 'indexingState cannot be null'" + ); + } + + @Test + public void test_upsertIndexingState_withEmptyFingerprint_throwsException() + { + // The exception ends up wrapped in a sql exception doe to the retryWithHandle so we will just check the message + Exception exception = assertThrows( + Exception.class, + () -> manager.upsertIndexingState("ds", "", createBasicIndexingState(), DateTimes.nowUtc()) + ); + + assertTrue( + exception.getMessage().contains("fingerprint cannot be empty"), + "Exception message should contain 'fingerprint cannot be empty'" + ); + } + + @Test + public void test_upsertIndexingState_verifyExistingFingerprintMarkedUsed() + { + String fingerprint = "existing_fingerprint"; + CompactionState state = createTestIndexingState(); + + // Persist initially + manager.upsertIndexingState("ds1", fingerprint, state, DateTimes.nowUtc()); + + // Verify it's marked as used + Boolean usedBefore = derbyConnector.retryWithHandle(handle -> + handle.createQuery( + "SELECT used FROM " + tablesConfig.getIndexingStatesTable() + + " WHERE fingerprint = :fp" + ).bind("fp", fingerprint) + .map((i, r, ctx) -> r.getBoolean("used")) + .first() + ); + assertTrue(usedBefore); + + // Manually mark it as unused + derbyConnector.retryWithHandle(handle -> + handle.createStatement( + "UPDATE " + tablesConfig.getIndexingStatesTable() + + " SET used = false WHERE fingerprint = :fp" + ).bind("fp", fingerprint).execute() + ); + + // Persist again with the same fingerprint (should UPDATE, not INSERT) + manager.upsertIndexingState("ds1", fingerprint, state, DateTimes.nowUtc()); + + // Verify it's marked as used again + Boolean usedAfter = derbyConnector.retryWithHandle(handle -> + handle.createQuery( + "SELECT used FROM " + tablesConfig.getIndexingStatesTable() + + " WHERE fingerprint = :fp" + ).bind("fp", fingerprint) + .map((i, r, ctx) -> r.getBoolean("used")) + .first() + ); + assertTrue(usedAfter); + + // Verify only 1 row exists (no duplicate insert) + Integer count = derbyConnector.retryWithHandle(handle -> + handle.createQuery("SELECT COUNT(*) FROM " + tablesConfig.getIndexingStatesTable()) + .map((i, r, ctx) -> r.getInt(1)) + .first() + ); + assertEquals(1, count); + } + + @Test + public void test_upsertIndexingState_whenAlreadyUsed_skipsUpdate() + { + String fingerprint = "already_used_fingerprint"; + CompactionState state = createTestIndexingState(); + DateTime initialTime = DateTimes.of("2024-01-01T00:00:00.000Z"); + + // Insert fingerprint as used initially + manager.upsertIndexingState("ds1", fingerprint, state, initialTime); + + // Verify it's marked as used with the initial timestamp + DateTime usedStatusBeforeUpdate = derbyConnector.retryWithHandle(handle -> + handle.createQuery( + "SELECT used_status_last_updated FROM " + tablesConfig.getIndexingStatesTable() + + " WHERE fingerprint = :fp" + ).bind("fp", fingerprint) + .map((i, r, ctx) -> DateTimes.of(r.getString("used_status_last_updated"))) + .first() + ); + assertEquals(initialTime, usedStatusBeforeUpdate); + + // Call upsert again with a different timestamp + // Since the fingerprint is already used, this should skip the UPDATE + DateTime laterTime = DateTimes.of("2024-01-02T00:00:00.000Z"); + manager.upsertIndexingState("ds1", fingerprint, state, laterTime); + + // Verify the used_status_last_updated timestamp DID NOT change + DateTime usedStatusAfterUpdate = derbyConnector.retryWithHandle(handle -> + handle.createQuery( + "SELECT used_status_last_updated FROM " + tablesConfig.getIndexingStatesTable() + + " WHERE fingerprint = :fp" + ).bind("fp", fingerprint) + .map((i, r, ctx) -> DateTimes.of(r.getString("used_status_last_updated"))) + .first() + ); + + assertEquals( + initialTime, + usedStatusAfterUpdate, + "used_status_last_updated should not change when upserting an already-used fingerprint" + ); + + // Verify still only 1 row + Integer count = derbyConnector.retryWithHandle(handle -> + handle.createQuery("SELECT COUNT(*) FROM " + tablesConfig.getIndexingStatesTable()) + .map((i, r, ctx) -> r.getInt(1)) + .first() + ); + assertEquals(1, count); + } + + @Test + public void test_markIndexingStateAsUsed_withEmptyList_returnsZero() + { + assertEquals(0, manager.markIndexingStatesAsUsed(List.of())); + } + + @Test + public void test_markIndexingStatesAsActive_marksPendingStateAsActive() + { + String fingerprint = "pending_fingerprint"; + String fingerprint2 = "other_pending_fingerprint"; + CompactionState state = createTestIndexingState(); + + manager.upsertIndexingState("ds1", fingerprint, state, DateTimes.nowUtc()); + manager.upsertIndexingState("ds1", fingerprint2, state, DateTimes.nowUtc()); + + Boolean pendingBefore = derbyConnector.retryWithHandle(handle -> + handle.createQuery("SELECT pending FROM " + tablesConfig.getIndexingStatesTable() + " WHERE fingerprint = :fp") + .bind("fp", fingerprint) + .map((i, r, ctx) -> r.getBoolean("pending")) + .first() + ); + assertTrue(pendingBefore); + + int rowsUpdated = manager.markIndexingStatesAsActive(List.of(fingerprint, fingerprint2)); + assertEquals(2, rowsUpdated); + + Boolean pendingAfter = derbyConnector.retryWithHandle(handle -> + handle.createQuery("SELECT pending FROM " + tablesConfig.getIndexingStatesTable() + " WHERE fingerprint = :fp") + .bind("fp", fingerprint) + .map((i, r, ctx) -> r.getBoolean("pending")) + .first() + ); + assertEquals(false, pendingAfter); + } + + @Test + public void test_markIndexingStatesAsActive_idempotent_returnsZeroWhenAlreadyActive() + { + String fingerprint = "already_active_fingerprint"; + CompactionState state = createTestIndexingState(); + + manager.upsertIndexingState("ds1", fingerprint, state, DateTimes.nowUtc()); + + int firstUpdate = manager.markIndexingStatesAsActive(List.of(fingerprint)); + assertEquals(1, firstUpdate); + + int secondUpdate = manager.markIndexingStatesAsActive(List.of(fingerprint)); + assertEquals(0, secondUpdate); + + Boolean pending = derbyConnector.retryWithHandle(handle -> + handle.createQuery("SELECT pending FROM " + tablesConfig.getIndexingStatesTable() + " WHERE fingerprint = :fp") + .bind("fp", fingerprint) + .map((i, r, ctx) -> r.getBoolean("pending")) + .first() + ); + assertEquals(false, pending); + } + + @Test + public void test_markIndexingStatesAsActive_nonExistentFingerprint_returnsZero() + { + int rowsUpdated = manager.markIndexingStatesAsActive(List.of("does_not_exist")); + assertEquals(0, rowsUpdated); + } + + // ===== Fingerprint Generation Tests ===== + + @Test + public void test_generateIndexingStateFingerprint_deterministicFingerprinting() + { + CompactionState indexingState1 = createBasicIndexingState(); + CompactionState indexingState2 = createBasicIndexingState(); + + String fingerprint1 = fingerprintMapper.generateFingerprint("test-ds", indexingState1); + String fingerprint2 = fingerprintMapper.generateFingerprint("test-ds", indexingState2); + + assertEquals( + fingerprint1, + fingerprint2, + "Same CompactionState should produce identical fingerprints when datasource is same" + ); + } + + @Test + public void test_generateIndexingStateFingerprint_differentDatasourcesWithSameState_differentFingerprints() + { + CompactionState indexingState = createBasicIndexingState(); + + String fingerprint1 = fingerprintMapper.generateFingerprint("ds1", indexingState); + String fingerprint2 = fingerprintMapper.generateFingerprint("ds2", indexingState); + + assertNotEquals( + fingerprint1, + fingerprint2, + "Different datasources should produce different fingerprints despite same state" + ); + } + + @Test + public void test_generateIndexingStateFingerprint_metricsListOrderDifferenceResultsInNewFingerprint() + { + List metrics1 = Arrays.asList( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("sum", "value") + ); + + List metrics2 = Arrays.asList( + new LongSumAggregatorFactory("sum", "value"), + new CountAggregatorFactory("count") + ); + + CompactionState state1 = new CompactionState( + new DynamicPartitionsSpec(null, null), + DimensionsSpec.EMPTY, + metrics1, + null, + IndexSpec.getDefault(), + null, + null + ); + + CompactionState state2 = new CompactionState( + new DynamicPartitionsSpec(null, null), + DimensionsSpec.EMPTY, + metrics2, + null, + IndexSpec.getDefault(), + null, + null + ); + + String fingerprint1 = fingerprintMapper.generateFingerprint("test-ds", state1); + String fingerprint2 = fingerprintMapper.generateFingerprint("test-ds", state2); + + assertNotEquals( + fingerprint1, + fingerprint2, + "Metrics order currently matters (arrays preserve order in JSON)" + ); + } + + @Test + public void test_generateIndexingStateFingerprint_dimensionsListOrderDifferenceResultsInNewFingerprint() + { + DimensionsSpec dimensions1 = new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim2", "dim3")) + ); + + DimensionsSpec dimensions2 = new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim3", "dim2", "dim1")) + ); + + CompactionState state1 = new CompactionState( + new DynamicPartitionsSpec(null, null), + dimensions1, + Collections.singletonList(new CountAggregatorFactory("count")), + null, + IndexSpec.getDefault(), + null, + null + ); + + CompactionState state2 = new CompactionState( + new DynamicPartitionsSpec(null, null), + dimensions2, + Collections.singletonList(new CountAggregatorFactory("count")), + null, + IndexSpec.getDefault(), + null, + null + ); + + String fingerprint1 = fingerprintMapper.generateFingerprint("test-ds", state1); + String fingerprint2 = fingerprintMapper.generateFingerprint("test-ds", state2); + + assertNotEquals( + fingerprint1, + fingerprint2, + "Dimensions order currently matters (arrays preserve order in JSON)" + ); + } + + @Test + public void testGenerateIndexingStateFingerprint_differentPartitionsSpec() + { + CompactionState state1 = new CompactionState( + new DynamicPartitionsSpec(5000000, null), + DimensionsSpec.EMPTY, + Collections.singletonList(new CountAggregatorFactory("count")), + null, + IndexSpec.getDefault(), + null, + null + ); + + CompactionState state2 = new CompactionState( + new HashedPartitionsSpec(null, 2, Collections.singletonList("dim1")), + DimensionsSpec.EMPTY, + Collections.singletonList(new CountAggregatorFactory("count")), + null, + IndexSpec.getDefault(), + null, + null + ); + + String fingerprint1 = fingerprintMapper.generateFingerprint("test-ds", state1); + String fingerprint2 = fingerprintMapper.generateFingerprint("test-ds", state2); + + assertNotEquals( + fingerprint1, + fingerprint2, + "Different PartitionsSpec should produce different fingerprints" + ); + } + + private CompactionState createBasicIndexingState() + { + return new CompactionState( + new DynamicPartitionsSpec(5000000, null), + DimensionsSpec.EMPTY, + Collections.singletonList(new CountAggregatorFactory("count")), + null, + IndexSpec.getDefault(), + null, + null + ); + } + + private CompactionState createTestIndexingState() + { + return new CompactionState( + new DynamicPartitionsSpec(100, null), + null, + null, + null, + IndexSpec.getDefault(), + null, + null + ); + } +} diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java index 7a68424db9f8..56ec8525bb83 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java @@ -143,7 +143,7 @@ public void testSimulate_withFixedIntervalOrderPolicy() final CompactionSimulateResult simulateResult = simulator.simulateRunWithConfig( DruidCompactionConfig .empty() - .withClusterConfig(new ClusterCompactionConfig(null, null, policy, null, null)) + .withClusterConfig(new ClusterCompactionConfig(null, null, policy, null, null, null)) .withDatasourceConfig( InlineSchemaDataSourceCompactionConfig.builder().forDataSource(dataSource).build() ), diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java index 7312e8635e11..2b274e805d4c 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java @@ -30,13 +30,20 @@ import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; import org.apache.druid.indexer.partitions.HashedPartitionsSpec; import org.apache.druid.indexer.partitions.PartitionsSpec; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.AutoTypeColumnSchema; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.data.CompressionStrategy; +import org.apache.druid.segment.metadata.DefaultIndexingStateFingerprintMapper; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; +import org.apache.druid.segment.metadata.IndexingStateCache; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; @@ -47,6 +54,7 @@ import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.util.Collections; @@ -58,6 +66,33 @@ public class CompactionStatusTest = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 0)) .size(100_000_000L) .build(); + private static final DataSegment WIKI_SEGMENT_2 + = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 1)) + .size(100_000_000L) + .build(); + + private HeapMemoryIndexingStateStorage indexingStateStorage; + private IndexingStateCache indexingStateCache; + private IndexingStateFingerprintMapper fingerprintMapper; + + @Before + public void setUp() + { + indexingStateStorage = new HeapMemoryIndexingStateStorage(); + indexingStateCache = new IndexingStateCache(); + fingerprintMapper = new DefaultIndexingStateFingerprintMapper( + indexingStateCache, + new DefaultObjectMapper() + ); + } + + /** + * Helper to sync the cache with states stored in the manager (for tests that persist states). + */ + private void syncCacheFromManager() + { + indexingStateCache.resetIndexingStatesForPublishedSegments(indexingStateStorage.getAllStoredStates()); + } @Test public void testFindPartitionsSpecWhenGivenIsNull() @@ -328,7 +363,8 @@ public void testStatusWhenLastCompactionStateSameAsRequired() final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.from(List.of(segment), Granularities.HOUR), - compactionConfig + compactionConfig, + fingerprintMapper ); Assert.assertTrue(status.isComplete()); } @@ -377,7 +413,8 @@ public void testStatusWhenProjectionsMatch() final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.from(List.of(segment), Granularities.HOUR), - compactionConfig + compactionConfig, + fingerprintMapper ); Assert.assertTrue(status.isComplete()); } @@ -431,7 +468,8 @@ public void testStatusWhenProjectionsMismatch() final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.from(List.of(segment), Granularities.HOUR), - compactionConfig + compactionConfig, + fingerprintMapper ); Assert.assertFalse(status.isComplete()); } @@ -484,7 +522,8 @@ public void testStatusWhenAutoSchemaMatch() final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.from(List.of(segment), null), - compactionConfig + compactionConfig, + fingerprintMapper ); Assert.assertTrue(status.isComplete()); } @@ -537,11 +576,257 @@ public void testStatusWhenAutoSchemaMismatch() final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.from(List.of(segment), null), - compactionConfig + compactionConfig, + fingerprintMapper ); Assert.assertFalse(status.isComplete()); } + @Test + public void test_evaluate_needsCompactionWhenAllSegmentsHaveUnexpectedIndexingStateFingerprint() + { + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() + ); + + final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + CompactionState wrongState = oldCompactionConfig.toCompactionState(); + + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); + syncCacheFromManager(); + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.from(segments, null), + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void test_evaluate_needsCompactionWhenSomeSegmentsHaveUnexpectedIndexingStateFingerprint() + { + final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + CompactionState wrongState = oldCompactionConfig.toCompactionState(); + + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() + ); + + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); + syncCacheFromManager(); + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.from(segments, null), + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void test_evaluate_noCompacationIfUnexpectedFingerprintHasExpectedIndexingState() + { + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", expectedState, DateTimes.nowUtc()); + syncCacheFromManager(); + + final CompactionStatus status = CompactionStatus.compute( + CompactionCandidate.from(segments, null), + compactionConfig, + fingerprintMapper + ); + Assert.assertTrue(status.isComplete()); + } + + @Test + public void test_evaluate_needsCompactionWhenUnexpectedFingerprintAndNoFingerprintInMetadataStore() + { + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.from(segments, null), + compactionConfig, + "One or more fingerprinted segments do not have a cached indexing state" + ); + } + + @Test + public void test_evaluate_noCompactionWhenAllSegmentsHaveExpectedIndexingStateFingerprint() + { + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(expectedFingerprint).build() + ); + + final CompactionStatus status = CompactionStatus.compute( + CompactionCandidate.from(segments, null), + compactionConfig, + fingerprintMapper + ); + Assert.assertTrue(status.isComplete()); + } + + @Test + public void test_evaluate_needsCompactionWhenNonFingerprintedSegmentsFailChecksOnLastCompactionState() + { + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); + syncCacheFromManager(); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.HOUR)).build() + ); + + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.from(segments, null), + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void test_evaluate_noCompactionWhenNonFingerprintedSegmentsPassChecksOnLastCompactionState() + { + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.DAY)).build() + ); + + final CompactionStatus status = CompactionStatus.compute( + CompactionCandidate.from(segments, null), + compactionConfig, + fingerprintMapper + ); + Assert.assertTrue(status.isComplete()); + } + + // ============================ + // SKIPPED status tests + // ============================ + + @Test + public void test_evaluate_isSkippedWhenInputBytesExceedLimit() + { + // Two segments with 100MB each = 200MB total + // inputSegmentSizeBytes is 150MB, so should be skipped + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withInputSegmentSizeBytes(150_000_000L) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + final CompactionState lastCompactionState = createCompactionStateWithGranularity(Granularities.HOUR); + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(), + DataSegment.builder(WIKI_SEGMENT_2).lastCompactionState(lastCompactionState).build() + ); + + final CompactionStatus status = CompactionStatus.compute( + CompactionCandidate.from(segments, null), + compactionConfig, + fingerprintMapper + ); + + Assert.assertFalse(status.isComplete()); + Assert.assertTrue(status.isSkipped()); + Assert.assertTrue(status.getReason().contains("'inputSegmentSize' exceeded")); + Assert.assertTrue(status.getReason().contains("200000000")); + Assert.assertTrue(status.getReason().contains("150000000")); + } + + /** + * Verify that the evaluation indicates compaction is needed for the expected reason. + * Allows customization of the segments in the compaction candidate. + */ + private void verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate candidate, + DataSourceCompactionConfig compactionConfig, + String expectedReason + ) + { + final CompactionStatus status = CompactionStatus.compute( + candidate, + compactionConfig, + fingerprintMapper + ); + + Assert.assertFalse(status.isComplete()); + Assert.assertEquals(expectedReason, status.getReason()); + } + private void verifyCompactionStatusIsPendingBecause( CompactionState lastCompactionState, DataSourceCompactionConfig compactionConfig, @@ -554,7 +839,8 @@ private void verifyCompactionStatusIsPendingBecause( .build(); final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.from(List.of(segment), null), - compactionConfig + compactionConfig, + fingerprintMapper ); Assert.assertFalse(status.isComplete()); @@ -582,4 +868,20 @@ private static UserCompactionTaskQueryTuningConfig createTuningConfig( null, null, null, null, null, null, null, null, null, null ); } + + /** + * Simple helper to create a CompactionState with only segmentGranularity set + */ + private static CompactionState createCompactionStateWithGranularity(Granularity segmentGranularity) + { + return new CompactionState( + null, + null, + null, + null, + IndexSpec.getDefault(), + new UniformGranularitySpec(segmentGranularity, null, null, null), + null + ); + } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java index 1c92c9a249c9..4e86abcd2468 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java @@ -47,6 +47,9 @@ import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.data.ConciseBitmapSerdeFactory; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; +import org.apache.druid.segment.metadata.DefaultIndexingStateFingerprintMapper; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; +import org.apache.druid.segment.metadata.NoopIndexingStateCache; import org.apache.druid.segment.transform.CompactionTransformSpec; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; @@ -81,6 +84,7 @@ public class NewestSegmentFirstPolicyTest private static final int DEFAULT_NUM_SEGMENTS_PER_SHARD = 4; private final ObjectMapper mapper = new DefaultObjectMapper(); private final NewestSegmentFirstPolicy policy = new NewestSegmentFirstPolicy(null); + private final IndexingStateFingerprintMapper fingerprintMapper = new DefaultIndexingStateFingerprintMapper(new NoopIndexingStateCache(), mapper); @Test public void testLargeOffsetAndSmallSegmentInterval() @@ -276,7 +280,8 @@ public void testSkipDataSourceWithNoSegments() .withNumPartitions(4) ) ), - Collections.emptyMap() + Collections.emptyMap(), + fingerprintMapper ); assertCompactSegmentIntervals( @@ -508,7 +513,8 @@ public void testWithSkipIntervals() Intervals.of("2017-11-15T00:00:00/2017-11-15T20:00:00"), Intervals.of("2017-11-13T00:00:00/2017-11-14T01:00:00") ) - ) + ), + fingerprintMapper ); assertCompactSegmentIntervals( @@ -547,7 +553,8 @@ public void testHoleInSearchInterval() Intervals.of("2017-11-16T04:00:00/2017-11-16T10:00:00"), Intervals.of("2017-11-16T14:00:00/2017-11-16T20:00:00") ) - ) + ), + fingerprintMapper ); assertCompactSegmentIntervals( @@ -1402,7 +1409,7 @@ public void testIteratorDoesNotReturnsSegmentsWhenPartitionDimensionsPrefixed() } @Test - public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentFilter() throws Exception + public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentFilter() { // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); @@ -2052,7 +2059,8 @@ TestDataSource.KOALA, configBuilder().forDataSource(TestDataSource.KOALA).build( TestDataSource.WIKI, SegmentTimeline.forSegments(wikiSegments), TestDataSource.KOALA, SegmentTimeline.forSegments(koalaSegments) ), - Collections.emptyMap() + Collections.emptyMap(), + fingerprintMapper ); // Verify that the segments of WIKI are preferred even though they are older @@ -2073,7 +2081,8 @@ private CompactionSegmentIterator createIterator(DataSourceCompactionConfig conf policy, Collections.singletonMap(TestDataSource.WIKI, config), Collections.singletonMap(TestDataSource.WIKI, timeline), - Collections.emptyMap() + Collections.emptyMap(), + fingerprintMapper ); } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CreateDataSegments.java b/server/src/test/java/org/apache/druid/server/coordinator/CreateDataSegments.java index 5772b9fb0478..1546eb76cdfb 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CreateDataSegments.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CreateDataSegments.java @@ -63,6 +63,7 @@ public class CreateDataSegments private String upgradedFromSegmentId; private String schemaFingerprint; private Integer numRows; + private String indexingStateFingerprint; public static CreateDataSegments ofDatasource(String datasource) { @@ -127,6 +128,12 @@ public CreateDataSegments withSchemaFingerprint(String schemaFingerprint) return this; } + public CreateDataSegments withIndexingStateFingerprint(String indexingStateFingerprint) + { + this.indexingStateFingerprint = indexingStateFingerprint; + return this; + } + public CreateDataSegments markUnused() { this.used = false; @@ -188,7 +195,8 @@ public List eachOfSize(long sizeInBytes) ++uniqueIdInInterval, compactionState, sizeInBytes, - numRows + numRows, + indexingStateFingerprint ) ); } @@ -207,7 +215,8 @@ private DataSegmentPlus plus(DataSegment segment) used, schemaFingerprint, numRows == null ? null : numRows.longValue(), - upgradedFromSegmentId + upgradedFromSegmentId, + indexingStateFingerprint ); } @@ -227,7 +236,8 @@ private NumberedDataSegment( int uniqueId, CompactionState compactionState, long size, - Integer numRows + Integer numRows, + String indexingStateFingerprint ) { super( @@ -243,6 +253,7 @@ private NumberedDataSegment( IndexIO.CURRENT_VERSION_ID, size, numRows, + indexingStateFingerprint, PruneSpecsHolder.DEFAULT ); this.uniqueId = uniqueId; diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigAuditEntryTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigAuditEntryTest.java index 65ea53586d3f..d32f64a49aef 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigAuditEntryTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigAuditEntryTest.java @@ -33,7 +33,7 @@ public class DataSourceCompactionConfigAuditEntryTest private final AuditInfo auditInfo = new AuditInfo("author", "identity", "comment", "ip"); private final DataSourceCompactionConfigAuditEntry firstEntry = new DataSourceCompactionConfigAuditEntry( - new ClusterCompactionConfig(0.1, 9, null, null, null), + new ClusterCompactionConfig(0.1, 9, null, null, null, null), InlineSchemaDataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(), auditInfo, DateTimes.nowUtc() @@ -43,7 +43,7 @@ public class DataSourceCompactionConfigAuditEntryTest public void testhasSameConfigWithSameBaseConfigIsTrue() { final DataSourceCompactionConfigAuditEntry secondEntry = new DataSourceCompactionConfigAuditEntry( - new ClusterCompactionConfig(0.1, 9, null, null, null), + new ClusterCompactionConfig(0.1, 9, null, null, null, null), InlineSchemaDataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(), auditInfo, DateTimes.nowUtc() @@ -56,7 +56,7 @@ public void testhasSameConfigWithSameBaseConfigIsTrue() public void testhasSameConfigWithDifferentClusterConfigIsFalse() { DataSourceCompactionConfigAuditEntry secondEntry = new DataSourceCompactionConfigAuditEntry( - new ClusterCompactionConfig(0.2, 9, null, null, null), + new ClusterCompactionConfig(0.2, 9, null, null, null, null), InlineSchemaDataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(), auditInfo, DateTimes.nowUtc() @@ -65,7 +65,7 @@ public void testhasSameConfigWithDifferentClusterConfigIsFalse() Assert.assertFalse(secondEntry.hasSameConfig(firstEntry)); secondEntry = new DataSourceCompactionConfigAuditEntry( - new ClusterCompactionConfig(0.1, 10, null, null, null), + new ClusterCompactionConfig(0.1, 10, null, null, null, null), InlineSchemaDataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(), auditInfo, DateTimes.nowUtc() @@ -78,7 +78,7 @@ public void testhasSameConfigWithDifferentClusterConfigIsFalse() public void testhasSameConfigWithDifferentDatasourceConfigIsFalse() { DataSourceCompactionConfigAuditEntry secondEntry = new DataSourceCompactionConfigAuditEntry( - new ClusterCompactionConfig(0.1, 9, null, null, null), + new ClusterCompactionConfig(0.1, 9, null, null, null, null), InlineSchemaDataSourceCompactionConfig.builder().forDataSource(TestDataSource.KOALA).build(), auditInfo, DateTimes.nowUtc() @@ -91,7 +91,7 @@ public void testhasSameConfigWithDifferentDatasourceConfigIsFalse() public void testhasSameConfigWithNullDatasourceConfigIsFalse() { final DataSourceCompactionConfigAuditEntry secondEntry = new DataSourceCompactionConfigAuditEntry( - new ClusterCompactionConfig(0.1, 9, null, null, null), + new ClusterCompactionConfig(0.1, 9, null, null, null, null), null, auditInfo, DateTimes.nowUtc() diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigHistoryTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigHistoryTest.java index 7208bdf0bce5..35cb1177af59 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigHistoryTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigHistoryTest.java @@ -177,7 +177,7 @@ public void testAddAndModifyClusterConfigShouldAddTwice() wikiAuditHistory.add(originalConfig, auditInfo, DateTimes.nowUtc()); final DruidCompactionConfig updatedConfig = originalConfig.withClusterConfig( - new ClusterCompactionConfig(0.2, null, null, null, null) + new ClusterCompactionConfig(0.2, null, null, null, null, null) ); wikiAuditHistory.add(updatedConfig, auditInfo, DateTimes.nowUtc()); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DruidCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DruidCompactionConfigTest.java index c64a254ae6ee..8fae5623db41 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/DruidCompactionConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/DruidCompactionConfigTest.java @@ -65,6 +65,7 @@ public void testSerdeWithDatasourceConfigs() throws Exception null, null, null, + null, null ); @@ -83,7 +84,8 @@ public void testCopyWithClusterConfig() 10, new NewestSegmentFirstPolicy(null), true, - CompactionEngine.MSQ + CompactionEngine.MSQ, + true ); final DruidCompactionConfig copy = config.withClusterConfig(clusterConfig); @@ -118,5 +120,6 @@ public void testDefaultConfigValues() Assert.assertEquals(CompactionEngine.NATIVE, config.getEngine()); Assert.assertEquals(0.1, config.getCompactionTaskSlotRatio(), 1e-9); Assert.assertEquals(Integer.MAX_VALUE, config.getMaxCompactionTaskSlots()); + Assert.assertTrue(config.isStoreCompactionStatePerSegment()); } } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java index 39213ff8616d..f47cb9fd9f8e 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java @@ -829,7 +829,7 @@ public void testSimulateRunWithEmptyDatasourceCompactionConfigs() .anyTimes(); EasyMock.replay(segmentsMetadataManager); CompactionSimulateResult result = coordinator.simulateRunWithConfigUpdate( - new ClusterCompactionConfig(0.2, null, null, null, null) + new ClusterCompactionConfig(0.2, null, null, null, null, null) ); Assert.assertEquals(Collections.emptyMap(), result.getCompactionStates()); } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index 301dd77493c2..cd92e8f1999a 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -1657,6 +1657,7 @@ private CoordinatorRunStats doCompactSegments( numCompactionTaskSlots, policy, null, + null, null ) ) diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java index f8ea67308385..37501cb849cc 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java @@ -44,6 +44,7 @@ import org.apache.druid.rpc.indexing.NoopOverlordClient; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.server.coordinator.CoordinatorDynamicConfig; import org.apache.druid.server.coordinator.DruidCoordinatorRuntimeParams; import org.apache.druid.server.coordinator.config.KillUnusedSegmentsConfig; @@ -117,7 +118,8 @@ public void setup() derbyConnectorRule.metadataTablesConfigSupplier().get(), connector, null, - CentralizedDatasourceSchemaConfig.create() + CentralizedDatasourceSchemaConfig.create(), + new HeapMemoryIndexingStateStorage() ); this.config = derbyConnectorRule.metadataTablesConfigSupplier().get(); diff --git a/server/src/test/java/org/apache/druid/server/http/DataSegmentPlusTest.java b/server/src/test/java/org/apache/druid/server/http/DataSegmentPlusTest.java index 1aa0bc145a72..c86a64bb077b 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSegmentPlusTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSegmentPlusTest.java @@ -77,6 +77,7 @@ public void testSerde() throws JsonProcessingException final Interval interval = Intervals.of("2011-10-01/2011-10-02"); final ImmutableMap loadSpec = ImmutableMap.of("something", "or_other"); + String indexingStateFingerprint = "abc123"; String createdDateStr = "2024-01-20T00:00:00.701Z"; String usedStatusLastUpdatedDateStr = "2024-01-20T01:00:00.701Z"; DateTime createdDate = DateTimes.of(createdDateStr); @@ -99,6 +100,7 @@ public void testSerde() throws JsonProcessingException MAPPER.convertValue(ImmutableMap.of(), GranularitySpec.class), null )) + .indexingStateFingerprint(indexingStateFingerprint) .binaryVersion(TEST_VERSION) .size(123L) .totalRows(12) @@ -108,7 +110,8 @@ public void testSerde() throws JsonProcessingException null, null, null, - null + null, + indexingStateFingerprint ); final Map objectMap = MAPPER.readValue( @@ -116,14 +119,14 @@ public void testSerde() throws JsonProcessingException JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - Assert.assertEquals(7, objectMap.size()); + Assert.assertEquals(8, objectMap.size()); final Map segmentObjectMap = MAPPER.readValue( MAPPER.writeValueAsString(segmentPlus.getDataSegment()), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); // verify dataSegment - Assert.assertEquals(13, segmentObjectMap.size()); + Assert.assertEquals(14, segmentObjectMap.size()); Assert.assertEquals("something", segmentObjectMap.get("dataSource")); Assert.assertEquals(interval.toString(), segmentObjectMap.get("interval")); Assert.assertEquals("1", segmentObjectMap.get("version")); @@ -139,6 +142,7 @@ public void testSerde() throws JsonProcessingException Assert.assertEquals(123, segmentObjectMap.get("size")); Assert.assertEquals(12, segmentObjectMap.get("totalRows")); Assert.assertEquals(6, ((Map) segmentObjectMap.get("lastCompactionState")).size()); + Assert.assertEquals("abc123", segmentObjectMap.get("indexingStateFingerprint")); // verify extra metadata Assert.assertEquals(createdDateStr, objectMap.get("createdDate")); diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index 4458fd44f6c8..16e315fc05d1 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -78,7 +78,7 @@ public class MetadataResourceTest .toArray(new DataSegment[0]); private final List segmentsPlus = Arrays.stream(segments) - .map(s -> new DataSegmentPlus(s, DateTimes.nowUtc(), DateTimes.nowUtc(), null, null, null, null)) + .map(s -> new DataSegmentPlus(s, DateTimes.nowUtc(), DateTimes.nowUtc(), null, null, null, null, null)) .collect(Collectors.toList()); private HttpServletRequest request; private SegmentsMetadataManager segmentsMetadataManager; diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java b/services/src/main/java/org/apache/druid/cli/CliOverlord.java index b23878453efd..6518766817ba 100644 --- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java +++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java @@ -90,8 +90,11 @@ import org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig; import org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningStrategy; import org.apache.druid.indexing.overlord.config.DefaultTaskConfig; +import org.apache.druid.indexing.overlord.config.IndexingStateCleanupConfig; +import org.apache.druid.indexing.overlord.config.OverlordKillConfigs; import org.apache.druid.indexing.overlord.config.TaskLockConfig; import org.apache.druid.indexing.overlord.config.TaskQueueConfig; +import org.apache.druid.indexing.overlord.duty.KillUnreferencedIndexingState; import org.apache.druid.indexing.overlord.duty.OverlordDuty; import org.apache.druid.indexing.overlord.duty.TaskLogAutoCleaner; import org.apache.druid.indexing.overlord.duty.TaskLogAutoCleanerConfig; @@ -409,6 +412,13 @@ public TaskStorageDirTracker getTaskStorageDirTracker(WorkerConfig workerConfig, return TaskStorageDirTracker.fromConfigs(workerConfig, taskConfig); } + @Provides + @LazySingleton + public IndexingStateCleanupConfig provideIndexingStateCleanupConfig(OverlordKillConfigs killConfigs) + { + return killConfigs.indexingStates(); + } + @Provides @LazySingleton @Named(ServiceStatusMonitor.HEARTBEAT_TAGS_BINDING) @@ -451,9 +461,11 @@ private void configureAutoscale(Binder binder) private void configureOverlordHelpers(Binder binder) { JsonConfigProvider.bind(binder, "druid.indexer.logs.kill", TaskLogAutoCleanerConfig.class); + JsonConfigProvider.bind(binder, "druid.overlord.kill", OverlordKillConfigs.class); final Multibinder dutyBinder = Multibinder.newSetBinder(binder, OverlordDuty.class); dutyBinder.addBinding().to(TaskLogAutoCleaner.class); dutyBinder.addBinding().to(UnusedSegmentsKiller.class).in(LazySingleton.class); + dutyBinder.addBinding().to(KillUnreferencedIndexingState.class); } /** diff --git a/services/src/main/java/org/apache/druid/guice/MetadataManagerModule.java b/services/src/main/java/org/apache/druid/guice/MetadataManagerModule.java index 6d501f9f768c..46fc9c835ac6 100644 --- a/services/src/main/java/org/apache/druid/guice/MetadataManagerModule.java +++ b/services/src/main/java/org/apache/druid/guice/MetadataManagerModule.java @@ -41,8 +41,11 @@ import org.apache.druid.metadata.segment.SqlSegmentMetadataTransactionFactory; import org.apache.druid.metadata.segment.cache.HeapMemorySegmentMetadataCache; import org.apache.druid.metadata.segment.cache.SegmentMetadataCache; +import org.apache.druid.segment.metadata.IndexingStateCache; +import org.apache.druid.segment.metadata.IndexingStateStorage; import org.apache.druid.segment.metadata.NoopSegmentSchemaCache; import org.apache.druid.segment.metadata.SegmentSchemaCache; +import org.apache.druid.segment.metadata.SqlIndexingStateStorage; import org.apache.druid.server.coordinator.CoordinatorConfigManager; import org.apache.druid.server.coordinator.MetadataManager; @@ -59,7 +62,9 @@ *

  • {@link IndexerMetadataStorageCoordinator}
    • *
  • {@link CoordinatorConfigManager}
    • *
  • {@link SegmentMetadataCache}
    • + *
  • {@link IndexingStateCache} - Overlord only
    • *
  • {@link SegmentSchemaCache} - Coordinator only
    • + *
  • {@link SqlIndexingStateStorage}
    • * */ public class MetadataManagerModule implements Module @@ -101,6 +106,9 @@ public void configure(Binder binder) binder.bind(SegmentMetadataCache.class) .to(HeapMemorySegmentMetadataCache.class) .in(LazySingleton.class); + binder.bind(IndexingStateStorage.class) + .to(SqlIndexingStateStorage.class) + .in(ManageLifecycle.class); // Coordinator-only dependencies if (nodeRoles.contains(NodeRole.COORDINATOR)) { @@ -128,6 +136,7 @@ public void configure(Binder binder) binder.bind(SegmentMetadataTransactionFactory.class) .to(SqlSegmentMetadataTransactionFactory.class) .in(LazySingleton.class); + binder.bind(IndexingStateCache.class).in(LazySingleton.class); } else { binder.bind(SegmentMetadataTransactionFactory.class) .to(SqlSegmentMetadataReadOnlyTransactionFactory.class) diff --git a/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-completions.ts b/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-completions.ts index 4dd7f7fc2650..1fa4b7b24160 100644 --- a/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-completions.ts +++ b/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-completions.ts @@ -44,6 +44,11 @@ export const COMPACTION_DYNAMIC_CONFIG_COMPLETIONS: JsonCompletionRule[] = [ value: 'engine', documentation: 'Engine used for running compaction tasks (native or msq)', }, + { + value: 'storeCompactionStatePerSegment', + documentation: + 'Whether to persist the full compaction state in segment metadata (default: true)', + }, ], }, // compactionTaskSlotRatio values @@ -116,4 +121,18 @@ export const COMPACTION_DYNAMIC_CONFIG_COMPLETIONS: JsonCompletionRule[] = [ condition: obj => !obj.useSupervisors, completions: [{ value: 'native', documentation: 'Native indexing engine (default)' }], }, + { + path: '$.storeCompactionStatePerSegment', + completions: [ + { + value: 'true', + documentation: 'Store full compaction state in segment metadata (legacy behavior, default)', + }, + { + value: 'false', + documentation: + 'Store only fingerprint reference in segment metadata (reduces storage overhead)', + }, + ], + }, ]; diff --git a/web-console/src/druid-models/compaction-dynamic-config/compaction-dynamic-config.tsx b/web-console/src/druid-models/compaction-dynamic-config/compaction-dynamic-config.tsx index 047f3f84f738..7d3c766c8aad 100644 --- a/web-console/src/druid-models/compaction-dynamic-config/compaction-dynamic-config.tsx +++ b/web-console/src/druid-models/compaction-dynamic-config/compaction-dynamic-config.tsx @@ -27,6 +27,7 @@ export interface CompactionDynamicConfig { compactionPolicy: { type: 'newestSegmentFirst'; priorityDatasource?: string | null }; useSupervisors: boolean; engine: 'native' | 'msq'; + storeCompactionStatePerSegment: boolean; } export const COMPACTION_DYNAMIC_CONFIG_DEFAULT_RATIO = 0.1; @@ -94,4 +95,29 @@ export const COMPACTION_DYNAMIC_CONFIG_FIELDS: Field[] ), }, + { + name: 'storeCompactionStatePerSegment', + label: 'Legacy: Persist last compaction state in segments', + type: 'boolean', + defaultValue: true, + info: ( + <> +

    + Whether to persist the full compaction state in segment metadata. When true{' '} + (default), compaction state is stored in both the segment metadata and the compaction + states table. +

    +

    + When false, only a fingerprint reference is stored in the segment metadata, + reducing storage overhead in the segments table. The actual compaction state is stored in + the compaction states table. +

    +

    + Note: Eventually this configuration will be removed and all compaction + will use the fingerprint method only. This configuration exists for operators to opt into + this future pattern early. +

    + + ), + }, ];