From 868cf2f2f4b9e80c7b4a65de272886f74a02405a Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 5 Dec 2025 14:35:10 -0800 Subject: [PATCH 01/17] nested-it --- ...CombiningInputSourceParallelIndexTest.java | 2 +- .../embedded/indexer/ITTransformTest.java | 2 +- .../testing/embedded/indexer/IndexerTest.java | 10 +- .../indexing/IndexParallelTaskTest.java | 4 +- .../indexing/KafkaClusterMetricsTest.java | 3 +- .../indexing/NestedDataFormatsTest.java | 134 ++++++++++++++++++ .../embedded/msq/BaseRealtimeQueryTest.java | 2 +- .../EmbeddedDurableShuffleStorageTest.java | 2 +- .../embedded/msq/ITMSQReindexTest.java | 2 +- .../query/ForcePushDownNestedQueryTest.java | 2 +- .../embedded/query/QueryErrorTest.java | 2 +- .../QueryRetryOnMissingSegmentsTest.java | 2 +- .../query/QueryVirtualStorageTest.java | 7 - .../embedded/query/UnionQueryTest.java | 2 +- .../indexing/common/task/TaskBuilder.java | 12 +- pom.xml | 2 +- .../druid/query/http/ClientSqlQuery.java | 5 + .../druid/query/http/ClientSqlQueryTest.java | 10 ++ .../metrics/LatchableEmitterConfig.java | 2 +- .../testing/embedded/EmbeddedClusterApis.java | 6 +- .../testing/embedded/indexing/Resources.java | 17 +++ .../junit5/EmbeddedClusterTestBase.java | 8 +- 22 files changed, 205 insertions(+), 33 deletions(-) create mode 100644 embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITCombiningInputSourceParallelIndexTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITCombiningInputSourceParallelIndexTest.java index d3ae98af4c80..afb5cfc0b7b0 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITCombiningInputSourceParallelIndexTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITCombiningInputSourceParallelIndexTest.java @@ -51,7 +51,7 @@ protected void addResources(EmbeddedDruidCluster cluster) public void testIndexData() throws Exception { final String indexDatasource = dataSource; - final String combiningDatasource = EmbeddedClusterApis.createTestDatasourceName(); + final String combiningDatasource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); Map inputFormatMap = new ImmutableMap .Builder() .put("type", "json") diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITTransformTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITTransformTest.java index 22fe4df46e03..2c36b21f72b5 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITTransformTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITTransformTest.java @@ -38,7 +38,7 @@ public class ITTransformTest extends AbstractITBatchIndexTest public void testIndexAndReIndexWithTransformSpec() throws IOException { final String indexDatasource = dataSource; - final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(); + final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); try ( final Closeable ignored1 = unloader(indexDatasource); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/IndexerTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/IndexerTest.java index 0b7a8d7faf6d..7e029ca83e5e 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/IndexerTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/IndexerTest.java @@ -79,8 +79,8 @@ protected void addResources(EmbeddedDruidCluster cluster) public void testIndexData() throws Exception { final String indexDatasource = dataSource; - final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(); - final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(); + final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); try ( final Closeable ignored1 = unloader(indexDatasource); final Closeable ignored2 = unloader(reindexDatasource); @@ -131,8 +131,8 @@ public void testIndexData() throws Exception public void testReIndexDataWithTimestamp() throws Exception { final String indexDatasource = dataSource; - final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(); - final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(); + final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); try ( final Closeable ignored1 = unloader(indexDatasource); final Closeable ignored2 = unloader(reindexDatasource); @@ -197,7 +197,7 @@ public void testReIndexWithNonExistingDatasource() throws Exception public void testMERGEIndexData() throws Exception { final String reindexDatasource = dataSource; - final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(); + final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); try ( final Closeable ignored2 = unloader(reindexDatasource); final Closeable ignored3 = unloader(reindexDatasourceWithDruidInputSource) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java index 23645dea5942..bfdc351871cb 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java @@ -159,7 +159,7 @@ public void test_runIndexTask_andReindexIntoAnotherDatasource(PartitionsSpec par runQueries(dataSource); // Re-index into a different datasource, indexing 1 segment per sub-task - final String dataSource2 = EmbeddedClusterApis.createTestDatasourceName(); + final String dataSource2 = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); final TaskBuilder.IndexParallel reindexTaskSplitBySegment = TaskBuilder.ofTypeIndexParallel() .dataSource(dataSource2) @@ -185,7 +185,7 @@ public void test_runIndexTask_andReindexIntoAnotherDatasource(PartitionsSpec par runQueries(dataSource2); // Re-index into a different datasource, indexing 1 file per sub-task - final String dataSource3 = EmbeddedClusterApis.createTestDatasourceName(); + final String dataSource3 = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); final TaskBuilder.IndexParallel reindexTaskSplitByFile = TaskBuilder.ofTypeIndexParallel() .dataSource(dataSource3) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java index 8200bb335566..1fcbb4837655 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java @@ -31,6 +31,7 @@ import org.apache.druid.query.DruidMetrics; import org.apache.druid.rpc.UpdateResponse; import org.apache.druid.rpc.indexing.OverlordClient; +import org.apache.druid.segment.TestDataSource; import org.apache.druid.server.DruidNode; import org.apache.druid.server.coordinator.ClusterCompactionConfig; import org.apache.druid.server.coordinator.CoordinatorDynamicConfig; @@ -62,7 +63,7 @@ @SuppressWarnings("resource") public class KafkaClusterMetricsTest extends EmbeddedClusterTestBase { - private static final String TOPIC = EmbeddedClusterApis.createTestDatasourceName(); + private static final String TOPIC = EmbeddedClusterApis.createTestDatasourceName(TestDataSource.WIKI); private final EmbeddedBroker broker = new EmbeddedBroker(); private final EmbeddedIndexer indexer = new EmbeddedIndexer(); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java new file mode 100644 index 000000000000..72f9f13a9642 --- /dev/null +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -0,0 +1,134 @@ +package org.apache.druid.testing.embedded.indexing; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.druid.indexing.common.task.TaskBuilder; +import org.apache.druid.java.util.common.jackson.JacksonUtils; +import org.apache.druid.query.http.ClientSqlQuery; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.TestDataSource; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec; +import org.apache.druid.segment.nested.ObjectStorageEncoding; +import org.apache.druid.testing.embedded.EmbeddedBroker; +import org.apache.druid.testing.embedded.EmbeddedClusterApis; +import org.apache.druid.testing.embedded.EmbeddedCoordinator; +import org.apache.druid.testing.embedded.EmbeddedDruidCluster; +import org.apache.druid.testing.embedded.EmbeddedHistorical; +import org.apache.druid.testing.embedded.EmbeddedIndexer; +import org.apache.druid.testing.embedded.EmbeddedOverlord; +import org.apache.druid.testing.embedded.EmbeddedRouter; +import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +/** + * Embedded tests for nested data, ingested in different {@link NestedCommonFormatColumnFormatSpec}. + */ +public class NestedDataFormatsTest extends EmbeddedClusterTestBase +{ + private final EmbeddedBroker broker = new EmbeddedBroker(); + private final EmbeddedOverlord overlord = new EmbeddedOverlord(); + private final EmbeddedCoordinator coordinator = new EmbeddedCoordinator(); + + private final String defaultFormat = "koala_default"; + + @Override + protected String getDatasourcePrefix() + { + return TestDataSource.KOALA; + } + + @Override + protected EmbeddedDruidCluster createCluster() + { + return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper() + .useLatchableEmitter() + .addServer(overlord) + .addServer(coordinator) + .addServer(new EmbeddedIndexer()) + .addServer(new EmbeddedHistorical()) + .addServer(broker) + .addServer(new EmbeddedRouter()); + } + + @Override + @BeforeAll + protected void setup() throws Exception + { + super.setup(); + super.refreshDatasourceName(); + final TaskBuilder.IndexParallel indexTask = + TaskBuilder.ofTypeIndexParallel() + .dataSource(defaultFormat) + .timestampColumn("timestamp") + .jsonInputFormat() + .inputSource(Resources.HttpData.kttm1Day()) + .schemaDiscovery(); + + final String taskId = EmbeddedClusterApis.newTaskId(defaultFormat); + cluster.callApi().runTask(indexTask.withId(taskId), overlord); + cluster.callApi().waitForAllSegmentsToBeAvailable(defaultFormat, coordinator, broker); + } + + @Test + public void test_objectStorageEncoding() + { + // Ingest kttm data with skipping smile raw json format, comparing diff with defaultFormat + NestedCommonFormatColumnFormatSpec spec = + NestedCommonFormatColumnFormatSpec.builder().setObjectStorageEncoding(ObjectStorageEncoding.NONE).build(); + final TaskBuilder.IndexParallel indexTask = + TaskBuilder.ofTypeIndexParallel() + .dataSource(dataSource) + .timestampColumn("timestamp") + .jsonInputFormat() + .inputSource(Resources.HttpData.kttm1Day()) + .schemaDiscovery() + .tuningConfig(t -> t.withIndexSpec(IndexSpec.builder().withAutoColumnFormatSpec(spec).build())); + final String taskId = EmbeddedClusterApis.newTaskId(dataSource); + cluster.callApi().runTask(indexTask.withId(taskId), overlord); + cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator, broker); + + final String resultAsJson = + cluster.callApi().onAnyBroker(b -> b.submitSqlQuery(ClientSqlQuery.simple("select * from sys.segments"))); + List> result = JacksonUtils.readValue( + TestHelper.JSON_MAPPER, + resultAsJson.getBytes(StandardCharsets.UTF_8), + new TypeReference<>() + { + } + ); + Map defaultFormatResult = + result.stream().filter(map -> defaultFormat.equals(map.get("datasource"))).findFirst().get(); + Map noneObjectStorageFormatResult = + result.stream().filter(map -> dataSource.equals(map.get("datasource"))).findFirst().get(); + // Test ingesting with skipping raw json smile format works, same row count, with ~20% storage saving + Assertions.assertEquals(465_346, defaultFormatResult.get("num_rows")); + Assertions.assertEquals(53_000_804, defaultFormatResult.get("size")); + Assertions.assertEquals(465_346, noneObjectStorageFormatResult.get("num_rows")); + Assertions.assertEquals(41_938_750, noneObjectStorageFormatResult.get("size")); + + // Test querying on a nested field works + final String groupByQuery = "select json_value(event, '$.type') as event_type, count(*) as total from %s group by 1 order by 2 desc, 1 asc limit 10"; + final String queryResultDefaultFormat = cluster.callApi().runSql(groupByQuery, defaultFormat); + final String queryResultNoneObjectStorage = cluster.callApi().runSql(groupByQuery, dataSource); + Assertions.assertEquals(queryResultDefaultFormat, queryResultNoneObjectStorage); + + // Test reconstruct json column works, the ordering of the fields has changed, but all values are perserved. + final String scanQuery = "select event, to_json_string(agent) as agent from %s where json_value(event, '$.type') = 'PercentClear' and json_value(agent, '$.os') = 'Android' order by __time asc limit 1"; + final String scanQueryResultDefaultFormat = cluster.callApi().runSql(scanQuery, defaultFormat); + final String scanQueryResultNoneObjectStorage = cluster.callApi().runSql(scanQuery, dataSource); + Assertions.assertEquals( + """ + "{""type"":""PercentClear"",""percentage"":85}","{""type"":""Mobile Browser"",""category"":""Smartphone"",""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""os"":""Android"",""platform"":""Android""}" + """.trim(), scanQueryResultDefaultFormat); + Assertions.assertEquals( + """ + "{""percentage"":85,""type"":""PercentClear""}","{""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""category"":""Smartphone"",""os"":""Android"",""platform"":""Android"",""type"":""Mobile Browser""}" + """.trim(), scanQueryResultNoneObjectStorage); + } +} diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java index ebc3ba723aaa..e7ce8b22809e 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java @@ -74,7 +74,7 @@ protected EmbeddedDruidCluster createCluster() void setupCreateKafkaTopicAndDatasource() { // Create Kafka topic. - topic = EmbeddedClusterApis.createTestDatasourceName(); + topic = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); kafka.createTopicWithPartitions(topic, 2); super.refreshDatasourceName(); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java index 62f168d9ba7c..db89ced3c98b 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java @@ -107,7 +107,7 @@ public EmbeddedDruidCluster createCluster() final void setupData() throws IOException { msqApis = new EmbeddedMSQApis(cluster, overlord); - dataSource = EmbeddedClusterApis.createTestDatasourceName(); + dataSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); loadWikipediaTable(); } diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/ITMSQReindexTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/ITMSQReindexTest.java index 22936a6421e7..30e345547913 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/ITMSQReindexTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/ITMSQReindexTest.java @@ -52,7 +52,7 @@ public static Stream test_cases() public void testMSQDruidInputSource(String sqlFileName, String reIndexSqlFileName, String reIndexQueryFileName) { final String indexDatasource = dataSource; - final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(); + final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); Map context = ImmutableMap.of(MultiStageQueryContext.CTX_FINALIZE_AGGREGATIONS, false, MultiStageQueryContext.CTX_MAX_NUM_TASKS, 5, GroupByQueryConfig.CTX_KEY_ENABLE_MULTI_VALUE_UNNESTING, false); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/ForcePushDownNestedQueryTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/ForcePushDownNestedQueryTest.java index b921d1c75ac4..a138e1110a35 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/ForcePushDownNestedQueryTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/ForcePushDownNestedQueryTest.java @@ -66,7 +66,7 @@ public class ForcePushDownNestedQueryTest extends QueryTestBase @Override public void beforeAll() throws IOException { - dataSource = EmbeddedClusterApis.createTestDatasourceName(); + dataSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); loadWikipediaTable(); } diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryErrorTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryErrorTest.java index e3896700823b..335218a284e5 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryErrorTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryErrorTest.java @@ -81,7 +81,7 @@ protected EmbeddedDruidCluster createCluster() @Override protected void beforeAll() { - tableName = EmbeddedClusterApis.createTestDatasourceName(); + tableName = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); EmbeddedMSQApis msqApi = new EmbeddedMSQApis(cluster, overlord); SqlTaskStatus ingestionStatus = msqApi.submitTaskSql(StringUtils.format( "REPLACE INTO %s\n" diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryRetryOnMissingSegmentsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryRetryOnMissingSegmentsTest.java index 2e1bdb694654..9d642703d9de 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryRetryOnMissingSegmentsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryRetryOnMissingSegmentsTest.java @@ -97,7 +97,7 @@ protected EmbeddedDruidCluster createCluster() public void beforeAll() { jsonMapper = overlord.bindings().jsonMapper(); - tableName = EmbeddedClusterApis.createTestDatasourceName(); + tableName = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); final String taskId = IdUtils.getRandomId(); final IndexTask task = MoreResources.Task.BASIC_INDEX.get().dataSource(tableName).withId(taskId); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java index a5489041b80f..dc9174b8e1d0 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java @@ -19,7 +19,6 @@ package org.apache.druid.testing.embedded.query; -import org.apache.druid.common.utils.IdUtils; import org.apache.druid.data.input.impl.LocalInputSource; import org.apache.druid.indexer.TaskState; import org.apache.druid.java.util.common.HumanReadableBytes; @@ -119,7 +118,6 @@ public EmbeddedDruidCluster createCluster() void loadData() throws IOException { msqApis = new EmbeddedMSQApis(cluster, overlord); - dataSource = createTestDatasourceName(); loadWikiData(); } @@ -274,11 +272,6 @@ private long getMetricLatestValue(LatchableEmitter emitter, String metricName, i return emitter.getLatestMetricEventValue(metricName, 0).longValue(); } - private String createTestDatasourceName() - { - return "wiki-" + IdUtils.getRandomId(); - } - /** * Stolen from {@link EmbeddedDurableShuffleStorageTest#loadWikipediaTable()} but with hourly granularity and no * durable shuffle location diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java index 61be086aec1d..09eed296887e 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java @@ -95,7 +95,7 @@ public void test_ingestData_andVerifyNativeAndSQLQueries() final List datasourceNames = IntStream .range(0, numDatasources) - .mapToObj(i -> EmbeddedClusterApis.createTestDatasourceName()) + .mapToObj(i -> EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix())) .collect(Collectors.toList()); for (String datasourceName : datasourceNames) { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskBuilder.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskBuilder.java index 15f0d62ac45d..31a8d783603e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskBuilder.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskBuilder.java @@ -62,9 +62,9 @@ * explicitly. * * @param Type of this builder itself - * @param Type of tuning config used by this builder. - * @param Type of task created by this builder. - * @param Type of tuning config builder + * @param Type of tuning config used by this builder. + * @param Type of task created by this builder. + * @param Type of tuning config builder * @see #ofTypeIndex() * @see #tuningConfig(Consumer) to specify the {@code tuningConfig}. */ @@ -286,6 +286,12 @@ public Self dimensions(String... dimensions) return (Self) this; } + public Self schemaDiscovery() + { + dataSchema.withDimensions(DimensionsSpec.builder().useSchemaDiscovery(true).build()); + return (Self) this; + } + public Self metricAggregates(AggregatorFactory... aggregators) { dataSchema.withAggregators(aggregators); diff --git a/pom.xml b/pom.xml index 13ce80b8f2ab..a56cd5be249b 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ - 11 + 17 ${java.version} UTF-8 0.9.0.M2 diff --git a/processing/src/main/java/org/apache/druid/query/http/ClientSqlQuery.java b/processing/src/main/java/org/apache/druid/query/http/ClientSqlQuery.java index 83484551b5de..0af4fa386b46 100644 --- a/processing/src/main/java/org/apache/druid/query/http/ClientSqlQuery.java +++ b/processing/src/main/java/org/apache/druid/query/http/ClientSqlQuery.java @@ -77,6 +77,11 @@ public ClientSqlQuery( this.parameters = parameters; } + public static ClientSqlQuery simple(String query) + { + return new ClientSqlQuery(query, null, false, false, false, Map.of(), List.of()); + } + public String getQuery() { return query; diff --git a/processing/src/test/java/org/apache/druid/query/http/ClientSqlQueryTest.java b/processing/src/test/java/org/apache/druid/query/http/ClientSqlQueryTest.java index 10379b3146ac..3cf3e5b0545d 100644 --- a/processing/src/test/java/org/apache/druid/query/http/ClientSqlQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/http/ClientSqlQueryTest.java @@ -26,6 +26,9 @@ import org.junit.Assert; import org.junit.Test; +import java.util.List; +import java.util.Map; + public class ClientSqlQueryTest { @Test @@ -59,4 +62,11 @@ public void testSerdeWithParameters() throws JsonProcessingException ); Assert.assertEquals(query, jsonMapper.readValue(jsonMapper.writeValueAsString(query), ClientSqlQuery.class)); } + + @Test + public void testSimple() + { + final ClientSqlQuery query = ClientSqlQuery.simple("select 1"); + Assert.assertEquals(new ClientSqlQuery("select 1", null, false, false, false, Map.of(), List.of()), query); + } } diff --git a/server/src/test/java/org/apache/druid/server/metrics/LatchableEmitterConfig.java b/server/src/test/java/org/apache/druid/server/metrics/LatchableEmitterConfig.java index 9ab358119aaf..71e0a8d14691 100644 --- a/server/src/test/java/org/apache/druid/server/metrics/LatchableEmitterConfig.java +++ b/server/src/test/java/org/apache/druid/server/metrics/LatchableEmitterConfig.java @@ -35,7 +35,7 @@ public LatchableEmitterConfig( @JsonProperty("defaultWaitTimeoutMillis") @Nullable Long defaultWaitTimeoutMillis ) { - this.defaultWaitTimeoutMillis = Configs.valueOrDefault(defaultWaitTimeoutMillis, 10_000); + this.defaultWaitTimeoutMillis = Configs.valueOrDefault(defaultWaitTimeoutMillis, 50_000); } /** diff --git a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java index 11b538530bdd..02772a49554a 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java @@ -412,11 +412,11 @@ public SupervisorStatus getSupervisorStatus(String supervisorId) // STATIC UTILITY METHODS /** - * Creates a random datasource name prefixed with {@link TestDataSource#WIKI}. + * Creates a random datasource name with the given prefixed. */ - public static String createTestDatasourceName() + public static String createTestDatasourceName(String prefix) { - return TestDataSource.WIKI + "_" + IdUtils.getRandomId(); + return prefix + "_" + IdUtils.getRandomId(); } /** diff --git a/services/src/test/java/org/apache/druid/testing/embedded/indexing/Resources.java b/services/src/test/java/org/apache/druid/testing/embedded/indexing/Resources.java index 1e707aa88354..f5535b4c9cd0 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/indexing/Resources.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/indexing/Resources.java @@ -131,6 +131,23 @@ public static HttpInputSource wikipedia1Day() throw new RuntimeException(e); } } + + public static HttpInputSource kttm1Day() + { + try { + return new HttpInputSource( + List.of(new URIBuilder("https://static.imply.io/example-data/kttm-nested-v2/kttm-nested-v2-2019-08-25.json.gz").build()), + null, + null, + null, + null, + new HttpInputSourceConfig(null, null) + ); + } + catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } } /** diff --git a/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java b/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java index 09bbc0975461..db7845cda9e7 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java @@ -20,6 +20,7 @@ package org.apache.druid.testing.embedded.junit5; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.TestDataSource; import org.apache.druid.server.DruidNode; import org.apache.druid.testing.embedded.EmbeddedClusterApis; import org.apache.druid.testing.embedded.EmbeddedDruidCluster; @@ -98,6 +99,11 @@ protected void tearDown() @BeforeEach protected void refreshDatasourceName() { - dataSource = EmbeddedClusterApis.createTestDatasourceName(); + dataSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + } + + protected String getDatasourcePrefix() + { + return TestDataSource.WIKI; } } From fc63ca03e305090a6a2692c733287412661460cc Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 5 Dec 2025 15:30:19 -0800 Subject: [PATCH 02/17] checkstyle --- .../druid/testing/embedded/indexing/NestedDataFormatsTest.java | 1 + pom.xml | 2 +- .../org/apache/druid/testing/embedded/EmbeddedClusterApis.java | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index 72f9f13a9642..5c91853adf7d 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -122,6 +122,7 @@ public void test_objectStorageEncoding() final String scanQuery = "select event, to_json_string(agent) as agent from %s where json_value(event, '$.type') = 'PercentClear' and json_value(agent, '$.os') = 'Android' order by __time asc limit 1"; final String scanQueryResultDefaultFormat = cluster.callApi().runSql(scanQuery, defaultFormat); final String scanQueryResultNoneObjectStorage = cluster.callApi().runSql(scanQuery, dataSource); + // CHECKSTYLE: text blocks not supported in current Checkstyle version Assertions.assertEquals( """ "{""type"":""PercentClear"",""percentage"":85}","{""type"":""Mobile Browser"",""category"":""Smartphone"",""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""os"":""Android"",""platform"":""Android""}" diff --git a/pom.xml b/pom.xml index a56cd5be249b..55c65a262d41 100644 --- a/pom.xml +++ b/pom.xml @@ -1572,7 +1572,7 @@ true true - *com/fasterxml/jackson/databind/* + *com/fasterxml/jackson/databind/*,**/NestedDataFormatsTest.java diff --git a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java index 02772a49554a..e7abf9cadc70 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java @@ -41,7 +41,6 @@ import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.http.ClientSqlQuery; import org.apache.druid.rpc.indexing.OverlordClient; -import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.TestHelper; import org.apache.druid.server.metrics.LatchableEmitter; import org.apache.druid.sql.http.ResultFormat; From f2eb056d6bfd74dc84a1ad034d4eb3319018decc Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 5 Dec 2025 15:32:27 -0800 Subject: [PATCH 03/17] license --- .../indexing/NestedDataFormatsTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index 5c91853adf7d..d9ac454774dd 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.testing.embedded.indexing; import com.fasterxml.jackson.core.type.TypeReference; From 4d1d6e87c26dc7ca0df57622e46f5ad42223ab95 Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 5 Dec 2025 17:33:42 -0800 Subject: [PATCH 04/17] fix-test --- .../druid/testing/embedded/query/QueryVirtualStorageTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java index dc9174b8e1d0..767749f2030f 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java @@ -118,6 +118,7 @@ public EmbeddedDruidCluster createCluster() void loadData() throws IOException { msqApis = new EmbeddedMSQApis(cluster, overlord); + dataSource = "wiki_data"; loadWikiData(); } From 299766b59652c60c8c22af25da46006712f6fd99 Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 5 Dec 2025 18:35:50 -0800 Subject: [PATCH 05/17] emitter-config --- .../embedded/indexing/NestedDataFormatsTest.java | 13 +++++++------ .../server/metrics/LatchableEmitterConfig.java | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index d9ac454774dd..0465e0c807de 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -67,6 +67,7 @@ protected EmbeddedDruidCluster createCluster() { return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper() .useLatchableEmitter() + .useDefaultTimeoutForLatchableEmitter(60) .addServer(overlord) .addServer(coordinator) .addServer(new EmbeddedIndexer()) @@ -143,12 +144,12 @@ public void test_objectStorageEncoding() final String scanQueryResultNoneObjectStorage = cluster.callApi().runSql(scanQuery, dataSource); // CHECKSTYLE: text blocks not supported in current Checkstyle version Assertions.assertEquals( - """ - "{""type"":""PercentClear"",""percentage"":85}","{""type"":""Mobile Browser"",""category"":""Smartphone"",""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""os"":""Android"",""platform"":""Android""}" - """.trim(), scanQueryResultDefaultFormat); + """ + "{""type"":""PercentClear"",""percentage"":85}","{""type"":""Mobile Browser"",""category"":""Smartphone"",""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""os"":""Android"",""platform"":""Android""}" + """.trim(), scanQueryResultDefaultFormat); Assertions.assertEquals( - """ - "{""percentage"":85,""type"":""PercentClear""}","{""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""category"":""Smartphone"",""os"":""Android"",""platform"":""Android"",""type"":""Mobile Browser""}" - """.trim(), scanQueryResultNoneObjectStorage); + """ + "{""percentage"":85,""type"":""PercentClear""}","{""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""category"":""Smartphone"",""os"":""Android"",""platform"":""Android"",""type"":""Mobile Browser""}" + """.trim(), scanQueryResultNoneObjectStorage); } } diff --git a/server/src/test/java/org/apache/druid/server/metrics/LatchableEmitterConfig.java b/server/src/test/java/org/apache/druid/server/metrics/LatchableEmitterConfig.java index 71e0a8d14691..9ab358119aaf 100644 --- a/server/src/test/java/org/apache/druid/server/metrics/LatchableEmitterConfig.java +++ b/server/src/test/java/org/apache/druid/server/metrics/LatchableEmitterConfig.java @@ -35,7 +35,7 @@ public LatchableEmitterConfig( @JsonProperty("defaultWaitTimeoutMillis") @Nullable Long defaultWaitTimeoutMillis ) { - this.defaultWaitTimeoutMillis = Configs.valueOrDefault(defaultWaitTimeoutMillis, 50_000); + this.defaultWaitTimeoutMillis = Configs.valueOrDefault(defaultWaitTimeoutMillis, 10_000); } /** From 1deaef6b98d91246bb0a22bb4fa1dad5c4cb1589 Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 8 Dec 2025 11:25:34 -0800 Subject: [PATCH 06/17] setup --- .../testing/embedded/indexing/NestedDataFormatsTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index 0465e0c807de..9eebfe2da7fe 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -76,12 +76,9 @@ protected EmbeddedDruidCluster createCluster() .addServer(new EmbeddedRouter()); } - @Override @BeforeAll - protected void setup() throws Exception + protected void ingestWithDefaultFormat() throws Exception { - super.setup(); - super.refreshDatasourceName(); final TaskBuilder.IndexParallel indexTask = TaskBuilder.ofTypeIndexParallel() .dataSource(defaultFormat) From e79b5c711986c235f4e5dc510e6331dca3f30eb1 Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 8 Dec 2025 12:08:48 -0800 Subject: [PATCH 07/17] prefix --- .../ITCombiningInputSourceParallelIndexTest.java | 2 +- .../testing/embedded/indexer/ITTransformTest.java | 2 +- .../druid/testing/embedded/indexer/IndexerTest.java | 10 +++++----- .../embedded/indexing/IndexParallelTaskTest.java | 4 ++-- .../embedded/indexing/NestedDataFormatsTest.java | 6 ------ .../testing/embedded/msq/BaseRealtimeQueryTest.java | 2 +- .../msq/EmbeddedDurableShuffleStorageTest.java | 2 +- .../druid/testing/embedded/msq/ITMSQReindexTest.java | 2 +- .../embedded/query/ForcePushDownNestedQueryTest.java | 2 +- .../druid/testing/embedded/query/QueryErrorTest.java | 2 +- .../druid/testing/embedded/query/QueryTestBase.java | 2 +- .../druid/testing/embedded/query/UnionQueryTest.java | 2 +- .../java/org/apache/druid/segment/TestDataSource.java | 1 + .../druid/testing/embedded/EmbeddedClusterApis.java | 6 ++++++ .../embedded/junit5/EmbeddedClusterTestBase.java | 8 +------- 15 files changed, 24 insertions(+), 29 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITCombiningInputSourceParallelIndexTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITCombiningInputSourceParallelIndexTest.java index afb5cfc0b7b0..d3ae98af4c80 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITCombiningInputSourceParallelIndexTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITCombiningInputSourceParallelIndexTest.java @@ -51,7 +51,7 @@ protected void addResources(EmbeddedDruidCluster cluster) public void testIndexData() throws Exception { final String indexDatasource = dataSource; - final String combiningDatasource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String combiningDatasource = EmbeddedClusterApis.createTestDatasourceName(); Map inputFormatMap = new ImmutableMap .Builder() .put("type", "json") diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITTransformTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITTransformTest.java index 2c36b21f72b5..22fe4df46e03 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITTransformTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/ITTransformTest.java @@ -38,7 +38,7 @@ public class ITTransformTest extends AbstractITBatchIndexTest public void testIndexAndReIndexWithTransformSpec() throws IOException { final String indexDatasource = dataSource; - final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(); try ( final Closeable ignored1 = unloader(indexDatasource); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/IndexerTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/IndexerTest.java index 7e029ca83e5e..0b7a8d7faf6d 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/IndexerTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/IndexerTest.java @@ -79,8 +79,8 @@ protected void addResources(EmbeddedDruidCluster cluster) public void testIndexData() throws Exception { final String indexDatasource = dataSource; - final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); - final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(); + final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(); try ( final Closeable ignored1 = unloader(indexDatasource); final Closeable ignored2 = unloader(reindexDatasource); @@ -131,8 +131,8 @@ public void testIndexData() throws Exception public void testReIndexDataWithTimestamp() throws Exception { final String indexDatasource = dataSource; - final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); - final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(); + final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(); try ( final Closeable ignored1 = unloader(indexDatasource); final Closeable ignored2 = unloader(reindexDatasource); @@ -197,7 +197,7 @@ public void testReIndexWithNonExistingDatasource() throws Exception public void testMERGEIndexData() throws Exception { final String reindexDatasource = dataSource; - final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String reindexDatasourceWithDruidInputSource = EmbeddedClusterApis.createTestDatasourceName(); try ( final Closeable ignored2 = unloader(reindexDatasource); final Closeable ignored3 = unloader(reindexDatasourceWithDruidInputSource) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java index bfdc351871cb..23645dea5942 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexParallelTaskTest.java @@ -159,7 +159,7 @@ public void test_runIndexTask_andReindexIntoAnotherDatasource(PartitionsSpec par runQueries(dataSource); // Re-index into a different datasource, indexing 1 segment per sub-task - final String dataSource2 = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String dataSource2 = EmbeddedClusterApis.createTestDatasourceName(); final TaskBuilder.IndexParallel reindexTaskSplitBySegment = TaskBuilder.ofTypeIndexParallel() .dataSource(dataSource2) @@ -185,7 +185,7 @@ public void test_runIndexTask_andReindexIntoAnotherDatasource(PartitionsSpec par runQueries(dataSource2); // Re-index into a different datasource, indexing 1 file per sub-task - final String dataSource3 = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String dataSource3 = EmbeddedClusterApis.createTestDatasourceName(); final TaskBuilder.IndexParallel reindexTaskSplitByFile = TaskBuilder.ofTypeIndexParallel() .dataSource(dataSource3) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index 9eebfe2da7fe..33b7c5202624 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -56,12 +56,6 @@ public class NestedDataFormatsTest extends EmbeddedClusterTestBase private final String defaultFormat = "koala_default"; - @Override - protected String getDatasourcePrefix() - { - return TestDataSource.KOALA; - } - @Override protected EmbeddedDruidCluster createCluster() { diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java index e7ce8b22809e..ebc3ba723aaa 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java @@ -74,7 +74,7 @@ protected EmbeddedDruidCluster createCluster() void setupCreateKafkaTopicAndDatasource() { // Create Kafka topic. - topic = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + topic = EmbeddedClusterApis.createTestDatasourceName(); kafka.createTopicWithPartitions(topic, 2); super.refreshDatasourceName(); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java index db89ced3c98b..62f168d9ba7c 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java @@ -107,7 +107,7 @@ public EmbeddedDruidCluster createCluster() final void setupData() throws IOException { msqApis = new EmbeddedMSQApis(cluster, overlord); - dataSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + dataSource = EmbeddedClusterApis.createTestDatasourceName(); loadWikipediaTable(); } diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/ITMSQReindexTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/ITMSQReindexTest.java index 30e345547913..22936a6421e7 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/ITMSQReindexTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/ITMSQReindexTest.java @@ -52,7 +52,7 @@ public static Stream test_cases() public void testMSQDruidInputSource(String sqlFileName, String reIndexSqlFileName, String reIndexQueryFileName) { final String indexDatasource = dataSource; - final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + final String reindexDatasource = EmbeddedClusterApis.createTestDatasourceName(); Map context = ImmutableMap.of(MultiStageQueryContext.CTX_FINALIZE_AGGREGATIONS, false, MultiStageQueryContext.CTX_MAX_NUM_TASKS, 5, GroupByQueryConfig.CTX_KEY_ENABLE_MULTI_VALUE_UNNESTING, false); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/ForcePushDownNestedQueryTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/ForcePushDownNestedQueryTest.java index 0eca0658c65e..d57e18f9c25c 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/ForcePushDownNestedQueryTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/ForcePushDownNestedQueryTest.java @@ -73,7 +73,7 @@ protected EmbeddedDruidCluster createCluster() @Override public void beforeAll() throws IOException { - dataSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + dataSource = EmbeddedClusterApis.createTestDatasourceName(); loadWikipediaTable(); } diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryErrorTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryErrorTest.java index 4fd2ef35e1f4..5b7846b05709 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryErrorTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryErrorTest.java @@ -60,7 +60,7 @@ public class QueryErrorTest extends QueryTestBase @Override protected void beforeAll() { - testDataSourceName = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + testDataSourceName = EmbeddedClusterApis.createTestDatasourceName(); EmbeddedMSQApis msqApi = new EmbeddedMSQApis(cluster, overlord); SqlTaskStatus ingestionStatus = msqApi.submitTaskSql(StringUtils.format( "REPLACE INTO %s\n" diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryTestBase.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryTestBase.java index 989d724b0682..e025c90d2f14 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryTestBase.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryTestBase.java @@ -122,7 +122,7 @@ void setUp() */ protected String ingestBasicData() { - String datasourceName = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); + String datasourceName = EmbeddedClusterApis.createTestDatasourceName(); final String taskId = IdUtils.getRandomId(); final IndexTask task = MoreResources.Task.BASIC_INDEX.get().dataSource(datasourceName).withId(taskId); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java index 09eed296887e..61be086aec1d 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java @@ -95,7 +95,7 @@ public void test_ingestData_andVerifyNativeAndSQLQueries() final List datasourceNames = IntStream .range(0, numDatasources) - .mapToObj(i -> EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix())) + .mapToObj(i -> EmbeddedClusterApis.createTestDatasourceName()) .collect(Collectors.toList()); for (String datasourceName : datasourceNames) { diff --git a/processing/src/test/java/org/apache/druid/segment/TestDataSource.java b/processing/src/test/java/org/apache/druid/segment/TestDataSource.java index 8789d5f3f9a1..df51c1ebe3ff 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestDataSource.java +++ b/processing/src/test/java/org/apache/druid/segment/TestDataSource.java @@ -24,6 +24,7 @@ */ public final class TestDataSource { + public static final String PREFIX = "datasource"; public static final String WIKI = "wiki"; public static final String KOALA = "koala"; } diff --git a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java index e7abf9cadc70..05cdde6e5b52 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java @@ -41,6 +41,7 @@ import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.http.ClientSqlQuery; import org.apache.druid.rpc.indexing.OverlordClient; +import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.TestHelper; import org.apache.druid.server.metrics.LatchableEmitter; import org.apache.druid.sql.http.ResultFormat; @@ -410,6 +411,11 @@ public SupervisorStatus getSupervisorStatus(String supervisorId) // STATIC UTILITY METHODS + public static String createTestDatasourceName() + { + return createTestDatasourceName(TestDataSource.PREFIX); + } + /** * Creates a random datasource name with the given prefixed. */ diff --git a/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java b/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java index db7845cda9e7..09bbc0975461 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java @@ -20,7 +20,6 @@ package org.apache.druid.testing.embedded.junit5; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.segment.TestDataSource; import org.apache.druid.server.DruidNode; import org.apache.druid.testing.embedded.EmbeddedClusterApis; import org.apache.druid.testing.embedded.EmbeddedDruidCluster; @@ -99,11 +98,6 @@ protected void tearDown() @BeforeEach protected void refreshDatasourceName() { - dataSource = EmbeddedClusterApis.createTestDatasourceName(getDatasourcePrefix()); - } - - protected String getDatasourcePrefix() - { - return TestDataSource.WIKI; + dataSource = EmbeddedClusterApis.createTestDatasourceName(); } } From 4c20bde7f2f031e8d432914c21ee393ce1595bd6 Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 8 Dec 2025 14:37:05 -0800 Subject: [PATCH 08/17] small-update --- .../embedded/indexing/KafkaClusterMetricsTest.java | 3 +-- .../embedded/query/QueryVirtualStorageTest.java | 8 +++++++- .../druid/testing/embedded/EmbeddedClusterApis.java | 11 +++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java index 1fcbb4837655..8200bb335566 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaClusterMetricsTest.java @@ -31,7 +31,6 @@ import org.apache.druid.query.DruidMetrics; import org.apache.druid.rpc.UpdateResponse; import org.apache.druid.rpc.indexing.OverlordClient; -import org.apache.druid.segment.TestDataSource; import org.apache.druid.server.DruidNode; import org.apache.druid.server.coordinator.ClusterCompactionConfig; import org.apache.druid.server.coordinator.CoordinatorDynamicConfig; @@ -63,7 +62,7 @@ @SuppressWarnings("resource") public class KafkaClusterMetricsTest extends EmbeddedClusterTestBase { - private static final String TOPIC = EmbeddedClusterApis.createTestDatasourceName(TestDataSource.WIKI); + private static final String TOPIC = EmbeddedClusterApis.createTestDatasourceName(); private final EmbeddedBroker broker = new EmbeddedBroker(); private final EmbeddedIndexer indexer = new EmbeddedIndexer(); diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java index 767749f2030f..a5489041b80f 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java @@ -19,6 +19,7 @@ package org.apache.druid.testing.embedded.query; +import org.apache.druid.common.utils.IdUtils; import org.apache.druid.data.input.impl.LocalInputSource; import org.apache.druid.indexer.TaskState; import org.apache.druid.java.util.common.HumanReadableBytes; @@ -118,7 +119,7 @@ public EmbeddedDruidCluster createCluster() void loadData() throws IOException { msqApis = new EmbeddedMSQApis(cluster, overlord); - dataSource = "wiki_data"; + dataSource = createTestDatasourceName(); loadWikiData(); } @@ -273,6 +274,11 @@ private long getMetricLatestValue(LatchableEmitter emitter, String metricName, i return emitter.getLatestMetricEventValue(metricName, 0).longValue(); } + private String createTestDatasourceName() + { + return "wiki-" + IdUtils.getRandomId(); + } + /** * Stolen from {@link EmbeddedDurableShuffleStorageTest#loadWikipediaTable()} but with hourly granularity and no * durable shuffle location diff --git a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java index 05cdde6e5b52..8b7e3470b7b0 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java @@ -411,17 +411,12 @@ public SupervisorStatus getSupervisorStatus(String supervisorId) // STATIC UTILITY METHODS - public static String createTestDatasourceName() - { - return createTestDatasourceName(TestDataSource.PREFIX); - } - /** - * Creates a random datasource name with the given prefixed. + * Creates a random datasource name prefixed with {@link TestDataSource#PREFIX}. */ - public static String createTestDatasourceName(String prefix) + public static String createTestDatasourceName() { - return prefix + "_" + IdUtils.getRandomId(); + return TestDataSource.PREFIX + "_" + IdUtils.getRandomId(); } /** From 2eceaa9b54294ac6ff951e59bf62ddc68aa2526f Mon Sep 17 00:00:00 2001 From: Cece Mei Date: Mon, 8 Dec 2025 20:09:20 -0800 Subject: [PATCH 09/17] Update embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java Co-authored-by: Kashif Faraz --- .../druid/testing/embedded/indexing/NestedDataFormatsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index 33b7c5202624..c9ac5f5c9085 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -54,7 +54,7 @@ public class NestedDataFormatsTest extends EmbeddedClusterTestBase private final EmbeddedOverlord overlord = new EmbeddedOverlord(); private final EmbeddedCoordinator coordinator = new EmbeddedCoordinator(); - private final String defaultFormat = "koala_default"; + private final String datasourceWithDefaultFormat = EmbeddedClusterApis.createTestDatasourceName(); @Override protected EmbeddedDruidCluster createCluster() From 2aca1133332bbd261c8f839dbfd8329d6864018f Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 8 Dec 2025 20:11:19 -0800 Subject: [PATCH 10/17] datasourceWithDefaultFormat --- .../embedded/indexing/NestedDataFormatsTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index c9ac5f5c9085..4c369efa563d 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -75,15 +75,15 @@ protected void ingestWithDefaultFormat() throws Exception { final TaskBuilder.IndexParallel indexTask = TaskBuilder.ofTypeIndexParallel() - .dataSource(defaultFormat) + .dataSource(datasourceWithDefaultFormat) .timestampColumn("timestamp") .jsonInputFormat() .inputSource(Resources.HttpData.kttm1Day()) .schemaDiscovery(); - final String taskId = EmbeddedClusterApis.newTaskId(defaultFormat); + final String taskId = EmbeddedClusterApis.newTaskId(datasourceWithDefaultFormat); cluster.callApi().runTask(indexTask.withId(taskId), overlord); - cluster.callApi().waitForAllSegmentsToBeAvailable(defaultFormat, coordinator, broker); + cluster.callApi().waitForAllSegmentsToBeAvailable(datasourceWithDefaultFormat, coordinator, broker); } @Test @@ -114,7 +114,7 @@ public void test_objectStorageEncoding() } ); Map defaultFormatResult = - result.stream().filter(map -> defaultFormat.equals(map.get("datasource"))).findFirst().get(); + result.stream().filter(map -> datasourceWithDefaultFormat.equals(map.get("datasource"))).findFirst().get(); Map noneObjectStorageFormatResult = result.stream().filter(map -> dataSource.equals(map.get("datasource"))).findFirst().get(); // Test ingesting with skipping raw json smile format works, same row count, with ~20% storage saving @@ -125,13 +125,13 @@ public void test_objectStorageEncoding() // Test querying on a nested field works final String groupByQuery = "select json_value(event, '$.type') as event_type, count(*) as total from %s group by 1 order by 2 desc, 1 asc limit 10"; - final String queryResultDefaultFormat = cluster.callApi().runSql(groupByQuery, defaultFormat); + final String queryResultDefaultFormat = cluster.callApi().runSql(groupByQuery, datasourceWithDefaultFormat); final String queryResultNoneObjectStorage = cluster.callApi().runSql(groupByQuery, dataSource); Assertions.assertEquals(queryResultDefaultFormat, queryResultNoneObjectStorage); // Test reconstruct json column works, the ordering of the fields has changed, but all values are perserved. final String scanQuery = "select event, to_json_string(agent) as agent from %s where json_value(event, '$.type') = 'PercentClear' and json_value(agent, '$.os') = 'Android' order by __time asc limit 1"; - final String scanQueryResultDefaultFormat = cluster.callApi().runSql(scanQuery, defaultFormat); + final String scanQueryResultDefaultFormat = cluster.callApi().runSql(scanQuery, datasourceWithDefaultFormat); final String scanQueryResultNoneObjectStorage = cluster.callApi().runSql(scanQuery, dataSource); // CHECKSTYLE: text blocks not supported in current Checkstyle version Assertions.assertEquals( From 09bdf30bee13b65ae760e62593a1cd3fb334bbba Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 8 Dec 2025 20:16:57 -0800 Subject: [PATCH 11/17] prefix --- .../test/java/org/apache/druid/segment/TestDataSource.java | 1 - .../apache/druid/testing/embedded/EmbeddedClusterApis.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/segment/TestDataSource.java b/processing/src/test/java/org/apache/druid/segment/TestDataSource.java index df51c1ebe3ff..8789d5f3f9a1 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestDataSource.java +++ b/processing/src/test/java/org/apache/druid/segment/TestDataSource.java @@ -24,7 +24,6 @@ */ public final class TestDataSource { - public static final String PREFIX = "datasource"; public static final String WIKI = "wiki"; public static final String KOALA = "koala"; } diff --git a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java index 8b7e3470b7b0..499f7ad4a3e4 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java @@ -412,11 +412,11 @@ public SupervisorStatus getSupervisorStatus(String supervisorId) // STATIC UTILITY METHODS /** - * Creates a random datasource name prefixed with {@link TestDataSource#PREFIX}. + * Creates a random datasource name prefixed with {@code datasource_}. */ public static String createTestDatasourceName() { - return TestDataSource.PREFIX + "_" + IdUtils.getRandomId(); + return "datasource_" + IdUtils.getRandomId(); } /** From 170d354b741a0a3150ac01ab7ddc9ea0a1b65f03 Mon Sep 17 00:00:00 2001 From: Cece Mei Date: Mon, 8 Dec 2025 20:45:50 -0800 Subject: [PATCH 12/17] Update embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java Co-authored-by: Kashif Faraz --- .../testing/embedded/indexing/NestedDataFormatsTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index 4c369efa563d..bb27ac8c31d4 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -141,6 +141,8 @@ public void test_objectStorageEncoding() Assertions.assertEquals( """ "{""percentage"":85,""type"":""PercentClear""}","{""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""category"":""Smartphone"",""os"":""Android"",""platform"":""Android"",""type"":""Mobile Browser""}" - """.trim(), scanQueryResultNoneObjectStorage); + """.trim(), + scanQueryResultNoneObjectStorage + ); } } From 1eaace774c36c32d812cb1e4388c4892e1af5c8e Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 8 Dec 2025 20:46:46 -0800 Subject: [PATCH 13/17] style --- .../druid/testing/embedded/indexing/NestedDataFormatsTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index bb27ac8c31d4..d8fce01efd6c 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -137,7 +137,8 @@ public void test_objectStorageEncoding() Assertions.assertEquals( """ "{""type"":""PercentClear"",""percentage"":85}","{""type"":""Mobile Browser"",""category"":""Smartphone"",""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""os"":""Android"",""platform"":""Android""}" - """.trim(), scanQueryResultDefaultFormat); + """.trim(), + scanQueryResultDefaultFormat); Assertions.assertEquals( """ "{""percentage"":85,""type"":""PercentClear""}","{""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""category"":""Smartphone"",""os"":""Android"",""platform"":""Android"",""type"":""Mobile Browser""}" From 0786920cc75899c16e1f1ec8848ea0d7e5e12a5d Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 8 Dec 2025 21:10:37 -0800 Subject: [PATCH 14/17] sql --- .../indexing/NestedDataFormatsTest.java | 49 ++++++------------- .../testing/embedded/EmbeddedClusterApis.java | 1 - 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index d8fce01efd6c..25d7e6df8d35 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -19,13 +19,9 @@ package org.apache.druid.testing.embedded.indexing; -import com.fasterxml.jackson.core.type.TypeReference; import org.apache.druid.indexing.common.task.TaskBuilder; -import org.apache.druid.java.util.common.jackson.JacksonUtils; -import org.apache.druid.query.http.ClientSqlQuery; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.TestDataSource; -import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec; import org.apache.druid.segment.nested.ObjectStorageEncoding; import org.apache.druid.testing.embedded.EmbeddedBroker; @@ -41,10 +37,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; - /** * Embedded tests for nested data, ingested in different {@link NestedCommonFormatColumnFormatSpec}. */ @@ -71,7 +63,7 @@ protected EmbeddedDruidCluster createCluster() } @BeforeAll - protected void ingestWithDefaultFormat() throws Exception + protected void ingestWithDefaultFormat() { final TaskBuilder.IndexParallel indexTask = TaskBuilder.ofTypeIndexParallel() @@ -104,46 +96,35 @@ public void test_objectStorageEncoding() cluster.callApi().runTask(indexTask.withId(taskId), overlord); cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator, broker); - final String resultAsJson = - cluster.callApi().onAnyBroker(b -> b.submitSqlQuery(ClientSqlQuery.simple("select * from sys.segments"))); - List> result = JacksonUtils.readValue( - TestHelper.JSON_MAPPER, - resultAsJson.getBytes(StandardCharsets.UTF_8), - new TypeReference<>() - { - } - ); - Map defaultFormatResult = - result.stream().filter(map -> datasourceWithDefaultFormat.equals(map.get("datasource"))).findFirst().get(); - Map noneObjectStorageFormatResult = - result.stream().filter(map -> dataSource.equals(map.get("datasource"))).findFirst().get(); // Test ingesting with skipping raw json smile format works, same row count, with ~20% storage saving - Assertions.assertEquals(465_346, defaultFormatResult.get("num_rows")); - Assertions.assertEquals(53_000_804, defaultFormatResult.get("size")); - Assertions.assertEquals(465_346, noneObjectStorageFormatResult.get("num_rows")); - Assertions.assertEquals(41_938_750, noneObjectStorageFormatResult.get("size")); + final String metadata = "select sum(num_rows), sum(size) from sys.segments where datasource = '%s' group by datasource"; + final String defaultFormatResult = cluster.runSql(metadata, datasourceWithDefaultFormat); + final String noneObjectStorageFormatResult = cluster.runSql(metadata, dataSource); + Assertions.assertEquals(StringUtils.format("%d,%d", 465_346, 53_000_804), defaultFormatResult); + Assertions.assertEquals(StringUtils.format("%d,%d", 465_346, 41_938_750), noneObjectStorageFormatResult); // Test querying on a nested field works final String groupByQuery = "select json_value(event, '$.type') as event_type, count(*) as total from %s group by 1 order by 2 desc, 1 asc limit 10"; - final String queryResultDefaultFormat = cluster.callApi().runSql(groupByQuery, datasourceWithDefaultFormat); - final String queryResultNoneObjectStorage = cluster.callApi().runSql(groupByQuery, dataSource); + final String queryResultDefaultFormat = cluster.runSql(groupByQuery, datasourceWithDefaultFormat); + final String queryResultNoneObjectStorage = cluster.runSql(groupByQuery, dataSource); Assertions.assertEquals(queryResultDefaultFormat, queryResultNoneObjectStorage); // Test reconstruct json column works, the ordering of the fields has changed, but all values are perserved. final String scanQuery = "select event, to_json_string(agent) as agent from %s where json_value(event, '$.type') = 'PercentClear' and json_value(agent, '$.os') = 'Android' order by __time asc limit 1"; - final String scanQueryResultDefaultFormat = cluster.callApi().runSql(scanQuery, datasourceWithDefaultFormat); - final String scanQueryResultNoneObjectStorage = cluster.callApi().runSql(scanQuery, dataSource); + final String scanQueryResultDefaultFormat = cluster.runSql(scanQuery, datasourceWithDefaultFormat); + final String scanQueryResultNoneObjectStorage = cluster.runSql(scanQuery, dataSource); // CHECKSTYLE: text blocks not supported in current Checkstyle version Assertions.assertEquals( """ "{""type"":""PercentClear"",""percentage"":85}","{""type"":""Mobile Browser"",""category"":""Smartphone"",""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""os"":""Android"",""platform"":""Android""}" """.trim(), - scanQueryResultDefaultFormat); + scanQueryResultDefaultFormat + ); Assertions.assertEquals( """ "{""percentage"":85,""type"":""PercentClear""}","{""browser"":""Chrome Mobile"",""browser_version"":""50.0.2661.89"",""category"":""Smartphone"",""os"":""Android"",""platform"":""Android"",""type"":""Mobile Browser""}" """.trim(), - scanQueryResultNoneObjectStorage - ); + scanQueryResultNoneObjectStorage + ); } } diff --git a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java index 499f7ad4a3e4..f727bc7c97b5 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java @@ -41,7 +41,6 @@ import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.http.ClientSqlQuery; import org.apache.druid.rpc.indexing.OverlordClient; -import org.apache.druid.segment.TestDataSource; import org.apache.druid.segment.TestHelper; import org.apache.druid.server.metrics.LatchableEmitter; import org.apache.druid.sql.http.ResultFormat; From 285204cf73b9ef05f32102b104ad686e33d5d4ae Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 8 Dec 2025 21:16:02 -0800 Subject: [PATCH 15/17] nested --- .../testing/embedded/indexing/NestedDataFormatsTest.java | 4 ++-- .../org/apache/druid/testing/embedded/indexing/Resources.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index 25d7e6df8d35..4b37588ab644 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -70,7 +70,7 @@ protected void ingestWithDefaultFormat() .dataSource(datasourceWithDefaultFormat) .timestampColumn("timestamp") .jsonInputFormat() - .inputSource(Resources.HttpData.kttm1Day()) + .inputSource(Resources.HttpData.kttmNested1Day()) .schemaDiscovery(); final String taskId = EmbeddedClusterApis.newTaskId(datasourceWithDefaultFormat); @@ -89,7 +89,7 @@ public void test_objectStorageEncoding() .dataSource(dataSource) .timestampColumn("timestamp") .jsonInputFormat() - .inputSource(Resources.HttpData.kttm1Day()) + .inputSource(Resources.HttpData.kttmNested1Day()) .schemaDiscovery() .tuningConfig(t -> t.withIndexSpec(IndexSpec.builder().withAutoColumnFormatSpec(spec).build())); final String taskId = EmbeddedClusterApis.newTaskId(dataSource); diff --git a/services/src/test/java/org/apache/druid/testing/embedded/indexing/Resources.java b/services/src/test/java/org/apache/druid/testing/embedded/indexing/Resources.java index f5535b4c9cd0..9070a90ac9d4 100644 --- a/services/src/test/java/org/apache/druid/testing/embedded/indexing/Resources.java +++ b/services/src/test/java/org/apache/druid/testing/embedded/indexing/Resources.java @@ -132,7 +132,7 @@ public static HttpInputSource wikipedia1Day() } } - public static HttpInputSource kttm1Day() + public static HttpInputSource kttmNested1Day() { try { return new HttpInputSource( From 11b1f5b9ed742617bdbd9bec05f9078116086d6f Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 8 Dec 2025 21:17:18 -0800 Subject: [PATCH 16/17] client-sql --- .../org/apache/druid/query/http/ClientSqlQuery.java | 5 ----- .../apache/druid/query/http/ClientSqlQueryTest.java | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/http/ClientSqlQuery.java b/processing/src/main/java/org/apache/druid/query/http/ClientSqlQuery.java index 0af4fa386b46..83484551b5de 100644 --- a/processing/src/main/java/org/apache/druid/query/http/ClientSqlQuery.java +++ b/processing/src/main/java/org/apache/druid/query/http/ClientSqlQuery.java @@ -77,11 +77,6 @@ public ClientSqlQuery( this.parameters = parameters; } - public static ClientSqlQuery simple(String query) - { - return new ClientSqlQuery(query, null, false, false, false, Map.of(), List.of()); - } - public String getQuery() { return query; diff --git a/processing/src/test/java/org/apache/druid/query/http/ClientSqlQueryTest.java b/processing/src/test/java/org/apache/druid/query/http/ClientSqlQueryTest.java index 3cf3e5b0545d..10379b3146ac 100644 --- a/processing/src/test/java/org/apache/druid/query/http/ClientSqlQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/http/ClientSqlQueryTest.java @@ -26,9 +26,6 @@ import org.junit.Assert; import org.junit.Test; -import java.util.List; -import java.util.Map; - public class ClientSqlQueryTest { @Test @@ -62,11 +59,4 @@ public void testSerdeWithParameters() throws JsonProcessingException ); Assert.assertEquals(query, jsonMapper.readValue(jsonMapper.writeValueAsString(query), ClientSqlQuery.class)); } - - @Test - public void testSimple() - { - final ClientSqlQuery query = ClientSqlQuery.simple("select 1"); - Assert.assertEquals(new ClientSqlQuery("select 1", null, false, false, false, Map.of(), List.of()), query); - } } From 1d44512a2b8590eb5096909a5e60fcd7d937734d Mon Sep 17 00:00:00 2001 From: cecemei Date: Tue, 9 Dec 2025 09:57:05 -0800 Subject: [PATCH 17/17] format --- .../indexing/NestedDataFormatsTest.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java index 4b37588ab644..2f892e86d02f 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/NestedDataFormatsTest.java @@ -53,7 +53,7 @@ protected EmbeddedDruidCluster createCluster() { return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper() .useLatchableEmitter() - .useDefaultTimeoutForLatchableEmitter(60) + .useDefaultTimeoutForLatchableEmitter(120) .addServer(overlord) .addServer(coordinator) .addServer(new EmbeddedIndexer()) @@ -97,20 +97,29 @@ public void test_objectStorageEncoding() cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator, broker); // Test ingesting with skipping raw json smile format works, same row count, with ~20% storage saving - final String metadata = "select sum(num_rows), sum(size) from sys.segments where datasource = '%s' group by datasource"; - final String defaultFormatResult = cluster.runSql(metadata, datasourceWithDefaultFormat); - final String noneObjectStorageFormatResult = cluster.runSql(metadata, dataSource); + final String metadataSql = "select sum(num_rows), sum(size) from sys.segments where datasource = '%s'"; + final String defaultFormatResult = cluster.runSql(metadataSql, datasourceWithDefaultFormat); + final String noneObjectStorageFormatResult = cluster.runSql(metadataSql, dataSource); Assertions.assertEquals(StringUtils.format("%d,%d", 465_346, 53_000_804), defaultFormatResult); Assertions.assertEquals(StringUtils.format("%d,%d", 465_346, 41_938_750), noneObjectStorageFormatResult); // Test querying on a nested field works - final String groupByQuery = "select json_value(event, '$.type') as event_type, count(*) as total from %s group by 1 order by 2 desc, 1 asc limit 10"; + final String groupByQuery = + """ + select json_value(event, '$.type') as event_type, count(*) as total from %s + group by 1 order by 2 desc, 1 asc limit 10 + """; final String queryResultDefaultFormat = cluster.runSql(groupByQuery, datasourceWithDefaultFormat); final String queryResultNoneObjectStorage = cluster.runSql(groupByQuery, dataSource); Assertions.assertEquals(queryResultDefaultFormat, queryResultNoneObjectStorage); // Test reconstruct json column works, the ordering of the fields has changed, but all values are perserved. - final String scanQuery = "select event, to_json_string(agent) as agent from %s where json_value(event, '$.type') = 'PercentClear' and json_value(agent, '$.os') = 'Android' order by __time asc limit 1"; + final String scanQuery = + """ + select event, to_json_string(agent) as agent from %s + where json_value(event, '$.type') = 'PercentClear' and json_value(agent, '$.os') = 'Android' + order by __time asc limit 1 + """; final String scanQueryResultDefaultFormat = cluster.runSql(scanQuery, datasourceWithDefaultFormat); final String scanQueryResultNoneObjectStorage = cluster.runSql(scanQuery, dataSource); // CHECKSTYLE: text blocks not supported in current Checkstyle version