From c30cdae7b9e2f3793465c894821f545c7b767948 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 13 Sep 2024 23:08:03 -0700 Subject: [PATCH 1/5] add DataSchema.Builder to tidy stuff up a bit --- .../k8s/overlord/common/K8sTestUtils.java | 26 +- .../MaterializedViewSupervisorSpec.java | 16 +- .../MaterializedViewSupervisorTest.java | 13 +- .../RabbitStreamSupervisorTest.java | 25 +- .../indexing/kafka/KafkaIndexTaskTest.java | 94 ++--- .../indexing/kafka/KafkaSamplerSpecTest.java | 87 ++-- .../kafka/supervisor/KafkaSupervisorTest.java | 27 +- .../kinesis/KinesisIndexTaskSerdeTest.java | 3 +- .../kinesis/KinesisIndexTaskTest.java | 11 +- .../kinesis/KinesisSamplerSpecTest.java | 62 +-- .../supervisor/KinesisSupervisorTest.java | 27 +- .../destination/SegmentGenerationUtils.java | 16 +- .../msq/indexing/MSQCompactionRunnerTest.java | 27 +- .../indexer/BatchDeltaIngestionTest.java | 51 +-- .../DetermineHashedPartitionsJobTest.java | 80 ++-- .../indexer/DeterminePartitionsJobTest.java | 58 +-- .../DetermineRangePartitionsJobTest.java | 60 +-- .../indexer/HadoopDruidIndexerConfigTest.java | 25 +- .../indexer/HadoopDruidIndexerMapperTest.java | 45 +-- ...cUpdateDatasourcePathSpecSegmentsTest.java | 24 +- .../indexer/IndexGeneratorCombinerTest.java | 51 +-- .../druid/indexer/IndexGeneratorJobTest.java | 24 +- .../apache/druid/indexer/JobHelperTest.java | 95 +++-- .../indexer/path/DatasourcePathSpecTest.java | 56 +-- .../indexer/path/GranularityPathSpecTest.java | 46 ++- .../indexer/path/StaticPathSpecTest.java | 3 +- .../overlord/sampler/InputSourceSampler.java | 15 +- .../druid/indexing/common/TestIndexTask.java | 3 +- .../task/CompactionTaskParallelRunTest.java | 25 +- .../common/task/HadoopIndexTaskTest.java | 23 +- .../common/task/IndexIngestionSpecTest.java | 27 +- .../indexing/common/task/IndexTaskTest.java | 135 +++---- .../indexing/common/task/TaskSerdeTest.java | 98 ++--- ...bstractMultiPhaseParallelIndexingTest.java | 30 +- .../parallel/HashPartitionTaskKillTest.java | 33 +- ...aseParallelIndexingWithNullColumnTest.java | 110 +++--- .../ParallelIndexSupervisorTaskKillTest.java | 28 +- ...rallelIndexSupervisorTaskResourceTest.java | 28 +- .../ParallelIndexSupervisorTaskSerdeTest.java | 24 +- .../ParallelIndexSupervisorTaskTest.java | 63 ++- .../parallel/ParallelIndexTestingFactory.java | 19 +- .../parallel/RangePartitionTaskKillTest.java | 16 +- .../SinglePhaseParallelIndexingTest.java | 173 ++++---- .../parallel/SinglePhaseSubTaskSpecTest.java | 14 +- .../batch/parallel/TombstoneHelperTest.java | 6 +- .../indexing/input/InputRowSchemasTest.java | 38 +- .../indexing/overlord/TaskLifecycleTest.java | 100 ++--- .../indexing/overlord/TaskQueueTest.java | 17 +- .../sampler/CsvInputSourceSamplerTest.java | 13 +- .../InputSourceSamplerDiscoveryTest.java | 71 ++-- .../sampler/InputSourceSamplerTest.java | 32 +- ...SeekableStreamIndexTaskRunnerAuthTest.java | 19 +- .../SeekableStreamIndexTaskTestBase.java | 100 +++-- .../SeekableStreamSamplerSpecTest.java | 62 +-- .../SeekableStreamSupervisorSpecTest.java | 27 +- .../SeekableStreamSupervisorStateTest.java | 27 +- .../indexing/worker/TaskAnnouncementTest.java | 3 +- .../druid/segment/indexing/DataSchema.java | 176 ++++++--- .../segment/indexing/DataSchemaTest.java | 373 +++++++++--------- .../appenderator/BatchAppenderatorTester.java | 25 +- .../StreamAppenderatorTester.java | 22 +- ...nifiedIndexerAppenderatorsManagerTest.java | 13 +- .../druid/segment/realtime/sink/SinkTest.java | 40 +- .../cli/validate/DruidJsonValidatorTest.java | 14 +- 64 files changed, 1519 insertions(+), 1575 deletions(-) diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java index b3fda99b222e..161e80569521 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java @@ -34,7 +34,6 @@ import org.apache.druid.indexing.common.task.Tasks; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.indexing.DataSchema; @@ -66,18 +65,19 @@ public static Task getTask() null, null, new IndexTask.IndexIngestionSpec( - new DataSchema( - "foo", - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, - new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P2D")) - ), - null - ), + DataSchema.builder() + .withDataSource("foo") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.EMPTY) + .withAggregators(new DoubleSumAggregatorFactory("met", "met")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P2D")) + ) + ) + .build(), new IndexTask.IndexIOConfig( new LocalInputSource(new File("lol"), "rofl"), new NoopInputFormat(), diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java index 01039375259e..b23af62f6309 100644 --- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java +++ b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java @@ -48,7 +48,6 @@ import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import org.apache.druid.segment.realtime.ChatHandlerProvider; -import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; @@ -211,14 +210,13 @@ public HadoopIndexTask createTask(Interval interval, String version, List runningTasks = runningTasksPair.lhs; Map runningVersion = runningTasksPair.rhs; - DataSchema dataSchema = new DataSchema( - "test_datasource", - null, - null, - null, - TransformSpec.NONE, - objectMapper - ); + DataSchema dataSchema = DataSchema.builder() + .withDataSource("test_datasource") + .withObjectMapper(objectMapper) + .build(); HadoopIOConfig hadoopIOConfig = new HadoopIOConfig(new HashMap<>(), null, null); HadoopIngestionSpec spec = new HadoopIngestionSpec(dataSchema, hadoopIOConfig, null); HadoopIndexTask task1 = new HadoopIndexTask( diff --git a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTest.java b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTest.java index a5b9b597afa5..e52ca2f29b64 100644 --- a/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTest.java +++ b/extensions-contrib/rabbit-stream-indexing-service/src/test/java/org/apache/druid/indexing/rabbitstream/supervisor/RabbitStreamSupervisorTest.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; @@ -44,7 +43,6 @@ import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.metrics.DruidMonitorSchedulerConfig; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; @@ -102,16 +100,19 @@ private static DataSchema getDataSchema(String dataSource) dimensions.add(StringDimensionSchema.create("dim1")); dimensions.add(StringDimensionSchema.create("dim2")); - return new DataSchema( - dataSource, - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec(dimensions), - new AggregatorFactory[] {new CountAggregatorFactory("rows")}, - new UniformGranularitySpec( - Granularities.HOUR, - Granularities.NONE, - ImmutableList.of()), - null); + return DataSchema.builder() + .withDataSource(dataSource) + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions(dimensions) + .withAggregators(new CountAggregatorFactory("rows")) + .withGranularity( + new UniformGranularitySpec( + Granularities.HOUR, + Granularities.NONE, + ImmutableList.of() + ) + ) + .build(); } @BeforeClass diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java index dd04c2309f6e..23bdeb14acb8 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -92,7 +92,6 @@ import org.apache.druid.query.QueryRunnerFactory; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.SegmentDescriptor; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.filter.SelectorDimFilter; @@ -1262,28 +1261,27 @@ public void testKafkaRecordEntityInputFormat() throws Exception final KafkaIndexTask task = createTask( null, - new DataSchema( - "test_ds", - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec( - Arrays.asList( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim1t"), - new StringDimensionSchema("dim2"), - new LongDimensionSchema("dimLong"), - new FloatDimensionSchema("dimFloat"), - new StringDimensionSchema("kafka.topic"), - new LongDimensionSchema("kafka.offset"), - new StringDimensionSchema("kafka.header.encoding") - ) - ), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null - ), + DataSchema.builder() + .withDataSource("test_ds") + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions( + new StringDimensionSchema("dim1"), + new StringDimensionSchema("dim1t"), + new StringDimensionSchema("dim2"), + new LongDimensionSchema("dimLong"), + new FloatDimensionSchema("dimFloat"), + new StringDimensionSchema("kafka.topic"), + new LongDimensionSchema("kafka.offset"), + new StringDimensionSchema("kafka.header.encoding") + ) + .withAggregators( + new DoubleSumAggregatorFactory("met1sum", "met1"), + new CountAggregatorFactory("rows") + ) + .withGranularity( + new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null) + ) + .build(), new KafkaIndexTaskIOConfig( 0, "sequence0", @@ -1337,26 +1335,25 @@ public void testKafkaInputFormat() throws Exception final KafkaIndexTask task = createTask( null, - new DataSchema( - "test_ds", - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec( - Arrays.asList( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim1t"), - new StringDimensionSchema("dim2"), - new LongDimensionSchema("dimLong"), - new FloatDimensionSchema("dimFloat"), - new StringDimensionSchema("kafka.testheader.encoding") - ) - ), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null - ), + DataSchema.builder() + .withDataSource("test_ds") + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions( + new StringDimensionSchema("dim1"), + new StringDimensionSchema("dim1t"), + new StringDimensionSchema("dim2"), + new LongDimensionSchema("dimLong"), + new FloatDimensionSchema("dimFloat"), + new StringDimensionSchema("kafka.testheader.encoding") + ) + .withAggregators( + new DoubleSumAggregatorFactory("met1sum", "met1"), + new CountAggregatorFactory("rows") + ) + .withGranularity( + new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null) + ) + .build(), new KafkaIndexTaskIOConfig( 0, "sequence0", @@ -2888,16 +2885,7 @@ private KafkaIndexTask createTask( private static DataSchema cloneDataSchema(final DataSchema dataSchema) { - return new DataSchema( - dataSchema.getDataSource(), - dataSchema.getTimestampSpec(), - dataSchema.getDimensionsSpec(), - dataSchema.getAggregators(), - dataSchema.getGranularitySpec(), - dataSchema.getTransformSpec(), - dataSchema.getParserMap(), - OBJECT_MAPPER - ); + return DataSchema.builder(dataSchema).withObjectMapper(OBJECT_MAPPER).build(); } @Override diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java index 0a0b64396a66..9cdc0ac0edcd 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java @@ -46,7 +46,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.parsers.JSONPathSpec; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.TestHelper; @@ -81,45 +80,30 @@ public class KafkaSamplerSpecTest extends InitializedNullHandlingTest private static final ObjectMapper OBJECT_MAPPER = TestHelper.makeJsonMapper(); private static final String TOPIC = "sampling"; - private static final DataSchema DATA_SCHEMA = new DataSchema( - "test_ds", - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec( - Arrays.asList( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim1t"), - new StringDimensionSchema("dim2"), - new LongDimensionSchema("dimLong"), - new FloatDimensionSchema("dimFloat") - ) - ), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null - ); - - private static final DataSchema DATA_SCHEMA_KAFKA_TIMESTAMP = new DataSchema( - "test_ds", - new TimestampSpec("kafka.timestamp", "iso", null), - new DimensionsSpec( - Arrays.asList( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim1t"), - new StringDimensionSchema("dim2"), - new LongDimensionSchema("dimLong"), - new FloatDimensionSchema("dimFloat") - ) - ), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null - ); + private static final DataSchema DATA_SCHEMA = + DataSchema.builder() + .withDataSource("test_ds") + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions( + new StringDimensionSchema("dim1"), + new StringDimensionSchema("dim1t"), + new StringDimensionSchema("dim2"), + new LongDimensionSchema("dimLong"), + new FloatDimensionSchema("dimFloat") + ) + .withAggregators( + new DoubleSumAggregatorFactory("met1sum", "met1"), + new CountAggregatorFactory("rows") + ) + .withGranularity( + new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null) + ) + .build(); + + private static final DataSchema DATA_SCHEMA_KAFKA_TIMESTAMP = + DataSchema.builder(DATA_SCHEMA) + .withTimestamp(new TimestampSpec("kafka.timestamp", "iso", null)) + .build(); private static TestingCluster zkServer; private static TestBroker kafkaServer; @@ -364,17 +348,18 @@ public void testWithInputRowParser() throws IOException ); InputRowParser parser = new StringInputRowParser(new JSONParseSpec(timestampSpec, dimensionsSpec, JSONPathSpec.DEFAULT, null, null), "UTF8"); - DataSchema dataSchema = new DataSchema( - "test_ds", - objectMapper.readValue(objectMapper.writeValueAsBytes(parser), Map.class), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null, - objectMapper - ); + DataSchema dataSchema = DataSchema.builder() + .withDataSource("test_ds") + .withParserMap( + objectMapper.readValue(objectMapper.writeValueAsBytes(parser), Map.class) + ) + .withAggregators( + new DoubleSumAggregatorFactory("met1sum", "met1"), + new CountAggregatorFactory("rows") + ) + .withGranularity(new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null)) + .withObjectMapper(objectMapper) + .build(); KafkaSupervisorSpec supervisorSpec = new KafkaSupervisorSpec( null, diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 127cb72efcb1..e436b8cd56a5 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -30,7 +30,6 @@ import org.apache.curator.test.TestingCluster; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; @@ -83,7 +82,6 @@ import org.apache.druid.java.util.emitter.service.AlertEvent; import org.apache.druid.java.util.metrics.DruidMonitorSchedulerConfig; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.incremental.ParseExceptionReport; @@ -5135,18 +5133,19 @@ private static DataSchema getDataSchema(String dataSource) dimensions.add(StringDimensionSchema.create("dim1")); dimensions.add(StringDimensionSchema.create("dim2")); - return new DataSchema( - dataSource, - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec(dimensions), - new AggregatorFactory[]{new CountAggregatorFactory("rows")}, - new UniformGranularitySpec( - Granularities.HOUR, - Granularities.NONE, - ImmutableList.of() - ), - null - ); + return DataSchema.builder() + .withDataSource(dataSource) + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions(dimensions) + .withAggregators(new CountAggregatorFactory("rows")) + .withGranularity( + new UniformGranularitySpec( + Granularities.HOUR, + Granularities.NONE, + ImmutableList.of() + ) + ) + .build(); } private KafkaIndexTask createKafkaIndexTask( diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskSerdeTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskSerdeTest.java index ed2758ddaefd..e84581af6013 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskSerdeTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskSerdeTest.java @@ -50,7 +50,8 @@ public class KinesisIndexTaskSerdeTest { - private static final DataSchema DATA_SCHEMA = new DataSchema("dataSource", null, null, null, null, null, null, null); + private static final DataSchema DATA_SCHEMA = + DataSchema.builder().withDataSource("dataSource").build(); private static final KinesisIndexTaskTuningConfig TUNING_CONFIG = new KinesisIndexTaskTuningConfig( null, null, diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java index d69e43ca660c..2ef391484008 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java @@ -2412,16 +2412,7 @@ private KinesisIndexTask createTask( private static DataSchema cloneDataSchema(final DataSchema dataSchema) { - return new DataSchema( - dataSchema.getDataSource(), - dataSchema.getTimestampSpec(), - dataSchema.getDimensionsSpec(), - dataSchema.getAggregators(), - dataSchema.getGranularitySpec(), - dataSchema.getTransformSpec(), - dataSchema.getParserMap(), - OBJECT_MAPPER - ); + return DataSchema.builder(dataSchema).withObjectMapper(OBJECT_MAPPER).build(); } @Override diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java index 63144c6a9353..102e2d8929f2 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java @@ -49,7 +49,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.parsers.JSONPathSpec; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; @@ -75,25 +74,25 @@ public class KinesisSamplerSpecTest extends EasyMockSupport { private static final String STREAM = "sampling"; private static final String SHARD_ID = "1"; - private static final DataSchema DATA_SCHEMA = new DataSchema( - "test_ds", - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec( - Arrays.asList( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim1t"), - new StringDimensionSchema("dim2"), - new LongDimensionSchema("dimLong"), - new FloatDimensionSchema("dimFloat") - ) - ), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null - ); + private static final DataSchema DATA_SCHEMA = + DataSchema.builder() + .withDataSource("test_ds") + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions( + new StringDimensionSchema("dim1"), + new StringDimensionSchema("dim1t"), + new StringDimensionSchema("dim2"), + new LongDimensionSchema("dimLong"), + new FloatDimensionSchema("dimFloat") + ) + .withAggregators( + new DoubleSumAggregatorFactory("met1sum", "met1"), + new CountAggregatorFactory("rows") + ) + .withGranularity( + new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null) + ) + .build(); static { NullHandling.initializeForTests(); @@ -192,17 +191,18 @@ public void testSampleWithInputRowParser() throws IOException, InterruptedExcept ); InputRowParser parser = new StringInputRowParser(new JSONParseSpec(timestampSpec, dimensionsSpec, JSONPathSpec.DEFAULT, null, null), "UTF8"); - DataSchema dataSchema = new DataSchema( - "test_ds", - objectMapper.readValue(objectMapper.writeValueAsBytes(parser), Map.class), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null, - objectMapper - ); + DataSchema dataSchema = DataSchema.builder() + .withDataSource("test_ds") + .withParserMap( + objectMapper.readValue(objectMapper.writeValueAsBytes(parser), Map.class) + ) + .withAggregators( + new DoubleSumAggregatorFactory("met1sum", "met1"), + new CountAggregatorFactory("rows") + ) + .withGranularity(new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null)) + .withObjectMapper(objectMapper) + .build(); KinesisSupervisorSpec supervisorSpec = new KinesisSupervisorSpec( null, diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java index 50b7203c629c..24d919918f47 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java @@ -28,7 +28,6 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; @@ -79,7 +78,6 @@ import org.apache.druid.java.util.emitter.service.AlertEvent; import org.apache.druid.java.util.metrics.DruidMonitorSchedulerConfig; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; @@ -5482,18 +5480,19 @@ private static DataSchema getDataSchema(String dataSource) dimensions.add(StringDimensionSchema.create("dim1")); dimensions.add(StringDimensionSchema.create("dim2")); - return new DataSchema( - dataSource, - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec(dimensions), - new AggregatorFactory[]{new CountAggregatorFactory("rows")}, - new UniformGranularitySpec( - Granularities.HOUR, - Granularities.NONE, - ImmutableList.of() - ), - null - ); + return DataSchema.builder() + .withDataSource(dataSource) + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions(dimensions) + .withAggregators(new CountAggregatorFactory("rows")) + .withGranularity( + new UniformGranularitySpec( + Granularities.HOUR, + Granularities.NONE, + ImmutableList.of() + ) + ) + .build(); } diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java index 09d79534337c..d2cb1a210384 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java @@ -96,14 +96,14 @@ public static DataSchema makeDataSchemaForIngestion( destination.getDimensionSchemas() ); - return new DataSchema( - destination.getDataSource(), - new TimestampSpec(ColumnHolder.TIME_COLUMN_NAME, "millis", null), - dimensionsAndAggregators.lhs, - dimensionsAndAggregators.rhs.toArray(new AggregatorFactory[0]), - makeGranularitySpecForIngestion(querySpec.getQuery(), querySpec.getColumnMappings(), isRollupQuery, jsonMapper), - new TransformSpec(null, Collections.emptyList()) - ); + return DataSchema.builder() + .withDataSource(destination.getDataSource()) + .withTimestamp(new TimestampSpec(ColumnHolder.TIME_COLUMN_NAME, "millis", null)) + .withDimensions(dimensionsAndAggregators.lhs) + .withAggregators(dimensionsAndAggregators.rhs.toArray(new AggregatorFactory[0])) + .withGranularity(makeGranularitySpecForIngestion(querySpec.getQuery(), querySpec.getColumnMappings(), isRollupQuery, jsonMapper)) + .withTransform(new TransformSpec(null, Collections.emptyList())) + .build(); } private static GranularitySpec makeGranularitySpecForIngestion( diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java index 4088d5cecb10..b3f4029c52da 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java @@ -259,19 +259,20 @@ public void testMSQControllerTaskSpecWithScanIsValid() throws JsonProcessingExce null ); - DataSchema dataSchema = new DataSchema( - DATA_SOURCE, - new TimestampSpec(TIMESTAMP_COLUMN, null, null), - new DimensionsSpec(DIMENSIONS), - new AggregatorFactory[]{}, - new UniformGranularitySpec( - SEGMENT_GRANULARITY.getDefaultGranularity(), - null, - false, - Collections.singletonList(COMPACTION_INTERVAL) - ), - new TransformSpec(dimFilter, Collections.emptyList()) - ); + DataSchema dataSchema = + DataSchema.builder() + .withDataSource(DATA_SOURCE) + .withTimestamp(new TimestampSpec(TIMESTAMP_COLUMN, null, null)) + .withDimensions(DIMENSIONS) + .withGranularity( + new UniformGranularitySpec( + SEGMENT_GRANULARITY.getDefaultGranularity(), + null, + false, + Collections.singletonList(COMPACTION_INTERVAL) + ) + ) + .build(); List msqControllerTasks = MSQ_COMPACTION_RUNNER.createMsqControllerTasks( diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/BatchDeltaIngestionTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/BatchDeltaIngestionTest.java index 1e4f62ca6e1f..41dae9c9e65e 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/BatchDeltaIngestionTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/BatchDeltaIngestionTest.java @@ -433,30 +433,33 @@ private HadoopDruidIndexerConfig makeHadoopDruidIndexerConfig( { HadoopDruidIndexerConfig config = new HadoopDruidIndexerConfig( new HadoopIngestionSpec( - new DataSchema( - "website", - MAPPER.convertValue( - new StringInputRowParser( - new CSVParseSpec( - new TimestampSpec("timestamp", "yyyyMMddHH", null), - new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("host"))), - null, - ImmutableList.of("timestamp", "host", "host2", "visited_num"), - false, - 0 - ), - null - ), - Map.class - ), - aggregators != null ? aggregators : new AggregatorFactory[]{ - new LongSumAggregatorFactory("visited_sum", "visited_num"), - new HyperUniquesAggregatorFactory("unique_hosts", "host2") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, ImmutableList.of(INTERVAL_FULL)), - null, - MAPPER - ), + DataSchema.builder() + .withDataSource("website") + .withParserMap(MAPPER.convertValue( + new StringInputRowParser( + new CSVParseSpec( + new TimestampSpec("timestamp", "yyyyMMddHH", null), + new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("host"))), + null, + ImmutableList.of("timestamp", "host", "host2", "visited_num"), + false, + 0 + ), + null + ), + Map.class + )) + .withAggregators(aggregators != null ? aggregators : new AggregatorFactory[]{ + new LongSumAggregatorFactory("visited_sum", "visited_num"), + new HyperUniquesAggregatorFactory("unique_hosts", "host2") + }) + .withGranularity(new UniformGranularitySpec( + Granularities.DAY, + Granularities.NONE, + ImmutableList.of(INTERVAL_FULL) + )) + .withObjectMapper(MAPPER) + .build(), new HadoopIOConfig( inputSpec, null, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineHashedPartitionsJobTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineHashedPartitionsJobTest.java index 24a8ee0ef7eb..dd22a95083ce 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineHashedPartitionsJobTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineHashedPartitionsJobTest.java @@ -32,7 +32,6 @@ import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.granularity.PeriodGranularity; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; @@ -158,46 +157,45 @@ public DetermineHashedPartitionsJobTest( } HadoopIngestionSpec ingestionSpec = new HadoopIngestionSpec( - new DataSchema( - "test_schema", - HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( - new StringInputRowParser( - new DelimitedParseSpec( - new TimestampSpec("ts", null, null), - new DimensionsSpec( - DimensionsSpec.getDefaultSchemas(ImmutableList.of( - "market", - "quality", - "placement", - "placementish" - )) - ), - "\t", - null, - Arrays.asList( - "ts", - "market", - "quality", - "placement", - "placementish", - "index" - ), - false, - 0 - ), - null - ), - Map.class - ), - new AggregatorFactory[]{new DoubleSumAggregatorFactory("index", "index")}, - new UniformGranularitySpec( - segmentGranularity, - Granularities.NONE, - intervals - ), - null, - HadoopDruidIndexerConfig.JSON_MAPPER - ), + DataSchema.builder() + .withDataSource("test_schema") + .withParserMap(HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( + new StringInputRowParser( + new DelimitedParseSpec( + new TimestampSpec("ts", null, null), + new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of( + "market", + "quality", + "placement", + "placementish" + )) + ), + "\t", + null, + Arrays.asList( + "ts", + "market", + "quality", + "placement", + "placementish", + "index" + ), + false, + 0 + ), + null + ), + Map.class + )) + .withAggregators(new DoubleSumAggregatorFactory("index", "index")) + .withGranularity(new UniformGranularitySpec( + segmentGranularity, + Granularities.NONE, + intervals + )) + .withObjectMapper(HadoopDruidIndexerConfig.JSON_MAPPER) + .build(), new HadoopIOConfig( ImmutableMap.of( "paths", diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java index a3c98f29565b..bfd28d2cfca0 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java @@ -29,7 +29,6 @@ import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; @@ -280,33 +279,36 @@ public DeterminePartitionsJobTest( config = new HadoopDruidIndexerConfig( new HadoopIngestionSpec( - new DataSchema( - "website", - HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( - new StringInputRowParser( - new CSVParseSpec( - new TimestampSpec("timestamp", "yyyyMMddHH", null), - new DimensionsSpec( - DimensionsSpec.getDefaultSchemas(ImmutableList.of("host", "country")) - ), - null, - ImmutableList.of("timestamp", "host", "country", "visited_num"), - false, - 0 - ), - null - ), - Map.class - ), - new AggregatorFactory[]{new LongSumAggregatorFactory("visited_num", "visited_num")}, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.NONE, - ImmutableList.of(Intervals.of(interval)) - ), - null, - HadoopDruidIndexerConfig.JSON_MAPPER - ), + DataSchema.builder() + .withDataSource("website") + .withParserMap( + HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( + new StringInputRowParser( + new CSVParseSpec( + new TimestampSpec("timestamp", "yyyyMMddHH", null), + new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("host", "country")) + ), + null, + ImmutableList.of("timestamp", "host", "country", "visited_num"), + false, + 0 + ), + null + ), + Map.class + ) + ) + .withAggregators(new LongSumAggregatorFactory("visited_num", "visited_num")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.NONE, + ImmutableList.of(Intervals.of(interval)) + ) + ) + .withObjectMapper(HadoopDruidIndexerConfig.JSON_MAPPER) + .build(), new HadoopIOConfig( ImmutableMap.of( "paths", diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineRangePartitionsJobTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineRangePartitionsJobTest.java index e79d066ab55c..3ff525c8b433 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineRangePartitionsJobTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineRangePartitionsJobTest.java @@ -29,7 +29,6 @@ import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; @@ -328,35 +327,36 @@ public DetermineRangePartitionsJobTest( config = new HadoopDruidIndexerConfig( new HadoopIngestionSpec( - new DataSchema( - "website", - null, - null, - new AggregatorFactory[]{new LongSumAggregatorFactory("visited_num", "visited_num")}, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.NONE, - ImmutableList.of(Intervals.of(interval)) - ), - null, - HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( - new StringInputRowParser( - new CSVParseSpec( - new TimestampSpec("timestamp", "yyyyMMddHH", null), - new DimensionsSpec( - DimensionsSpec.getDefaultSchemas(ImmutableList.of("host", "country")) - ), - null, - ImmutableList.of("timestamp", "host", "country", "visited_num"), - false, - 0 - ), - null - ), - Map.class - ), - HadoopDruidIndexerConfig.JSON_MAPPER - ), + DataSchema.builder() + .withDataSource("website") + .withAggregators(new LongSumAggregatorFactory("visited_num", "visited_num")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.NONE, + ImmutableList.of(Intervals.of(interval)) + ) + ) + .withParserMap( + HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( + new StringInputRowParser( + new CSVParseSpec( + new TimestampSpec("timestamp", "yyyyMMddHH", null), + new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("host", "country")) + ), + null, + ImmutableList.of("timestamp", "host", "country", "visited_num"), + false, + 0 + ), + null + ), + Map.class + ) + ) + .withObjectMapper(HadoopDruidIndexerConfig.JSON_MAPPER) + .build(), new HadoopIOConfig( ImmutableMap.of( "paths", diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerConfigTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerConfigTest.java index 8aead05d625b..ed47d180b432 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerConfigTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerConfigTest.java @@ -37,7 +37,6 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec; @@ -217,18 +216,18 @@ public void testGetTargetPartitionSizeWithSingleDimensionPartitionsMaxRowsPerSeg private static class HadoopIngestionSpecBuilder { - private static final DataSchema DATA_SCHEMA = new DataSchema( - "foo", - null, - new AggregatorFactory[0], - new UniformGranularitySpec( - Granularities.MINUTE, - Granularities.MINUTE, - ImmutableList.of(Intervals.of("2010-01-01/P1D")) - ), - null, - HadoopDruidIndexerConfigTest.JSON_MAPPER - ); + private static final DataSchema DATA_SCHEMA = + DataSchema.builder() + .withDataSource("foo") + .withGranularity( + new UniformGranularitySpec( + Granularities.MINUTE, + Granularities.MINUTE, + ImmutableList.of(Intervals.of("2010-01-01/P1D")) + ) + ) + .withObjectMapper(HadoopDruidIndexerConfigTest.JSON_MAPPER) + .build(); private static final HadoopIOConfig HADOOP_IO_CONFIG = new HadoopIOConfig( ImmutableMap.of("paths", "bar", "type", "static"), diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerMapperTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerMapperTest.java index db20ed8a1847..da57b8ccf4a3 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerMapperTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerMapperTest.java @@ -31,7 +31,6 @@ import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.common.parsers.JSONPathSpec; import org.apache.druid.math.expr.ExprMacroTable; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.segment.TestHelper; @@ -58,27 +57,29 @@ public class HadoopDruidIndexerMapperTest { private static final ObjectMapper JSON_MAPPER = TestHelper.makeJsonMapper(); - private static final DataSchema DATA_SCHEMA = new DataSchema( - "test_ds", - JSON_MAPPER.convertValue( - new HadoopyStringInputRowParser( - new JSONParseSpec( - new TimestampSpec("t", "auto", null), - new DimensionsSpec( - DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim1t", "dim2")) - ), - new JSONPathSpec(true, ImmutableList.of()), - ImmutableMap.of(), - null - ) - ), - JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT - ), - new AggregatorFactory[]{new CountAggregatorFactory("rows")}, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null, - JSON_MAPPER - ); + private static final DataSchema DATA_SCHEMA = + DataSchema.builder() + .withDataSource("test_ds") + .withParserMap( + JSON_MAPPER.convertValue( + new HadoopyStringInputRowParser( + new JSONParseSpec( + new TimestampSpec("t", "auto", null), + new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim1t", "dim2")) + ), + new JSONPathSpec(true, ImmutableList.of()), + ImmutableMap.of(), + null + ) + ), + JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT + ) + ) + .withAggregators(new CountAggregatorFactory("rows")) + .withGranularity(new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null)) + .withObjectMapper(JSON_MAPPER) + .build(); private static final HadoopIOConfig IO_CONFIG = new HadoopIOConfig( JSON_MAPPER.convertValue( diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest.java index 6402721e73c6..afcfb4023595 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest.java @@ -34,7 +34,6 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; import org.apache.druid.timeline.DataSegment; @@ -274,18 +273,17 @@ private HadoopDruidIndexerConfig testRunUpdateSegmentListIfDatasourcePathSpecIsU throws Exception { HadoopIngestionSpec spec = new HadoopIngestionSpec( - new DataSchema( - "foo", - null, - new AggregatorFactory[0], - new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P1D")) - ), - null, - jsonMapper - ), + DataSchema.builder() + .withDataSource("foo") + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P1D")) + ) + ) + .withObjectMapper(jsonMapper) + .build(), new HadoopIOConfig( jsonMapper.convertValue(datasourcePathSpec, Map.class), null, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java index 385c28ff0fb0..37cdbb7300d7 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java @@ -64,30 +64,33 @@ public void setUp() throws Exception { HadoopDruidIndexerConfig config = new HadoopDruidIndexerConfig( new HadoopIngestionSpec( - new DataSchema( - "website", - HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( - new StringInputRowParser( - new TimeAndDimsParseSpec( - new TimestampSpec("timestamp", "yyyyMMddHH", null), - new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("host", "keywords"))) - ), - null - ), - Map.class - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("visited_sum", "visited"), - new HyperUniquesAggregatorFactory("unique_hosts", "host") - }, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.NONE, - ImmutableList.of(Intervals.of("2010/2011")) - ), - null, - HadoopDruidIndexerConfig.JSON_MAPPER - ), + DataSchema.builder() + .withDataSource("website") + .withParserMap( + HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( + new StringInputRowParser( + new TimeAndDimsParseSpec( + new TimestampSpec("timestamp", "yyyyMMddHH", null), + new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("host", "keywords"))) + ), + null + ), + Map.class + ) + ) + .withAggregators( + new LongSumAggregatorFactory("visited_sum", "visited"), + new HyperUniquesAggregatorFactory("unique_hosts", "host") + ) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.NONE, + ImmutableList.of(Intervals.of("2010/2011")) + ) + ) + .withObjectMapper(HadoopDruidIndexerConfig.JSON_MAPPER) + .build(), new HadoopIOConfig( ImmutableMap.of( "paths", diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorJobTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorJobTest.java index e14ade454f4c..241746ca58d6 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorJobTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorJobTest.java @@ -506,17 +506,19 @@ public void setUp() throws Exception config = new HadoopDruidIndexerConfig( new HadoopIngestionSpec( - new DataSchema( - datasourceName, - mapper.convertValue( - inputRowParser, - Map.class - ), - aggs, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, ImmutableList.of(this.interval)), - null, - mapper - ), + DataSchema.builder() + .withDataSource(datasourceName) + .withParserMap(mapper.convertValue(inputRowParser, Map.class)) + .withAggregators(aggs) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.NONE, + ImmutableList.of(interval) + ) + ) + .withObjectMapper(mapper) + .build(), new HadoopIOConfig( ImmutableMap.copyOf(inputSpec), null, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/JobHelperTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/JobHelperTest.java index 7069e9a78de3..530c0f657a51 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/JobHelperTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/JobHelperTest.java @@ -34,7 +34,6 @@ import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.common.parsers.JSONPathSpec; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.TestHelper; @@ -68,27 +67,30 @@ public class JobHelperTest { private static final ObjectMapper JSON_MAPPER = TestHelper.makeJsonMapper(); - private static final DataSchema DATA_SCHEMA = new DataSchema( - "test_ds", - JSON_MAPPER.convertValue( - new HadoopyStringInputRowParser( - new JSONParseSpec( - new TimestampSpec("t", "auto", null), - new DimensionsSpec( - DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim1t", "dim2")) - ), - new JSONPathSpec(true, ImmutableList.of()), - ImmutableMap.of(), - null - ) - ), - JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT - ), - new AggregatorFactory[]{new CountAggregatorFactory("rows")}, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null, - JSON_MAPPER - ); + + private static final DataSchema DATA_SCHEMA = + DataSchema.builder() + .withDataSource("test_ds") + .withParserMap( + JSON_MAPPER.convertValue( + new HadoopyStringInputRowParser( + new JSONParseSpec( + new TimestampSpec("t", "auto", null), + new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim1t", "dim2")) + ), + new JSONPathSpec(true, ImmutableList.of()), + ImmutableMap.of(), + null + ) + ), + JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT + ) + ) + .withAggregators(new CountAggregatorFactory("rows")) + .withGranularity(new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null)) + .withObjectMapper(JSON_MAPPER) + .build(); private static final HadoopIOConfig IO_CONFIG = new HadoopIOConfig( JSON_MAPPER.convertValue( @@ -123,27 +125,34 @@ public void setup() throws Exception dataFile = temporaryFolder.newFile(); config = new HadoopDruidIndexerConfig( new HadoopIngestionSpec( - new DataSchema( - "website", - HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( - new StringInputRowParser( - new CSVParseSpec( - new TimestampSpec("timestamp", "yyyyMMddHH", null), - new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("host"))), - null, - ImmutableList.of("timestamp", "host", "visited_num"), - false, - 0 - ), - null - ), - Map.class - ), - new AggregatorFactory[]{new LongSumAggregatorFactory("visited_num", "visited_num")}, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, ImmutableList.of(this.interval)), - null, - HadoopDruidIndexerConfig.JSON_MAPPER - ), + DataSchema.builder() + .withDataSource("website") + .withParserMap( + HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( + new StringInputRowParser( + new CSVParseSpec( + new TimestampSpec("timestamp", "yyyyMMddHH", null), + new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("host"))), + null, + ImmutableList.of("timestamp", "host", "visited_num"), + false, + 0 + ), + null + ), + Map.class + ) + ) + .withAggregators(new LongSumAggregatorFactory("visited_num", "visited_num")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.NONE, + ImmutableList.of(this.interval) + ) + ) + .withObjectMapper(HadoopDruidIndexerConfig.JSON_MAPPER) + .build(), new HadoopIOConfig( ImmutableMap.of( "paths", diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/DatasourcePathSpecTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/DatasourcePathSpecTest.java index e8caea0256e0..75a4fe45eee6 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/DatasourcePathSpecTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/DatasourcePathSpecTest.java @@ -44,7 +44,6 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; @@ -308,33 +307,34 @@ private HadoopDruidIndexerConfig makeHadoopDruidIndexerConfig() { return new HadoopDruidIndexerConfig( new HadoopIngestionSpec( - new DataSchema( - ingestionSpec1.getDataSource(), - HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( - new StringInputRowParser( - new CSVParseSpec( - new TimestampSpec("timestamp", "yyyyMMddHH", null), - DimensionsSpec.EMPTY, - null, - ImmutableList.of("timestamp", "host", "visited"), - false, - 0 - ), - null - ), - Map.class - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("visited_sum", "visited") - }, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.NONE, - ImmutableList.of(Intervals.of("2000/3000")) - ), - null, - HadoopDruidIndexerConfig.JSON_MAPPER - ), + DataSchema.builder() + .withDataSource(ingestionSpec1.getDataSource()) + .withParserMap( + HadoopDruidIndexerConfig.JSON_MAPPER.convertValue( + new StringInputRowParser( + new CSVParseSpec( + new TimestampSpec("timestamp", "yyyyMMddHH", null), + DimensionsSpec.EMPTY, + null, + ImmutableList.of("timestamp", "host", "visited"), + false, + 0 + ), + null + ), + Map.class + ) + ) + .withAggregators(new LongSumAggregatorFactory("visited_sum", "visited")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.NONE, + ImmutableList.of(Intervals.of("2000/3000")) + ) + ) + .withObjectMapper(HadoopDruidIndexerConfig.JSON_MAPPER) + .build(), new HadoopIOConfig( ImmutableMap.of( "paths", diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java index 8af77ca0e4fd..52258f201619 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java @@ -152,18 +152,17 @@ public void testAddInputPath() throws Exception { UserGroupInformation.setLoginUser(UserGroupInformation.createUserForTesting("test", new String[]{"testGroup"})); HadoopIngestionSpec spec = new HadoopIngestionSpec( - new DataSchema( - "foo", - null, - new AggregatorFactory[0], - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - ImmutableList.of(Intervals.of("2015-11-06T00:00Z/2015-11-07T00:00Z")) - ), - null, - jsonMapper - ), + DataSchema.builder() + .withDataSource("foo") + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + ImmutableList.of(Intervals.of("2015-11-06T00:00Z/2015-11-07T00:00Z")) + ) + ) + .withObjectMapper(jsonMapper) + .build(), new HadoopIOConfig(null, null, null), DEFAULT_TUNING_CONFIG ); @@ -204,18 +203,17 @@ public void testIntervalTrimming() throws Exception { UserGroupInformation.setLoginUser(UserGroupInformation.createUserForTesting("test", new String[]{"testGroup"})); HadoopIngestionSpec spec = new HadoopIngestionSpec( - new DataSchema( - "foo", - null, - new AggregatorFactory[0], - new UniformGranularitySpec( - Granularities.DAY, - Granularities.ALL, - ImmutableList.of(Intervals.of("2015-01-01T11Z/2015-01-02T05Z")) - ), - null, - jsonMapper - ), + DataSchema.builder() + .withDataSource("foo") + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.ALL, + ImmutableList.of(Intervals.of("2015-01-01T11Z/2015-01-02T05Z")) + ) + ) + .withObjectMapper(jsonMapper) + .build(), new HadoopIOConfig(null, null, null), DEFAULT_TUNING_CONFIG ); diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/StaticPathSpecTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/StaticPathSpecTest.java index 06a1416ad83f..fa1b2f59f48e 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/StaticPathSpecTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/StaticPathSpecTest.java @@ -24,7 +24,6 @@ import org.apache.druid.indexer.HadoopIOConfig; import org.apache.druid.indexer.HadoopIngestionSpec; import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; @@ -54,7 +53,7 @@ public void testAddingPaths() throws Exception Job job = new Job(); StaticPathSpec pathSpec = new StaticPathSpec("/a/c,/a/b/{c,d}", null); - DataSchema schema = new DataSchema("ds", null, new AggregatorFactory[0], null, null, jsonMapper); + DataSchema schema = DataSchema.builder().withDataSource("ds").withObjectMapper(jsonMapper).build(); HadoopIOConfig io = new HadoopIOConfig(null, null, null); pathSpec.addInputPaths(new HadoopDruidIndexerConfig(new HadoopIngestionSpec(schema, io, null)), job); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java index f98287124ed0..8b4795d75f51 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java @@ -67,14 +67,13 @@ public class InputSourceSampler { private static final String SAMPLER_DATA_SOURCE = "sampler"; - private static final DataSchema DEFAULT_DATA_SCHEMA = new DataSchema( - SAMPLER_DATA_SOURCE, - new TimestampSpec(null, null, null), - new DimensionsSpec(null), - null, - null, - null - ); + + private static final DataSchema DEFAULT_DATA_SCHEMA = + DataSchema.builder() + .withDataSource(SAMPLER_DATA_SOURCE) + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.builder().build()) + .build(); // We want to be able to sort the list of processed results back into the same order that we read them from the input // source so that the rows in the data loader are not always changing. To do this, we add a temporary column to the diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/TestIndexTask.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/TestIndexTask.java index f6732f68a6c4..ff0aa674ef3f 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/TestIndexTask.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/TestIndexTask.java @@ -33,7 +33,6 @@ import org.apache.druid.indexing.common.task.TaskResource; import org.apache.druid.indexing.common.task.TuningConfigBuilder; import org.apache.druid.indexing.overlord.SegmentPublishResult; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.SegmentSchemaMapping; import org.apache.druid.segment.indexing.DataSchema; @@ -62,7 +61,7 @@ public TestIndexTask( id, taskResource, new IndexIngestionSpec( - new DataSchema(dataSource, null, new AggregatorFactory[]{}, null, null, mapper), + DataSchema.builder().withDataSource(dataSource).withObjectMapper(mapper).build(), new IndexTask.IndexIOConfig( new LocalInputSource(new File("lol"), "rofl"), new JsonInputFormat(null, null, null, null, null), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java index 1b742971eb95..377f4ece0657 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java @@ -939,18 +939,19 @@ private void runIndexTask(@Nullable PartitionsSpec partitionsSpec, boolean appen null, null, new ParallelIndexIngestionSpec( - new DataSchema( - DATA_SOURCE, - new TimestampSpec("ts", "auto", null), - new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("ts", "dim"))), - new AggregatorFactory[]{new LongSumAggregatorFactory("val", "val")}, - new UniformGranularitySpec( - Granularities.HOUR, - Granularities.MINUTE, - ImmutableList.of(INTERVAL_TO_INDEX) - ), - null - ), + DataSchema.builder() + .withDataSource(DATA_SOURCE) + .withTimestamp(new TimestampSpec("ts", "auto", null)) + .withDimensions(DimensionsSpec.getDefaultSchemas(Arrays.asList("ts", "dim"))) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity( + new UniformGranularitySpec( + Granularities.HOUR, + Granularities.MINUTE, + ImmutableList.of(INTERVAL_TO_INDEX) + ) + ) + .build(), ioConfig, tuningConfig ), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/HadoopIndexTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/HadoopIndexTaskTest.java index ff828f16789d..0a72b77e1db7 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/HadoopIndexTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/HadoopIndexTaskTest.java @@ -27,7 +27,6 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; import org.apache.druid.server.security.Action; @@ -50,15 +49,19 @@ public void testCorrectInputSourceResources() final HadoopIndexTask task = new HadoopIndexTask( null, new HadoopIngestionSpec( - new DataSchema( - "foo", null, new AggregatorFactory[0], new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P1D")) - ), - null, - jsonMapper - ), new HadoopIOConfig(ImmutableMap.of("paths", "bar"), null, null), null + DataSchema.builder() + .withDataSource("foo") + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P1D")) + ) + ) + .withObjectMapper(jsonMapper) + .build(), + new HadoopIOConfig(ImmutableMap.of("paths", "bar"), null, null), + null ), null, null, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexIngestionSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexIngestionSpecTest.java index ab953ba954ac..d84aa154fd26 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexIngestionSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexIngestionSpecTest.java @@ -25,7 +25,6 @@ import org.apache.druid.indexing.common.task.IndexTask.IndexIOConfig; import org.apache.druid.indexing.common.task.IndexTask.IndexIngestionSpec; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import org.junit.Rule; @@ -45,14 +44,11 @@ public void testParserAndInputFormat() "Cannot use parser and inputSource together. Try using inputFormat instead of parser." ); final IndexIngestionSpec spec = new IndexIngestionSpec( - new DataSchema( - "dataSource", - ImmutableMap.of("fake", "parser map"), - new AggregatorFactory[0], - new ArbitraryGranularitySpec(Granularities.NONE, null), - null, - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withParserMap(ImmutableMap.of("fake", "parser map")) + .withGranularity(new ArbitraryGranularitySpec(Granularities.NONE, null)) + .build(), new IndexIOConfig( new NoopInputSource(), new NoopInputFormat(), @@ -69,14 +65,11 @@ public void testParserAndInputSource() expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Cannot use parser and inputSource together."); final IndexIngestionSpec spec = new IndexIngestionSpec( - new DataSchema( - "dataSource", - ImmutableMap.of("fake", "parser map"), - new AggregatorFactory[0], - new ArbitraryGranularitySpec(Granularities.NONE, null), - null, - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withParserMap(ImmutableMap.of("fake", "parser map")) + .withGranularity(new ArbitraryGranularitySpec(Granularities.NONE, null)) + .build(), new IndexIOConfig( new NoopInputSource(), null, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java index d03ccf465e57..defa2107b9ac 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java @@ -166,6 +166,25 @@ public class IndexTaskTest extends IngestionTestBase 0 ); + private static final DataSchema DATA_SCHEMA = + DataSchema.builder() + .withDataSource("test-json") + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions( + new StringDimensionSchema("ts"), + new StringDimensionSchema("dim"), + new LongDimensionSchema("valDim") + ) + .withAggregators(new LongSumAggregatorFactory("valMet", "val")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + Collections.singletonList(Intervals.of("2014/P1D")) + ) + ) + .build(); + @Parameterized.Parameters(name = "{0}, useInputFormatApi={1}") public static Iterable constructorFeeder() { @@ -225,24 +244,7 @@ public void testCorrectInputSourceResources() { IndexTask indexTask = createIndexTask( new IndexIngestionSpec( - new DataSchema( - "test-json", - DEFAULT_TIMESTAMP_SPEC, - new DimensionsSpec( - ImmutableList.of( - new StringDimensionSchema("ts"), - new StringDimensionSchema("dim"), - new LongDimensionSchema("valDim") - ) - ), - new AggregatorFactory[]{new LongSumAggregatorFactory("valMet", "val")}, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(Intervals.of("2014/P1D")) - ), - null - ), + DATA_SCHEMA, new IndexIOConfig( new LocalInputSource(tmpDir, "druid*"), DEFAULT_INPUT_FORMAT, @@ -275,24 +277,7 @@ public void testIngestNullOnlyColumns() throws Exception IndexTask indexTask = createIndexTask( new IndexIngestionSpec( - new DataSchema( - "test-json", - DEFAULT_TIMESTAMP_SPEC, - new DimensionsSpec( - ImmutableList.of( - new StringDimensionSchema("ts"), - new StringDimensionSchema("dim"), - new LongDimensionSchema("valDim") - ) - ), - new AggregatorFactory[]{new LongSumAggregatorFactory("valMet", "val")}, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(Intervals.of("2014/P1D")) - ), - null - ), + DATA_SCHEMA, new IndexIOConfig( new LocalInputSource(tmpDir, "druid*"), DEFAULT_INPUT_FORMAT, @@ -337,24 +322,7 @@ public void testIngestNullOnlyColumns_storeEmptyColumnsOff_shouldNotStoreEmptyCo IndexTask indexTask = createIndexTask( new IndexIngestionSpec( - new DataSchema( - "test-json", - DEFAULT_TIMESTAMP_SPEC, - new DimensionsSpec( - ImmutableList.of( - new StringDimensionSchema("ts"), - new StringDimensionSchema("dim"), - new LongDimensionSchema("valDim") - ) - ), - new AggregatorFactory[]{new LongSumAggregatorFactory("valMet", "val")}, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(Intervals.of("2014/P1D")) - ), - null - ), + DATA_SCHEMA, new IndexIOConfig( new LocalInputSource(tmpDir, "druid*"), DEFAULT_INPUT_FORMAT, @@ -2698,20 +2666,20 @@ private static IndexIngestionSpec createIngestionSpec( if (inputFormat != null) { Preconditions.checkArgument(parseSpec == null, "Can't use parseSpec"); return new IndexIngestionSpec( - new DataSchema( - DATASOURCE, - Preconditions.checkNotNull(timestampSpec, "timestampSpec"), - Preconditions.checkNotNull(dimensionsSpec, "dimensionsSpec"), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - granularitySpec != null ? granularitySpec : new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(Intervals.of("2014/2015")) - ), - transformSpec - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(Preconditions.checkNotNull(timestampSpec, "timestampSpec")) + .withDimensions(Preconditions.checkNotNull(dimensionsSpec, "dimensionsSpec")) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity( + granularitySpec != null ? granularitySpec : new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + Collections.singletonList(Intervals.of("2014/2015")) + ) + ) + .withTransform(transformSpec) + .build(), new IndexIOConfig( new LocalInputSource(baseDir, "druid*"), inputFormat, @@ -2723,22 +2691,21 @@ private static IndexIngestionSpec createIngestionSpec( } else { parseSpec = parseSpec != null ? parseSpec : DEFAULT_PARSE_SPEC; return new IndexIngestionSpec( - new DataSchema( - DATASOURCE, - parseSpec.getTimestampSpec(), - parseSpec.getDimensionsSpec(), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - granularitySpec != null ? granularitySpec : new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(Intervals.of("2014/2015")) - ), - transformSpec, - null, - objectMapper - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(parseSpec.getTimestampSpec()) + .withDimensions(parseSpec.getDimensionsSpec()) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity( + granularitySpec != null ? granularitySpec : new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + Collections.singletonList(Intervals.of("2014/2015")) + ) + ) + .withTransform(transformSpec) + .withObjectMapper(objectMapper) + .build(), new IndexIOConfig( new LocalInputSource(baseDir, "druid*"), createInputFormatFromParseSpec(parseSpec), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskSerdeTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskSerdeTest.java index e6ea0e1329ae..99b0f8e7a759 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskSerdeTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskSerdeTest.java @@ -38,7 +38,6 @@ import org.apache.druid.indexing.common.task.batch.parallel.ParallelIndexTuningConfig; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.indexing.DataSchema; @@ -220,18 +219,19 @@ public void testIndexTaskSerde() throws Exception null, null, new IndexIngestionSpec( - new DataSchema( - "foo", - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, - new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P2D")) - ), - null - ), + DataSchema.builder() + .withDataSource("foo") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.EMPTY) + .withAggregators(new DoubleSumAggregatorFactory("met", "met")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P2D")) + ) + ) + .build(), new IndexIOConfig(new LocalInputSource(new File("lol"), "rofl"), new NoopInputFormat(), true, false), TuningConfigBuilder.forIndexTask() .withMaxRowsInMemory(10) @@ -288,18 +288,19 @@ public void testIndexTaskwithResourceSerde() throws Exception null, new TaskResource("rofl", 2), new IndexIngestionSpec( - new DataSchema( - "foo", - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, - new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P2D")) - ), - null - ), + DataSchema.builder() + .withDataSource("foo") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.EMPTY) + .withAggregators(new DoubleSumAggregatorFactory("met", "met")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P2D")) + ) + ) + .build(), new IndexIOConfig(new LocalInputSource(new File("lol"), "rofl"), new NoopInputFormat(), true, false), TuningConfigBuilder.forIndexTask() .withMaxRowsInMemory(10) @@ -412,15 +413,19 @@ public void testHadoopIndexTaskSerde() throws Exception final HadoopIndexTask task = new HadoopIndexTask( null, new HadoopIngestionSpec( - new DataSchema( - "foo", null, new AggregatorFactory[0], new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P1D")) - ), - null, - jsonMapper - ), new HadoopIOConfig(ImmutableMap.of("paths", "bar"), null, null), null + DataSchema.builder() + .withDataSource("foo") + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P1D")) + ) + ) + .withObjectMapper(jsonMapper) + .build(), + new HadoopIOConfig(ImmutableMap.of("paths", "bar"), null, null), + null ), null, null, @@ -454,19 +459,18 @@ public void testHadoopIndexTaskWithContextSerde() throws Exception final HadoopIndexTask task = new HadoopIndexTask( null, new HadoopIngestionSpec( - new DataSchema( - "foo", - null, - null, - new AggregatorFactory[0], - new UniformGranularitySpec( - Granularities.DAY, - null, ImmutableList.of(Intervals.of("2010-01-01/P1D")) - ), - null, - null, - jsonMapper - ), new HadoopIOConfig(ImmutableMap.of("paths", "bar"), null, null), null + DataSchema.builder() + .withDataSource("foo") + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, ImmutableList.of(Intervals.of("2010-01-01/P1D")) + ) + ) + .withObjectMapper(jsonMapper) + .build(), + new HadoopIOConfig(ImmutableMap.of("paths", "bar"), null, null), + null ), null, null, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java index 44b8284f407d..6caab0a0652e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java @@ -221,14 +221,13 @@ protected ParallelIndexSupervisorTask createTask( dropExisting ); ingestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - DATASOURCE, - timestampSpec, - dimensionsSpec, - DEFAULT_METRICS_SPEC, - granularitySpec, - null - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(timestampSpec) + .withDimensions(dimensionsSpec) + .withAggregators(DEFAULT_METRICS_SPEC) + .withGranularity(granularitySpec) + .build(), ioConfig, tuningConfig ); @@ -241,14 +240,13 @@ protected ParallelIndexSupervisorTask createTask( dropExisting ); ingestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - DATASOURCE, - parseSpec.getTimestampSpec(), - parseSpec.getDimensionsSpec(), - DEFAULT_METRICS_SPEC, - granularitySpec, - null - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(parseSpec.getTimestampSpec()) + .withDimensions(parseSpec.getDimensionsSpec()) + .withAggregators(DEFAULT_METRICS_SPEC) + .withGranularity(granularitySpec) + .build(), ioConfig, tuningConfig ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionTaskKillTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionTaskKillTest.java index b8c59d042a31..a21dbb84616b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionTaskKillTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionTaskKillTest.java @@ -41,7 +41,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Comparators; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.GranularitySpec; @@ -250,14 +249,13 @@ private ParallelIndexSupervisorTask createTestTask( null ); ingestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - DATASOURCE, - timestampSpec, - dimensionsSpec, - new AggregatorFactory[]{new LongSumAggregatorFactory("val", "val")}, - granularitySpec, - null - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(timestampSpec) + .withDimensions(dimensionsSpec) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity(granularitySpec) + .build(), ioConfig, tuningConfig ); @@ -271,16 +269,13 @@ private ParallelIndexSupervisorTask createTestTask( ); //noinspection unchecked ingestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - parseSpec.getTimestampSpec(), - parseSpec.getDimensionsSpec(), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - granularitySpec, - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(parseSpec.getTimestampSpec()) + .withDimensions(parseSpec.getDimensionsSpec()) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity(granularitySpec) + .build(), ioConfig, tuningConfig ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/MultiPhaseParallelIndexingWithNullColumnTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/MultiPhaseParallelIndexingWithNullColumnTest.java index 0d19cd86e03c..3adc154bb226 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/MultiPhaseParallelIndexingWithNullColumnTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/MultiPhaseParallelIndexingWithNullColumnTest.java @@ -127,18 +127,19 @@ public void testIngestNullColumn() throws JsonProcessingException null, null, new ParallelIndexIngestionSpec( - new DataSchema( - DATASOURCE, - TIMESTAMP_SPEC, - DIMENSIONS_SPEC.withDimensions(dimensionSchemas), - DEFAULT_METRICS_SPEC, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - INTERVAL_TO_INDEX - ), - null - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions(DEFAULT_DIMENSIONS_SPEC.withDimensions(dimensionSchemas)) + .withAggregators(DEFAULT_METRICS_SPEC) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + INTERVAL_TO_INDEX + ) + ) + .build(), new ParallelIndexIOConfig( getInputSource(), JSON_FORMAT, @@ -177,18 +178,21 @@ public void testIngestNullColumn_useFieldDiscovery_includeAllDimensions_shouldSt null, null, new ParallelIndexIngestionSpec( - new DataSchema( - DATASOURCE, - TIMESTAMP_SPEC, - new DimensionsSpec.Builder().setDimensions(dimensionSchemas).setIncludeAllDimensions(true).build(), - DEFAULT_METRICS_SPEC, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - INTERVAL_TO_INDEX - ), - null - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(TIMESTAMP_SPEC) + .withDimensions( + DimensionsSpec.builder().setDimensions(dimensionSchemas).setIncludeAllDimensions(true).build() + ) + .withAggregators(DEFAULT_METRICS_SPEC) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + INTERVAL_TO_INDEX + ) + ) + .build(), new ParallelIndexIOConfig( getInputSource(), new JsonInputFormat( @@ -237,18 +241,21 @@ public void testIngestNullColumn_explicitPathSpec_useFieldDiscovery_includeAllDi null, null, new ParallelIndexIngestionSpec( - new DataSchema( - DATASOURCE, - TIMESTAMP_SPEC, - new DimensionsSpec.Builder().setIncludeAllDimensions(true).build(), - DEFAULT_METRICS_SPEC, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - null - ), - null - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(TIMESTAMP_SPEC) + .withDimensions( + DimensionsSpec.builder().setIncludeAllDimensions(true).build() + ) + .withAggregators(DEFAULT_METRICS_SPEC) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + null + ) + ) + .build(), new ParallelIndexIOConfig( getInputSource(), new JsonInputFormat( @@ -303,20 +310,23 @@ public void testIngestNullColumn_storeEmptyColumnsOff_shouldNotStoreEmptyColumns null, null, new ParallelIndexIngestionSpec( - new DataSchema( - DATASOURCE, - TIMESTAMP_SPEC, - DIMENSIONS_SPEC.withDimensions( - DimensionsSpec.getDefaultSchemas(Arrays.asList("ts", "unknownDim")) - ), - DEFAULT_METRICS_SPEC, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - INTERVAL_TO_INDEX - ), - null - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(TIMESTAMP_SPEC) + .withDimensions( + DIMENSIONS_SPEC.withDimensions( + DimensionsSpec.getDefaultSchemas(Arrays.asList("ts", "unknownDim")) + ) + ) + .withAggregators(DEFAULT_METRICS_SPEC) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + INTERVAL_TO_INDEX + ) + ) + .build(), new ParallelIndexIOConfig( getInputSource(), JSON_FORMAT, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskKillTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskKillTest.java index cbf711469734..2ea9385a0145 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskKillTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskKillTest.java @@ -36,7 +36,6 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; @@ -147,20 +146,19 @@ private ParallelIndexSupervisorTask newTask( final int numTotalSubTasks = inputSource.estimateNumSplits(new NoopInputFormat(), null); // set up ingestion spec final ParallelIndexIngestionSpec ingestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - DEFAULT_TIMESTAMP_SPEC, - DEFAULT_DIMENSIONS_SPEC, - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - interval == null ? null : Collections.singletonList(interval) - ), - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions(DEFAULT_DIMENSIONS_SPEC) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + interval == null ? null : Collections.singletonList(interval) + ) + ) + .build(), ioConfig, TuningConfigBuilder.forParallelIndexTask().withMaxNumConcurrentSubTasks(numTotalSubTasks).build() ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskResourceTest.java index c9858b80847b..4587ef6ce7e3 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskResourceTest.java @@ -45,7 +45,6 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; @@ -400,20 +399,19 @@ private TestSupervisorTask newTask( ) { final ParallelIndexIngestionSpec ingestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - DEFAULT_TIMESTAMP_SPEC, - DEFAULT_DIMENSIONS_SPEC, - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - interval == null ? null : Collections.singletonList(interval) - ), - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions(DEFAULT_DIMENSIONS_SPEC) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + interval == null ? null : Collections.singletonList(interval) + ) + ) + .build(), ioConfig, TuningConfigBuilder.forParallelIndexTask().withMaxNumConcurrentSubTasks(NUM_SUB_TASKS).build() ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskSerdeTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskSerdeTest.java index e11fb2ef001c..fe5188fec6c5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskSerdeTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskSerdeTest.java @@ -34,7 +34,6 @@ import org.apache.druid.indexing.common.task.TuningConfigBuilder; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; @@ -220,16 +219,19 @@ ParallelIndexIngestionSpecBuilder partitionsSpec(PartitionsSpec partitionsSpec) ParallelIndexIngestionSpec build() { - DataSchema dataSchema = new DataSchema( - "dataSource", - TIMESTAMP_SPEC, - DIMENSIONS_SPEC, - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.MINUTE, inputIntervals), - null - ); + DataSchema dataSchema = DataSchema.builder() + .withDataSource("datasource") + .withTimestamp(TIMESTAMP_SPEC) + .withDimensions(DIMENSIONS_SPEC) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + inputIntervals + ) + ) + .build(); ParallelIndexTuningConfig tuningConfig = TuningConfigBuilder .forParallelIndexTask() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java index 57dbafa173f9..b908d274e6c3 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java @@ -263,14 +263,11 @@ public void testFailToConstructWhenBothAppendToExistingAndForceGuaranteedRollupA .withLogParseExceptions(false) .build(); final ParallelIndexIngestionSpec indexIngestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - "datasource", - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - null, - null, - null - ), + DataSchema.builder() + .withDataSource("datasource") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.EMPTY) + .build(), ioConfig, tuningConfig ); @@ -325,25 +322,24 @@ public void testFailToConstructWhenBothInputSourceAndParserAreSet() expectedException.expect(IAE.class); expectedException.expectMessage("Cannot use parser and inputSource together. Try using inputFormat instead of parser."); new ParallelIndexIngestionSpec( - new DataSchema( - "datasource", - mapper.convertValue( - new StringInputRowParser( - new JSONParseSpec( - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - null, - null, - null - ) - ), - Map.class - ), - null, - null, - null, - mapper - ), + DataSchema.builder() + .withDataSource("datasource") + .withParserMap( + mapper.convertValue( + new StringInputRowParser( + new JSONParseSpec( + new TimestampSpec(null, null, null), + DimensionsSpec.EMPTY, + null, + null, + null + ) + ), + Map.class + ) + ) + .withObjectMapper(mapper) + .build(), ioConfig, tuningConfig ); @@ -559,14 +555,11 @@ public void testCompactionTaskDoesntCleanup() throws Exception .build(); final ParallelIndexIngestionSpec indexIngestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - "datasource", - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - null, - null, - null - ), + DataSchema.builder() + .withDataSource("datasource") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.EMPTY) + .build(), ioConfig, tuningConfig ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTestingFactory.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTestingFactory.java index 2455ce692b95..f93ea8d0a8b8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTestingFactory.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTestingFactory.java @@ -31,11 +31,9 @@ import org.apache.druid.indexing.common.task.TaskResource; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import org.apache.druid.segment.indexing.granularity.GranularitySpec; -import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.timeline.partition.BuildingHashBasedNumberedShardSpec; import org.apache.druid.timeline.partition.HashPartitionFunction; import org.joda.time.Interval; @@ -97,16 +95,13 @@ static DataSchema createDataSchema(List granularitySpecInputIntervals) DimensionsSpec.getDefaultSchemas(ImmutableList.of(SCHEMA_DIMENSION)) ); - return new DataSchema( - DATASOURCE, - timestampSpec, - dimensionsSpec, - new AggregatorFactory[]{}, - granularitySpec, - TransformSpec.NONE, - null, - NESTED_OBJECT_MAPPER - ); + return DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(timestampSpec) + .withDimensions(dimensionsSpec) + .withGranularity(granularitySpec) + .withObjectMapper(NESTED_OBJECT_MAPPER) + .build(); } static ParallelIndexIngestionSpec createIngestionSpec( diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java index 96494b8ac794..814e3f646424 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java @@ -39,7 +39,6 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.GranularitySpec; @@ -329,14 +328,13 @@ protected ParallelIndexSupervisorTask newTask( null ); ingestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - DATASOURCE, - timestampSpec, - dimensionsSpec, - new AggregatorFactory[]{new LongSumAggregatorFactory("val", "val")}, - granularitySpec, - null - ), + DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(timestampSpec) + .withDimensions(dimensionsSpec) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity(granularitySpec) + .build(), ioConfig, tuningConfig ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseParallelIndexingTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseParallelIndexingTest.java index ac8371acaa0d..b51224908644 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseParallelIndexingTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseParallelIndexingTest.java @@ -43,7 +43,6 @@ import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.parsers.JSONPathSpec; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.DataSegmentsWithSchemas; @@ -391,20 +390,19 @@ public void testRunInParallelIngestNullColumn() null, null, new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - DEFAULT_TIMESTAMP_SPEC, - DEFAULT_DIMENSIONS_SPEC.withDimensions(dimensionSchemas), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(Intervals.of("2017-12/P1M")) - ), - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions(DEFAULT_DIMENSIONS_SPEC.withDimensions(dimensionSchemas)) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + Collections.singletonList(Intervals.of("2017-12/P1M")) + ) + ) + .build(), new ParallelIndexIOConfig( new SettableSplittableLocalInputSource(inputDir, VALID_INPUT_SOURCE_FILTER, true), DEFAULT_INPUT_FORMAT, @@ -444,20 +442,19 @@ public void testRunInParallelIngestNullColumn_storeEmptyColumnsOff_shouldNotStor null, null, new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - DEFAULT_TIMESTAMP_SPEC, - DEFAULT_DIMENSIONS_SPEC.withDimensions(dimensionSchemas), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(Intervals.of("2017-12/P1M")) - ), - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions(DEFAULT_DIMENSIONS_SPEC.withDimensions(dimensionSchemas)) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + Collections.singletonList(Intervals.of("2017-12/P1M")) + ) + ) + .build(), new ParallelIndexIOConfig( new SettableSplittableLocalInputSource(inputDir, VALID_INPUT_SOURCE_FILTER, true), DEFAULT_INPUT_FORMAT, @@ -785,21 +782,24 @@ public void testIngestBothExplicitAndImplicitDims() throws IOException null, null, new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - DEFAULT_TIMESTAMP_SPEC, - DimensionsSpec.builder() - .setDefaultSchemaDimensions(ImmutableList.of("ts", "explicitDim")) - .setIncludeAllDimensions(true) - .build(), - new AggregatorFactory[]{new CountAggregatorFactory("cnt")}, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(interval) - ), - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions( + DimensionsSpec.builder() + .setDefaultSchemaDimensions(ImmutableList.of("ts", "explicitDim")) + .setIncludeAllDimensions(true) + .build() + ) + .withAggregators(new CountAggregatorFactory("cnt")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + Collections.singletonList(interval) + ) + ) + .build(), new ParallelIndexIOConfig( new SettableSplittableLocalInputSource(inputDir, "*.json", true), new JsonInputFormat( @@ -868,21 +868,24 @@ public void testIngestBothExplicitAndImplicitDimsSchemaDiscovery() throws IOExce null, null, new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - DEFAULT_TIMESTAMP_SPEC, - DimensionsSpec.builder() - .setDefaultSchemaDimensions(ImmutableList.of("ts", "explicitDim")) - .useSchemaDiscovery(true) - .build(), - new AggregatorFactory[]{new CountAggregatorFactory("cnt")}, - new UniformGranularitySpec( - Granularities.DAY, - Granularities.MINUTE, - Collections.singletonList(interval) - ), - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions( + DimensionsSpec.builder() + .setDefaultSchemaDimensions(ImmutableList.of("ts", "explicitDim")) + .useSchemaDiscovery(true) + .build() + ) + .withAggregators(new CountAggregatorFactory("cnt")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + Granularities.MINUTE, + Collections.singletonList(interval) + ) + ) + .build(), new ParallelIndexIOConfig( new SettableSplittableLocalInputSource(inputDir, "*.json", true), new JsonInputFormat( @@ -948,20 +951,19 @@ private ParallelIndexSupervisorTask newTask( final ParallelIndexIngestionSpec ingestionSpec; if (useInputFormatApi) { ingestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - DEFAULT_TIMESTAMP_SPEC, - DEFAULT_DIMENSIONS_SPEC, - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - }, - new UniformGranularitySpec( - segmentGranularity, - Granularities.MINUTE, - interval == null ? null : Collections.singletonList(interval) - ), - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions(DEFAULT_DIMENSIONS_SPEC) + .withAggregators(new LongSumAggregatorFactory("val", "val")) + .withGranularity( + new UniformGranularitySpec( + segmentGranularity, + Granularities.MINUTE, + interval == null ? null : Collections.singletonList(interval) + ) + ) + .build(), new ParallelIndexIOConfig( new SettableSplittableLocalInputSource(inputDir, inputSourceFilter, splittableInputSource), DEFAULT_INPUT_FORMAT, @@ -972,18 +974,19 @@ private ParallelIndexSupervisorTask newTask( ); } else { ingestionSpec = new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - DEFAULT_TIMESTAMP_SPEC, - DEFAULT_DIMENSIONS_SPEC, - DEFAULT_METRICS_SPEC, - new UniformGranularitySpec( - segmentGranularity, - Granularities.MINUTE, - interval == null ? null : Collections.singletonList(interval) - ), - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(DEFAULT_TIMESTAMP_SPEC) + .withDimensions(DEFAULT_DIMENSIONS_SPEC) + .withAggregators(DEFAULT_METRICS_SPEC) + .withGranularity( + new UniformGranularitySpec( + segmentGranularity, + Granularities.MINUTE, + interval == null ? null : Collections.singletonList(interval) + ) + ) + .build(), new ParallelIndexIOConfig( new LocalInputSource(inputDir, inputSourceFilter), createInputFormatFromParseSpec(DEFAULT_PARSE_SPEC), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseSubTaskSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseSubTaskSpecTest.java index 4acc3d3f5912..577dce1255d0 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseSubTaskSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/SinglePhaseSubTaskSpecTest.java @@ -26,7 +26,6 @@ import org.apache.druid.data.input.impl.LocalInputSource; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.indexing.common.TestUtils; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.Resource; @@ -48,14 +47,11 @@ public class SinglePhaseSubTaskSpecTest "groupId", "supervisorTaskId", new ParallelIndexIngestionSpec( - new DataSchema( - "dataSource", - new TimestampSpec(null, null, null), - new DimensionsSpec(null), - new AggregatorFactory[0], - null, - null - ), + DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.builder().build()) + .build(), new ParallelIndexIOConfig( new LocalInputSource(new File("baseDir"), "filter"), new JsonInputFormat(null, null, null, null, null), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TombstoneHelperTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TombstoneHelperTest.java index aea98e9e1036..fcb92543914a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TombstoneHelperTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/TombstoneHelperTest.java @@ -67,8 +67,7 @@ public void noTombstonesWhenNoDataInInputIntervalAndNoExistingSegments() throws GranularitySpec granularitySpec = new UniformGranularitySpec(Granularities.DAY, null, false, Collections.singletonList(interval) ); - DataSchema dataSchema = - new DataSchema("test", null, null, null, granularitySpec, null); + DataSchema dataSchema = DataSchema.builder().withDataSource("test").withGranularity(granularitySpec).build(); // no segments will be pushed when all rows are thrown away, assume that: List pushedSegments = Collections.emptyList(); @@ -93,8 +92,7 @@ public void tombstonesCreatedWhenNoDataInInputIntervalAndExistingSegments() thro GranularitySpec granularitySpec = new UniformGranularitySpec(Granularities.DAY, null, false, Collections.singletonList(interval) ); - DataSchema dataSchema = - new DataSchema("test", null, null, null, granularitySpec, null); + DataSchema dataSchema = DataSchema.builder().withDataSource("test").withGranularity(granularitySpec).build(); // no segments will be pushed when all rows are thrown away, assume that: List pushedSegments = Collections.emptyList(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/input/InputRowSchemasTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/input/InputRowSchemasTest.java index 44850ad0558f..35aeef9715d3 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/input/InputRowSchemasTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/input/InputRowSchemasTest.java @@ -122,17 +122,19 @@ public void testFromDataSchema() new DoubleDimensionSchema("d5") ) ); - DataSchema schema = new DataSchema( - "dataSourceName", - new TimestampSpec(null, null, null), - dimensionsSpec, - new AggregatorFactory[]{ - new CountAggregatorFactory("count"), - new LongSumAggregatorFactory("met", "met") - }, - new UniformGranularitySpec(Granularities.MINUTE, Granularities.NONE, null), - null - ); + DataSchema schema = + DataSchema.builder() + .withDataSource("dataSourceName") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(dimensionsSpec) + .withAggregators( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("met", "met") + ) + .withGranularity( + new UniformGranularitySpec(Granularities.MINUTE, Granularities.NONE, null) + ) + .build(); InputRowSchema inputRowSchema = InputRowSchemas.fromDataSchema(schema); Assert.assertEquals(timestampSpec, inputRowSchema.getTimestampSpec()); @@ -154,14 +156,12 @@ public void testFromDataSchemaWithNoAggregator() new DoubleDimensionSchema("d5") ) ); - DataSchema schema = new DataSchema( - "dataSourceName", - new TimestampSpec(null, null, null), - dimensionsSpec, - new AggregatorFactory[]{}, - new UniformGranularitySpec(Granularities.MINUTE, Granularities.NONE, null), - null - ); + DataSchema schema = DataSchema.builder() + .withDataSource("dataSourceName") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(dimensionsSpec) + .withGranularity(new UniformGranularitySpec(Granularities.MINUTE, Granularities.NONE, null)) + .build(); InputRowSchema inputRowSchema = InputRowSchemas.fromDataSchema(schema); Assert.assertEquals(timestampSpec, inputRowSchema.getTimestampSpec()); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java index ddbed6be7c74..522134c4556f 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java @@ -109,7 +109,6 @@ import org.apache.druid.query.ForwardingQueryProcessingPool; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.SegmentDescriptor; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.IndexMergerV9Factory; @@ -672,18 +671,19 @@ public void testIndexTask() null, null, new IndexIngestionSpec( - new DataSchema( - "foo", - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, - new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P2D")) - ), - null - ), + DataSchema.builder() + .withDataSource("foo") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.EMPTY) + .withAggregators(new DoubleSumAggregatorFactory("met", "met")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P2D")) + ) + ) + .build(), new IndexIOConfig(new MockInputSource(), new NoopInputFormat(), false, false), TuningConfigBuilder.forIndexTask() .withMaxRowsPerSegment(10000) @@ -735,18 +735,18 @@ public void testIndexTaskFailure() null, null, new IndexIngestionSpec( - new DataSchema( - "foo", - null, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, - new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P1D")) - ), - null, - mapper - ), + DataSchema.builder() + .withDataSource("foo") + .withAggregators(new DoubleSumAggregatorFactory("met", "met")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P1D")) + ) + ) + .withObjectMapper(mapper) + .build(), new IndexIOConfig(new MockExceptionInputSource(), new NoopInputFormat(), false, false), TuningConfigBuilder.forIndexTask() .withMaxRowsPerSegment(10000) @@ -1165,18 +1165,19 @@ public void testResumeTasks() throws Exception null, null, new IndexIngestionSpec( - new DataSchema( - "foo", - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, - new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P2D")) - ), - null - ), + DataSchema.builder() + .withDataSource("foo") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.EMPTY) + .withAggregators(new DoubleSumAggregatorFactory("met", "met")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P2D")) + ) + ) + .build(), new IndexIOConfig(new MockInputSource(), new NoopInputFormat(), false, false), TuningConfigBuilder.forIndexTask() .withMaxRowsPerSegment(10000) @@ -1253,18 +1254,19 @@ public void testUnifiedAppenderatorsManagerCleanup() throws Exception null, null, new IndexIngestionSpec( - new DataSchema( - "foo", - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - new AggregatorFactory[]{new DoubleSumAggregatorFactory("met", "met")}, - new UniformGranularitySpec( - Granularities.DAY, - null, - ImmutableList.of(Intervals.of("2010-01-01/P2D")) - ), - null - ), + DataSchema.builder() + .withDataSource("foo") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.EMPTY) + .withAggregators(new DoubleSumAggregatorFactory("met", "met")) + .withGranularity( + new UniformGranularitySpec( + Granularities.DAY, + null, + ImmutableList.of(Intervals.of("2010-01-01/P2D")) + ) + ) + .build(), new IndexIOConfig(new MockInputSource(), new NoopInputFormat(), false, false), TuningConfigBuilder.forIndexTask() .withMaxRowsPerSegment(10000) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java index c7b7b13ef7ea..8f1393f2c675 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java @@ -549,14 +549,15 @@ public void testGetActiveTaskRedactsPassword() throws JsonProcessingException new NoopTaskContextEnricher() ); - final DataSchema dataSchema = new DataSchema( - "DS", - new TimestampSpec(null, null, null), - new DimensionsSpec(null), - null, - new UniformGranularitySpec(Granularities.YEAR, Granularities.DAY, null), - null - ); + final DataSchema dataSchema = + DataSchema.builder() + .withDataSource("DS") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.builder().build()) + .withGranularity( + new UniformGranularitySpec(Granularities.YEAR, Granularities.DAY, null) + ) + .build(); final ParallelIndexIOConfig ioConfig = new ParallelIndexIOConfig( new HttpInputSource(Collections.singletonList(URI.create("http://host.org")), "user", diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/CsvInputSourceSamplerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/CsvInputSourceSamplerTest.java index e788545507cd..1730d4b638fb 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/CsvInputSourceSamplerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/CsvInputSourceSamplerTest.java @@ -47,14 +47,11 @@ public void testCSVColumnAllNull() { final TimestampSpec timestampSpec = new TimestampSpec(null, null, DateTimes.of("1970")); final DimensionsSpec dimensionsSpec = new DimensionsSpec(null); - final DataSchema dataSchema = new DataSchema( - "sampler", - timestampSpec, - dimensionsSpec, - null, - null, - null - ); + final DataSchema dataSchema = DataSchema.builder() + .withDataSource("sampler") + .withTimestamp(timestampSpec) + .withDimensions(dimensionsSpec) + .build(); final List strCsvRows = ImmutableList.of( "FirstName,LastName,Number,Gender", diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java index c486c15f0f23..0220aacd8922 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java @@ -66,14 +66,11 @@ public void testDiscoveredTypesNonStrictBooleans() final SamplerResponse response = inputSourceSampler.sample( inputSource, new JsonInputFormat(null, null, null, null, null), - new DataSchema( - "test", - new TimestampSpec("t", null, null), - DimensionsSpec.builder().useSchemaDiscovery(true).build(), - null, - null, - null - ), + DataSchema.builder() + .withDataSource("test") + .withTimestamp(new TimestampSpec("t", null, null)) + .withDimensions(DimensionsSpec.builder().useSchemaDiscovery(true).build()) + .build(), null ); @@ -131,14 +128,11 @@ public void testDiscoveredTypesStrictBooleans() final SamplerResponse response = inputSourceSampler.sample( inputSource, new JsonInputFormat(null, null, null, null, null), - new DataSchema( - "test", - new TimestampSpec("t", null, null), - DimensionsSpec.builder().useSchemaDiscovery(true).build(), - null, - null, - null - ), + DataSchema.builder() + .withDataSource("test") + .withTimestamp(new TimestampSpec("t", null, null)) + .withDimensions(DimensionsSpec.builder().useSchemaDiscovery(true).build()) + .build(), null ); @@ -189,14 +183,12 @@ public void testDiscoveredTypesStrictBooleans() public void testTypesClassicDiscovery() { final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n')); - final DataSchema dataSchema = new DataSchema( - "test", - new TimestampSpec("t", null, null), - DimensionsSpec.builder().build(), - null, - null, - null - ); + final DataSchema dataSchema = + DataSchema.builder() + .withDataSource("test") + .withTimestamp(new TimestampSpec("t", null, null)) + .withDimensions(DimensionsSpec.builder().build()) + .build(); final SamplerResponse response = inputSourceSampler.sample( inputSource, new JsonInputFormat(null, null, null, null, null), @@ -248,23 +240,20 @@ public void testTypesClassicDiscovery() public void testTypesNoDiscoveryExplicitSchema() { final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n')); - final DataSchema dataSchema = new DataSchema( - "test", - new TimestampSpec("t", null, null), - DimensionsSpec.builder().setDimensions( - ImmutableList.of(new StringDimensionSchema("string"), - new LongDimensionSchema("long"), - new DoubleDimensionSchema("double"), - new StringDimensionSchema("bool"), - new AutoTypeColumnSchema("variant", null), - new AutoTypeColumnSchema("array", null), - new AutoTypeColumnSchema("nested", null) - ) - ).build(), - null, - null, - null - ); + final DataSchema dataSchema = + DataSchema.builder() + .withDataSource("test") + .withTimestamp(new TimestampSpec("t", null, null)) + .withDimensions( + new StringDimensionSchema("string"), + new LongDimensionSchema("long"), + new DoubleDimensionSchema("double"), + new StringDimensionSchema("bool"), + new AutoTypeColumnSchema("variant", null), + new AutoTypeColumnSchema("array", null), + new AutoTypeColumnSchema("nested", null) + ) + .build(); final SamplerResponse response = inputSourceSampler.sample( inputSource, new JsonInputFormat(null, null, null, null, null), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java index 087b12cef40e..80d88e0be17c 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java @@ -1497,24 +1497,24 @@ private DataSchema createDataSchema( ) throws IOException { if (useInputFormatApi) { - return new DataSchema( - "sampler", - timestampSpec, - dimensionsSpec, - aggregators, - granularitySpec, - transformSpec - ); + return DataSchema.builder() + .withDataSource("sampler") + .withTimestamp(timestampSpec) + .withDimensions(dimensionsSpec) + .withAggregators(aggregators) + .withGranularity(granularitySpec) + .withTransform(transformSpec) + .build(); } else { final Map parserMap = getParserMap(createInputRowParser(timestampSpec, dimensionsSpec)); - return new DataSchema( - "sampler", - parserMap, - aggregators, - granularitySpec, - transformSpec, - OBJECT_MAPPER - ); + return DataSchema.builder() + .withDataSource("sampler") + .withParserMap(parserMap) + .withAggregators(aggregators) + .withGranularity(granularitySpec) + .withTransform(transformSpec) + .withObjectMapper(OBJECT_MAPPER) + .build(); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunnerAuthTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunnerAuthTest.java index 0f280059e0a5..7f44d44a00db 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunnerAuthTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunnerAuthTest.java @@ -33,10 +33,8 @@ import org.apache.druid.indexing.seekablestream.common.StreamPartition; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.granularity.AllGranularity; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; -import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.AuthConfig; @@ -107,16 +105,13 @@ public Authorizer getAuthorizer(String name) } }; - DataSchema dataSchema = new DataSchema( - "datasource", - new TimestampSpec(null, null, null), - new DimensionsSpec(Collections.emptyList()), - new AggregatorFactory[]{}, - new ArbitraryGranularitySpec(new AllGranularity(), Collections.emptyList()), - TransformSpec.NONE, - null, - null - ); + DataSchema dataSchema = + DataSchema.builder() + .withDataSource("datasource") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(new DimensionsSpec(Collections.emptyList())) + .withGranularity(new ArbitraryGranularitySpec(new AllGranularity(), Collections.emptyList())) + .build(); SeekableStreamIndexTaskTuningConfig tuningConfig = mock(SeekableStreamIndexTaskTuningConfig.class); SeekableStreamIndexTaskIOConfig ioConfig = new TestSeekableStreamIndexTaskIOConfig(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java index 258ebff7b504..7346fe9fb6e1 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java @@ -95,7 +95,6 @@ import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.Result; import org.apache.druid.query.SegmentDescriptor; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; @@ -164,25 +163,23 @@ public abstract class SeekableStreamIndexTaskTestBase extends EasyMockSupport protected static final ObjectMapper OBJECT_MAPPER; protected static final DataSchema OLD_DATA_SCHEMA; - protected static final DataSchema NEW_DATA_SCHEMA = new DataSchema( - "test_ds", - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec( - Arrays.asList( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim1t"), - new StringDimensionSchema("dim2"), - new LongDimensionSchema("dimLong"), - new FloatDimensionSchema("dimFloat") - ) - ), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null - ); + protected static final DataSchema NEW_DATA_SCHEMA = + DataSchema.builder() + .withDataSource("test_ds") + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions( + new StringDimensionSchema("dim1"), + new StringDimensionSchema("dim1t"), + new StringDimensionSchema("dim2"), + new LongDimensionSchema("dimLong"), + new FloatDimensionSchema("dimFloat") + ) + .withAggregators( + new DoubleSumAggregatorFactory("met1sum", "met1"), + new CountAggregatorFactory("rows") + ) + .withGranularity(new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null)) + .build(); protected static final InputFormat INPUT_FORMAT = new JsonInputFormat( new JSONPathSpec(true, ImmutableList.of()), ImmutableMap.of(), @@ -211,37 +208,38 @@ public abstract class SeekableStreamIndexTaskTestBase extends EasyMockSupport static { OBJECT_MAPPER = new TestUtils().getTestObjectMapper(); OBJECT_MAPPER.registerSubtypes(new NamedType(JSONParseSpec.class, "json")); - OLD_DATA_SCHEMA = new DataSchema( - "test_ds", - OBJECT_MAPPER.convertValue( - new StringInputRowParser( - new JSONParseSpec( - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec( - Arrays.asList( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim1t"), - new StringDimensionSchema("dim2"), - new LongDimensionSchema("dimLong"), - new FloatDimensionSchema("dimFloat") - ) - ), - new JSONPathSpec(true, ImmutableList.of()), - ImmutableMap.of(), - false - ), - StandardCharsets.UTF_8.name() - ), - Map.class - ), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null, - OBJECT_MAPPER - ); + OLD_DATA_SCHEMA = DataSchema.builder() + .withDataSource("test_ds") + .withParserMap( + OBJECT_MAPPER.convertValue( + new StringInputRowParser( + new JSONParseSpec( + new TimestampSpec("timestamp", "iso", null), + new DimensionsSpec( + Arrays.asList( + new StringDimensionSchema("dim1"), + new StringDimensionSchema("dim1t"), + new StringDimensionSchema("dim2"), + new LongDimensionSchema("dimLong"), + new FloatDimensionSchema("dimFloat") + ) + ), + new JSONPathSpec(true, ImmutableList.of()), + ImmutableMap.of(), + false + ), + StandardCharsets.UTF_8.name() + ), + Map.class + ) + ) + .withAggregators( + new DoubleSumAggregatorFactory("met1sum", "met1"), + new CountAggregatorFactory("rows") + ) + .withGranularity(new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null)) + .withObjectMapper(OBJECT_MAPPER) + .build(); } public SeekableStreamIndexTaskTestBase( diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSamplerSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSamplerSpecTest.java index 87cd196c268f..6510e2cfbdc5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSamplerSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSamplerSpecTest.java @@ -48,7 +48,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.parsers.JSONPathSpec; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; @@ -106,36 +105,37 @@ private static List> gene @Test(timeout = 10_000L) public void testSampleWithInputRowParser() throws Exception { - final DataSchema dataSchema = new DataSchema( - "test_ds", - OBJECT_MAPPER.convertValue( - new StringInputRowParser( - new JSONParseSpec( - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec( - Arrays.asList( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim1t"), - new StringDimensionSchema("dim2"), - new LongDimensionSchema("dimLong"), - new FloatDimensionSchema("dimFloat") - ) - ), - new JSONPathSpec(true, ImmutableList.of()), - ImmutableMap.of(), - false - ) - ), - Map.class - ), - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("met1sum", "met1"), - new CountAggregatorFactory("rows") - }, - new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null), - null, - OBJECT_MAPPER - ); + DataSchema dataSchema = DataSchema.builder() + .withDataSource("test_ds") + .withParserMap( + OBJECT_MAPPER.convertValue( + new StringInputRowParser( + new JSONParseSpec( + new TimestampSpec("timestamp", "iso", null), + new DimensionsSpec( + Arrays.asList( + new StringDimensionSchema("dim1"), + new StringDimensionSchema("dim1t"), + new StringDimensionSchema("dim2"), + new LongDimensionSchema("dimLong"), + new FloatDimensionSchema("dimFloat") + ) + ), + new JSONPathSpec(true, ImmutableList.of()), + ImmutableMap.of(), + false + ) + ), + Map.class + ) + ) + .withAggregators( + new DoubleSumAggregatorFactory("met1sum", "met1"), + new CountAggregatorFactory("rows") + ) + .withGranularity(new UniformGranularitySpec(Granularities.DAY, Granularities.NONE, null)) + .withObjectMapper(OBJECT_MAPPER) + .build(); final SeekableStreamSupervisorIOConfig supervisorIOConfig = new TestableSeekableStreamSupervisorIOConfig( STREAM, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 4deee6ce9b8d..baff5fc765b2 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -25,7 +25,6 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; @@ -59,7 +58,6 @@ import org.apache.druid.java.util.metrics.DruidMonitorSchedulerConfig; import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DruidMetrics; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; @@ -1261,18 +1259,19 @@ private static DataSchema getDataSchema() dimensions.add(StringDimensionSchema.create("dim1")); dimensions.add(StringDimensionSchema.create("dim2")); - return new DataSchema( - DATASOURCE, - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec(dimensions), - new AggregatorFactory[]{new CountAggregatorFactory("rows")}, - new UniformGranularitySpec( - Granularities.HOUR, - Granularities.NONE, - ImmutableList.of() - ), - null - ); + return DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions(dimensions) + .withAggregators(new CountAggregatorFactory("rows")) + .withGranularity( + new UniformGranularitySpec( + Granularities.HOUR, + Granularities.NONE, + ImmutableList.of() + ) + ) + .build(); } private SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scaleOut) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 1f42ba7ce996..40bbe84b623a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -31,7 +31,6 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; @@ -83,7 +82,6 @@ import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.metadata.PendingSegmentRecord; import org.apache.druid.query.DruidMetrics; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; @@ -2562,18 +2560,19 @@ private static DataSchema getDataSchema() dimensions.add(StringDimensionSchema.create("dim1")); dimensions.add(StringDimensionSchema.create("dim2")); - return new DataSchema( - DATASOURCE, - new TimestampSpec("timestamp", "iso", null), - new DimensionsSpec(dimensions), - new AggregatorFactory[]{new CountAggregatorFactory("rows")}, - new UniformGranularitySpec( - Granularities.HOUR, - Granularities.NONE, - ImmutableList.of() - ), - null - ); + return DataSchema.builder() + .withDataSource(DATASOURCE) + .withTimestamp(new TimestampSpec("timestamp", "iso", null)) + .withDimensions(dimensions) + .withAggregators(new CountAggregatorFactory("rows")) + .withGranularity( + new UniformGranularitySpec( + Granularities.HOUR, + Granularities.NONE, + ImmutableList.of() + ) + ) + .build(); } private static SeekableStreamSupervisorIOConfig getIOConfig() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/worker/TaskAnnouncementTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/worker/TaskAnnouncementTest.java index 88249509ef97..61396fc7ae61 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/worker/TaskAnnouncementTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/worker/TaskAnnouncementTest.java @@ -28,7 +28,6 @@ import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.common.task.TaskResource; import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.junit.Assert; import org.junit.Test; @@ -51,7 +50,7 @@ public void testBackwardsCompatibleSerde() throws Exception "theid", new TaskResource("rofl", 2), new IndexTask.IndexIngestionSpec( - new DataSchema("foo", null, new AggregatorFactory[0], null, null, new DefaultObjectMapper()), + DataSchema.builder().withDataSource("foo").withObjectMapper(new DefaultObjectMapper()).build(), ioConfig, null ), diff --git a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java index bda884018812..55b82924787c 100644 --- a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java +++ b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java @@ -25,7 +25,6 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Multiset; @@ -64,6 +63,17 @@ public class DataSchema { private static final Logger log = new Logger(DataSchema.class); + + public static Builder builder() + { + return new Builder(); + } + + public static Builder builder(DataSchema schema) + { + return new Builder(schema); + } + private final String dataSource; private final AggregatorFactory[] aggregators; private final GranularitySpec granularitySpec; @@ -126,33 +136,6 @@ public DataSchema( } } - @VisibleForTesting - public DataSchema( - String dataSource, - TimestampSpec timestampSpec, - DimensionsSpec dimensionsSpec, - AggregatorFactory[] aggregators, - GranularitySpec granularitySpec, - TransformSpec transformSpec - ) - { - this(dataSource, timestampSpec, dimensionsSpec, aggregators, granularitySpec, transformSpec, null, null); - } - - // old constructor for backward compatibility - @Deprecated - public DataSchema( - String dataSource, - Map parserMap, - AggregatorFactory[] aggregators, - GranularitySpec granularitySpec, - TransformSpec transformSpec, - ObjectMapper objectMapper - ) - { - this(dataSource, null, null, aggregators, granularitySpec, transformSpec, parserMap, objectMapper); - } - private static void validateDatasourceName(String dataSource) { IdUtils.validateId("dataSource", dataSource); @@ -403,44 +386,17 @@ public InputRowParser getParser() public DataSchema withGranularitySpec(GranularitySpec granularitySpec) { - return new DataSchema( - dataSource, - timestampSpec, - dimensionsSpec, - aggregators, - granularitySpec, - transformSpec, - parserMap, - objectMapper - ); + return builder(this).withGranularity(granularitySpec).build(); } public DataSchema withTransformSpec(TransformSpec transformSpec) { - return new DataSchema( - dataSource, - timestampSpec, - dimensionsSpec, - aggregators, - granularitySpec, - transformSpec, - parserMap, - objectMapper - ); + return builder(this).withTransform(transformSpec).build(); } public DataSchema withDimensionsSpec(DimensionsSpec dimensionsSpec) { - return new DataSchema( - dataSource, - timestampSpec, - dimensionsSpec, - aggregators, - granularitySpec, - transformSpec, - parserMap, - objectMapper - ); + return builder(this).withDimensions(dimensionsSpec).build(); } @Override @@ -457,4 +413,108 @@ public String toString() ", inputRowParser=" + inputRowParser + '}'; } + + public static class Builder + { + private String dataSource; + private AggregatorFactory[] aggregators; + private GranularitySpec granularitySpec; + private TransformSpec transformSpec; + private Map parserMap; + private ObjectMapper objectMapper; + + // The below fields can be initialized lazily from parser for backward compatibility. + private TimestampSpec timestampSpec; + private DimensionsSpec dimensionsSpec; + + public Builder() + { + + } + + public Builder(DataSchema schema) + { + this.dataSource = schema.getDataSource(); + this.aggregators = schema.getAggregators(); + this.granularitySpec = schema.getGranularitySpec(); + this.transformSpec = schema.getTransformSpec(); + this.parserMap = schema.getParserMap(); + this.objectMapper = schema.objectMapper; + this.timestampSpec = schema.getTimestampSpec(); + this.dimensionsSpec = schema.getDimensionsSpec(); + } + + public Builder withDataSource(String dataSource) + { + this.dataSource = dataSource; + return this; + } + + public Builder withTimestamp(TimestampSpec timestampSpec) + { + this.timestampSpec = timestampSpec; + return this; + } + + public Builder withDimensions(DimensionsSpec dimensionsSpec) + { + this.dimensionsSpec = dimensionsSpec; + return this; + } + + public Builder withDimensions(List dimensions) + { + this.dimensionsSpec = DimensionsSpec.builder().setDimensions(dimensions).build(); + return this; + } + + public Builder withDimensions(DimensionSchema... dimensions) + { + return withDimensions(Arrays.asList(dimensions)); + } + + public Builder withAggregators(AggregatorFactory... aggregators) + { + this.aggregators = aggregators; + return this; + } + + public Builder withGranularity(GranularitySpec granularitySpec) + { + this.granularitySpec = granularitySpec; + return this; + } + + public Builder withTransform(TransformSpec transformSpec) + { + this.transformSpec = transformSpec; + return this; + } + + public Builder withObjectMapper(ObjectMapper objectMapper) + { + this.objectMapper = objectMapper; + return this; + } + + public Builder withParserMap(Map parserMap) + { + this.parserMap = parserMap; + return this; + } + + public DataSchema build() + { + return new DataSchema( + dataSource, + timestampSpec, + dimensionsSpec, + aggregators, + granularitySpec, + transformSpec, + parserMap, + objectMapper + ); + } + } } diff --git a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java index 90297dd4af9d..e994bc054d3c 100644 --- a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java @@ -71,6 +71,11 @@ public class DataSchemaTest extends InitializedNullHandlingTest { + private static ArbitraryGranularitySpec ARBITRARY_GRANULARITY = new ArbitraryGranularitySpec( + Granularities.DAY, + ImmutableList.of(Intervals.of("2014/2015")) + ); + @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -92,17 +97,16 @@ public void testDefaultExclusions() ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - parser, - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), - }, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - jsonMapper - ); + DataSchema schema = DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withParserMap(parser) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); Assert.assertEquals( ImmutableSet.of("__time", "time", "col1", "col2", "metric1", "metric2"), @@ -130,18 +134,16 @@ public void testExplicitInclude() null ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - parser, - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), - }, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - jsonMapper - ); + DataSchema schema = DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withParserMap(parser) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); Assert.assertEquals( ImmutableSet.of("__time", "dimC", "col1", "metric1", "metric2"), @@ -167,22 +169,24 @@ public void testTransformSpec() ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - parserMap, - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), - }, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - new TransformSpec( - new SelectorDimFilter("dimA", "foo", null), - ImmutableList.of( - new ExpressionTransform("expr", "concat(dimA,dimA)", TestExprMacroTable.INSTANCE) - ) - ), - jsonMapper - ); + DataSchema schema = DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withParserMap(parserMap) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withTransform( + new TransformSpec( + new SelectorDimFilter("dimA", "foo", null), + ImmutableList.of( + new ExpressionTransform("expr", "concat(dimA,dimA)", TestExprMacroTable.INSTANCE) + ) + ) + ) + .withObjectMapper(jsonMapper) + .build(); // Test hack that produces a StringInputRowParser. final StringInputRowParser parser = (StringInputRowParser) schema.getParser(); @@ -233,17 +237,16 @@ public void testOverlapMetricNameAndDim() ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - parser, - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), - }, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - jsonMapper - ); + DataSchema schema = DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withParserMap(parser) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); expectedException.expect(DruidException.class); expectedException.expectMessage( @@ -256,25 +259,24 @@ public void testOverlapMetricNameAndDim() @Test public void testOverlapTimeAndDimPositionZero() { - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - new TimestampSpec("time", "auto", null), - DimensionsSpec.builder() - .setDimensions( - ImmutableList.of( - new LongDimensionSchema("__time"), - new StringDimensionSchema("dimA"), - new StringDimensionSchema("dimB") - ) - ) - .setDimensionExclusions(ImmutableList.of("dimC")) - .build(), - null, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - null, - jsonMapper - ); + DataSchema schema = DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withTimestamp(new TimestampSpec("time", "auto", null)) + .withDimensions( + DimensionsSpec.builder() + .setDimensions( + ImmutableList.of( + new LongDimensionSchema("__time"), + new StringDimensionSchema("dimA"), + new StringDimensionSchema("dimB") + ) + ) + .setDimensionExclusions(ImmutableList.of("dimC")) + .build() + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); Assert.assertEquals( ImmutableList.of("__time", "dimA", "dimB"), @@ -290,25 +292,24 @@ public void testOverlapTimeAndDimPositionZeroWrongType() expectedException.expect(DruidException.class); expectedException.expectMessage("Encountered dimension[__time] with incorrect type[STRING]. Type must be 'long'."); - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - new TimestampSpec("time", "auto", null), - DimensionsSpec.builder() - .setDimensions( - ImmutableList.of( - new StringDimensionSchema("__time"), - new StringDimensionSchema("dimA"), - new StringDimensionSchema("dimB") - ) - ) - .setDimensionExclusions(ImmutableList.of("dimC")) - .build(), - null, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - null, - jsonMapper - ); + DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withTimestamp(new TimestampSpec("time", "auto", null)) + .withDimensions( + DimensionsSpec.builder() + .setDimensions( + ImmutableList.of( + new StringDimensionSchema("__time"), + new StringDimensionSchema("dimA"), + new StringDimensionSchema("dimB") + ) + ) + .setDimensionExclusions(ImmutableList.of("dimC")) + .build() + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); } @Test @@ -321,50 +322,49 @@ public void testOverlapTimeAndDimPositionOne() + DimensionsSpec.WARNING_NON_TIME_SORT_ORDER ); - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - new TimestampSpec("time", "auto", null), - DimensionsSpec.builder() - .setDimensions( - ImmutableList.of( - new StringDimensionSchema("dimA"), - new LongDimensionSchema("__time"), - new StringDimensionSchema("dimB") - ) - ) - .setDimensionExclusions(ImmutableList.of("dimC")) - .build(), - null, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - null, - jsonMapper - ); + DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withTimestamp(new TimestampSpec("time", "auto", null)) + .withDimensions( + DimensionsSpec.builder() + .setDimensions( + ImmutableList.of( + new StringDimensionSchema("dimA"), + new LongDimensionSchema("__time"), + new StringDimensionSchema("dimB") + ) + ) + .setDimensionExclusions(ImmutableList.of("dimC")) + .build() + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); } @Test public void testOverlapTimeAndDimPositionOne_withExplicitSortOrder() { - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - new TimestampSpec("time", "auto", null), - DimensionsSpec.builder() - .setDimensions( - ImmutableList.of( - new StringDimensionSchema("dimA"), - new LongDimensionSchema("__time"), - new StringDimensionSchema("dimB") - ) - ) - .setDimensionExclusions(ImmutableList.of("dimC")) - .setForceSegmentSortByTime(false) - .build(), - null, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - null, - jsonMapper - ); + DataSchema schema = + DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withTimestamp(new TimestampSpec("time", "auto", null)) + .withDimensions( + DimensionsSpec.builder() + .setDimensions( + ImmutableList.of( + new StringDimensionSchema("dimA"), + new LongDimensionSchema("__time"), + new StringDimensionSchema("dimB") + ) + ) + .setDimensionExclusions(ImmutableList.of("dimC")) + .setForceSegmentSortByTime(false) + .build() + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); Assert.assertEquals( ImmutableList.of("dimA", "__time", "dimB"), @@ -402,14 +402,13 @@ public void testOverlapTimeAndDimLegacy() ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - parser, - null, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - jsonMapper - ); + DataSchema schema = DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withParserMap(parser) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); + expectedException.expect(DruidException.class); expectedException.expectMessage("Encountered dimension[__time] with incorrect type[STRING]. Type must be 'long'."); @@ -442,20 +441,19 @@ public void testDuplicateAggregators() + "[metric3] seen in metricsSpec list (2 occurrences)" ); - DataSchema schema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - parser, - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), - new DoubleSumAggregatorFactory("metric1", "col3"), - new DoubleSumAggregatorFactory("metric3", "col4"), - new DoubleSumAggregatorFactory("metric3", "col5"), - }, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - jsonMapper - ); + DataSchema schema = DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withParserMap(parser) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2"), + new DoubleSumAggregatorFactory("metric1", "col3"), + new DoubleSumAggregatorFactory("metric3", "col4"), + new DoubleSumAggregatorFactory("metric3", "col5") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); } @Test @@ -514,20 +512,17 @@ public void testEmptyDatasource() .invalidInput() .expectMessageIs("Invalid value for field [dataSource]: must not be null") .assertThrowsAndMatches( - () -> new DataSchema( - "", - parser, - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), - }, - new ArbitraryGranularitySpec( - Granularities.DAY, - ImmutableList.of(Intervals.of("2014/2015")) - ), - null, - jsonMapper - )); + () -> DataSchema.builder() + .withDataSource("") + .withParserMap(parser) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build() + ); } @@ -547,14 +542,11 @@ public void testInvalidWhitespaceDatasource() dataSource ); DruidExceptionMatcher.invalidInput().expectMessageIs(msg).assertThrowsAndMatches( - () -> new DataSchema( - dataSource, - Collections.emptyMap(), - null, - null, - null, - jsonMapper - ) + () -> DataSchema.builder() + .withDataSource(dataSource) + .withParserMap(Collections.emptyMap()) + .withObjectMapper(jsonMapper) + .build() ); } } @@ -686,17 +678,16 @@ public void testSerdeWithUpdatedDataSchemaAddedField() throws IOException ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - DataSchema originalSchema = new DataSchema( - IdUtilsTest.VALID_ID_CHARS, - parser, - new AggregatorFactory[]{ - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2"), - }, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), - null, - jsonMapper - ); + DataSchema originalSchema = DataSchema.builder() + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withParserMap(parser) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); String serialized = jsonMapper.writeValueAsString(originalSchema); TestModifiedDataSchema deserialized = jsonMapper.readValue(serialized, TestModifiedDataSchema.class); @@ -734,7 +725,7 @@ public void testSerdeWithUpdatedDataSchemaRemovedField() throws IOException new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2"), }, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), + ARBITRARY_GRANULARITY, null, parser, jsonMapper, @@ -765,10 +756,16 @@ public void testWithDimensionSpec() Map parserMap = Mockito.mock(Map.class); Mockito.when(newDimSpec.withDimensionExclusions(ArgumentMatchers.any(Set.class))).thenReturn(newDimSpec); - DataSchema oldSchema = new DataSchema("dataSource", tsSpec, oldDimSpec, - new AggregatorFactory[]{aggFactory}, gSpec, - transSpec, parserMap, jsonMapper - ); + DataSchema oldSchema = DataSchema.builder() + .withDataSource("dataSource") + .withTimestamp(tsSpec) + .withDimensions(oldDimSpec) + .withAggregators(aggFactory) + .withGranularity(gSpec) + .withTransform(transSpec) + .withParserMap(parserMap) + .withObjectMapper(jsonMapper) + .build(); DataSchema newSchema = oldSchema.withDimensionsSpec(newDimSpec); Assert.assertSame(oldSchema.getDataSource(), newSchema.getDataSource()); Assert.assertSame(oldSchema.getTimestampSpec(), newSchema.getTimestampSpec()); @@ -795,7 +792,7 @@ public void testCombinedDataSchemaSetsMultiValuedColumnsInfo() .setDimensionExclusions(ImmutableList.of("dimC")) .build(), null, - new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), + ARBITRARY_GRANULARITY, null, multiValuedDimensions ); diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorTester.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorTester.java index 22034aa33aa9..7e3d1e233a06 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorTester.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorTester.java @@ -151,19 +151,18 @@ public BatchAppenderatorTester( Map.class ); - schema = new DataSchema( - DATASOURCE, - null, - null, - new AggregatorFactory[]{ - new CountAggregatorFactory("count"), - new LongSumAggregatorFactory("met", "met") - }, - new UniformGranularitySpec(Granularities.MINUTE, Granularities.NONE, null), - null, - parserMap, - objectMapper - ); + schema = DataSchema.builder() + .withDataSource(DATASOURCE) + .withAggregators( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("met", "met") + ) + .withGranularity( + new UniformGranularitySpec(Granularities.MINUTE, Granularities.NONE, null) + ) + .withParserMap(parserMap) + .withObjectMapper(objectMapper) + .build(); tuningConfig = new TestAppenderatorConfig( TuningConfig.DEFAULT_APPENDABLE_INDEX, diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java index cd990e76f892..29d758aaed02 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java @@ -44,7 +44,6 @@ import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; import org.apache.druid.query.ForwardingQueryProcessingPool; import org.apache.druid.query.QueryRunnerTestHelper; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.expression.TestExprMacroTable; @@ -135,17 +134,16 @@ public StreamAppenderatorTester( ), Map.class ); - schema = new DataSchema( - DATASOURCE, - parserMap, - new AggregatorFactory[]{ - new CountAggregatorFactory("count"), - new LongSumAggregatorFactory("met", "met") - }, - new UniformGranularitySpec(Granularities.MINUTE, Granularities.NONE, null), - null, - objectMapper - ); + schema = DataSchema.builder() + .withDataSource(DATASOURCE) + .withParserMap(parserMap) + .withAggregators( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("met", "met") + ) + .withGranularity(new UniformGranularitySpec(Granularities.MINUTE, Granularities.NONE, null)) + .withObjectMapper(objectMapper) + .build(); tuningConfig = new TestAppenderatorConfig( TuningConfig.DEFAULT_APPENDABLE_INDEX, maxRowsInMemory, diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java index 23ac93db0096..21f627baa085 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java @@ -98,14 +98,11 @@ public void setup() EasyMock.replay(appenderatorConfig); appenderator = manager.createBatchAppenderatorForTask( "taskId", - new DataSchema( - "myDataSource", - new TimestampSpec("__time", "millis", null), - null, - null, - new UniformGranularitySpec(Granularities.HOUR, Granularities.HOUR, false, Collections.emptyList()), - null - ), + DataSchema.builder() + .withDataSource("myDataSource") + .withTimestamp(new TimestampSpec("__time", "millis", null)) + .withGranularity(new UniformGranularitySpec(Granularities.HOUR, Granularities.HOUR, false, Collections.emptyList())) + .build(), appenderatorConfig, new SegmentGenerationMetrics(), new NoopDataSegmentPusher(), diff --git a/server/src/test/java/org/apache/druid/segment/realtime/sink/SinkTest.java b/server/src/test/java/org/apache/druid/segment/realtime/sink/SinkTest.java index 9d85ec6c8e6b..750ea06c6534 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/sink/SinkTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/sink/SinkTest.java @@ -34,7 +34,6 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequences; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.segment.RowAdapters; import org.apache.druid.segment.RowBasedSegment; @@ -76,14 +75,14 @@ public class SinkTest extends InitializedNullHandlingTest @Test public void testSwap() throws Exception { - final DataSchema schema = new DataSchema( - "test", - new TimestampSpec(null, null, null), - DimensionsSpec.EMPTY, - new AggregatorFactory[]{new CountAggregatorFactory("rows")}, - new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), - null - ); + final DataSchema schema = + DataSchema.builder() + .withDataSource("test") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions(DimensionsSpec.EMPTY) + .withAggregators(new CountAggregatorFactory("rows")) + .withGranularity(new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null)) + .build(); final Interval interval = Intervals.of("2013-01-01/2013-01-02"); final String version = DateTimes.nowUtc().toString(); @@ -256,18 +255,17 @@ public void testAcquireSegmentReferences_twoWithOneSwappedToNull() @Test public void testGetSinkSignature() throws IndexSizeExceededException { - final DataSchema schema = new DataSchema( - "test", - new TimestampSpec(null, null, null), - new DimensionsSpec( - Arrays.asList( - new StringDimensionSchema("dim1"), - new LongDimensionSchema("dimLong") - )), - new AggregatorFactory[]{new CountAggregatorFactory("rows")}, - new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), - null - ); + final DataSchema schema = + DataSchema.builder() + .withDataSource("test") + .withTimestamp(new TimestampSpec(null, null, null)) + .withDimensions( + new StringDimensionSchema("dim1"), + new LongDimensionSchema("dimLong") + ) + .withAggregators(new CountAggregatorFactory("rows")) + .withGranularity(new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null)) + .build(); final Interval interval = Intervals.of("2013-01-01/2013-01-02"); final String version = DateTimes.nowUtc().toString(); diff --git a/services/src/test/java/org/apache/druid/cli/validate/DruidJsonValidatorTest.java b/services/src/test/java/org/apache/druid/cli/validate/DruidJsonValidatorTest.java index b617b7e6b877..c0634ed403c0 100644 --- a/services/src/test/java/org/apache/druid/cli/validate/DruidJsonValidatorTest.java +++ b/services/src/test/java/org/apache/druid/cli/validate/DruidJsonValidatorTest.java @@ -30,7 +30,6 @@ import org.apache.druid.indexing.common.task.TaskResource; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; @@ -133,14 +132,11 @@ public void testTaskValidator() throws Exception null, new TaskResource("rofl", 2), new IndexTask.IndexIngestionSpec( - new DataSchema( - "foo", - null, - new AggregatorFactory[0], - new UniformGranularitySpec(Granularities.HOUR, Granularities.NONE, null), - null, - jsonMapper - ), + DataSchema.builder() + .withDataSource("foo") + .withGranularity(new UniformGranularitySpec(Granularities.HOUR, Granularities.NONE, null)) + .withObjectMapper(jsonMapper) + .build(), new IndexTask.IndexIOConfig( new LocalInputSource(new File("lol"), "rofl"), new JsonInputFormat(null, null, null, null, null), From 3c341d9e5c4fad912e1d5353201f18322b3bcf8e Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sat, 14 Sep 2024 05:07:51 -0700 Subject: [PATCH 2/5] fixes --- .../apache/druid/segment/indexing/DataSchema.java | 14 +++++++------- .../appenderator/BatchAppenderatorTester.java | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java index 55b82924787c..39b21ae3f1d4 100644 --- a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java +++ b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java @@ -434,14 +434,14 @@ public Builder() public Builder(DataSchema schema) { - this.dataSource = schema.getDataSource(); - this.aggregators = schema.getAggregators(); - this.granularitySpec = schema.getGranularitySpec(); - this.transformSpec = schema.getTransformSpec(); - this.parserMap = schema.getParserMap(); + this.dataSource = schema.dataSource; + this.aggregators = schema.aggregators; + this.granularitySpec = schema.granularitySpec; + this.transformSpec = schema.transformSpec; + this.parserMap = schema.parserMap; this.objectMapper = schema.objectMapper; - this.timestampSpec = schema.getTimestampSpec(); - this.dimensionsSpec = schema.getDimensionsSpec(); + this.timestampSpec = schema.timestampSpec; + this.dimensionsSpec = schema.dimensionsSpec; } public Builder withDataSource(String dataSource) diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorTester.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorTester.java index 7e3d1e233a06..5f43236e075b 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorTester.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/BatchAppenderatorTester.java @@ -30,7 +30,6 @@ import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.core.NoopEmitter; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.IndexIO; From c54781153fd93236f984de7571d071773e44573c Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sat, 14 Sep 2024 14:18:33 -0700 Subject: [PATCH 3/5] fixes --- .../msq/indexing/MSQCompactionRunnerTest.java | 1 + .../segment/indexing/DataSchemaTest.java | 29 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java index b3f4029c52da..15b12be15753 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java @@ -272,6 +272,7 @@ public void testMSQControllerTaskSpecWithScanIsValid() throws JsonProcessingExce Collections.singletonList(COMPACTION_INTERVAL) ) ) + .withTransform(new TransformSpec(dimFilter, Collections.emptyList())) .build(); diff --git a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java index e994bc054d3c..4b7abe284340 100644 --- a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java @@ -508,21 +508,20 @@ public void testEmptyDatasource() ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - DruidExceptionMatcher - .invalidInput() - .expectMessageIs("Invalid value for field [dataSource]: must not be null") - .assertThrowsAndMatches( - () -> DataSchema.builder() - .withDataSource("") - .withParserMap(parser) - .withAggregators( - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2") - ) - .withGranularity(ARBITRARY_GRANULARITY) - .withObjectMapper(jsonMapper) - .build() - ); + DruidExceptionMatcher.invalidInput() + .expectMessageIs("Invalid value for field [dataSource]: must not be null") + .assertThrowsAndMatches( + () -> DataSchema.builder() + .withDataSource("") + .withParserMap(parser) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build() + ); } From 5c6df6ee0720c41c561e4fb2a0fcc1803adbb27f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sat, 14 Sep 2024 17:18:48 -0700 Subject: [PATCH 4/5] more style fixes --- .../indexer/path/GranularityPathSpecTest.java | 1 - .../segment/indexing/DataSchemaTest.java | 48 ++++++++++--------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java index 52258f201619..92bd8595560c 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java @@ -34,7 +34,6 @@ import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.granularity.PeriodGranularity; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; import org.apache.hadoop.mapreduce.Job; diff --git a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java index 4b7abe284340..94bd77e810df 100644 --- a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java @@ -181,7 +181,11 @@ public void testTransformSpec() new TransformSpec( new SelectorDimFilter("dimA", "foo", null), ImmutableList.of( - new ExpressionTransform("expr", "concat(dimA,dimA)", TestExprMacroTable.INSTANCE) + new ExpressionTransform( + "expr", + "concat(dimA,dimA)", + TestExprMacroTable.INSTANCE + ) ) ) ) @@ -508,20 +512,20 @@ public void testEmptyDatasource() ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); + DruidExceptionMatcher.ThrowingSupplier thrower = + () -> DataSchema.builder() + .withDataSource("") + .withParserMap(parser) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); DruidExceptionMatcher.invalidInput() .expectMessageIs("Invalid value for field [dataSource]: must not be null") - .assertThrowsAndMatches( - () -> DataSchema.builder() - .withDataSource("") - .withParserMap(parser) - .withAggregators( - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2") - ) - .withGranularity(ARBITRARY_GRANULARITY) - .withObjectMapper(jsonMapper) - .build() - ); + .assertThrowsAndMatches(thrower); } @@ -678,15 +682,15 @@ public void testSerdeWithUpdatedDataSchemaAddedField() throws IOException ); DataSchema originalSchema = DataSchema.builder() - .withDataSource(IdUtilsTest.VALID_ID_CHARS) - .withParserMap(parser) - .withAggregators( - new DoubleSumAggregatorFactory("metric1", "col1"), - new DoubleSumAggregatorFactory("metric2", "col2") - ) - .withGranularity(ARBITRARY_GRANULARITY) - .withObjectMapper(jsonMapper) - .build(); + .withDataSource(IdUtilsTest.VALID_ID_CHARS) + .withParserMap(parser) + .withAggregators( + new DoubleSumAggregatorFactory("metric1", "col1"), + new DoubleSumAggregatorFactory("metric2", "col2") + ) + .withGranularity(ARBITRARY_GRANULARITY) + .withObjectMapper(jsonMapper) + .build(); String serialized = jsonMapper.writeValueAsString(originalSchema); TestModifiedDataSchema deserialized = jsonMapper.readValue(serialized, TestModifiedDataSchema.class); From 29de434466f3482d3d9ab78c9af78d737609cac9 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sun, 15 Sep 2024 01:23:00 -0700 Subject: [PATCH 5/5] review stuff --- .../druid/msq/indexing/destination/SegmentGenerationUtils.java | 3 --- .../java/org/apache/druid/segment/indexing/DataSchema.java | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java index d2cb1a210384..b37a29f53e20 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/destination/SegmentGenerationUtils.java @@ -53,14 +53,12 @@ import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import org.apache.druid.segment.indexing.granularity.GranularitySpec; -import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.sql.calcite.planner.ColumnMappings; import org.apache.druid.sql.calcite.rel.DruidQuery; import org.apache.druid.utils.CollectionUtils; import javax.annotation.Nullable; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; @@ -102,7 +100,6 @@ public static DataSchema makeDataSchemaForIngestion( .withDimensions(dimensionsAndAggregators.lhs) .withAggregators(dimensionsAndAggregators.rhs.toArray(new AggregatorFactory[0])) .withGranularity(makeGranularitySpecForIngestion(querySpec.getQuery(), querySpec.getColumnMappings(), isRollupQuery, jsonMapper)) - .withTransform(new TransformSpec(null, Collections.emptyList())) .build(); } diff --git a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java index 39b21ae3f1d4..22ee4ec41025 100644 --- a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java +++ b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java @@ -491,12 +491,14 @@ public Builder withTransform(TransformSpec transformSpec) return this; } + @Deprecated public Builder withObjectMapper(ObjectMapper objectMapper) { this.objectMapper = objectMapper; return this; } + @Deprecated public Builder withParserMap(Map parserMap) { this.parserMap = parserMap;