diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java index e9c570cea1d2..58bbdc9119d3 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java @@ -179,7 +179,7 @@ public void setup() throws IOException indexFile = INDEX_MERGER_V9.persist( incIndex, tmpDir, - new IndexSpec(), + IndexSpec.DEFAULT, null ); qIndex = INDEX_IO.loadIndex(indexFile); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java index 9c42bca4e44d..ee07de775917 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java @@ -282,7 +282,7 @@ public void setup(FilteredAggregatorBenchmark global) throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndex, qIndexesDir, - new IndexSpec(), + IndexSpec.DEFAULT, null ); incIndex.close(); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java index cd6c48a16ea6..a37fab082283 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java @@ -324,7 +324,7 @@ public void setup() throws IOException final File file = INDEX_MERGER_V9.persist( index, new File(tmpDir, String.valueOf(i)), - new IndexSpec(), + IndexSpec.DEFAULT, null ); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/TopNTypeInterfaceBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/TopNTypeInterfaceBenchmark.java index b6078aecb5c4..984d01ebde2e 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/TopNTypeInterfaceBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/TopNTypeInterfaceBenchmark.java @@ -287,7 +287,7 @@ public void setup() throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndexes.get(i), tmpFile, - new IndexSpec(), + IndexSpec.DEFAULT, null ); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexMergeBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexMergeBenchmark.java index 86d735eadc9a..b2d32cd50710 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexMergeBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexMergeBenchmark.java @@ -141,7 +141,7 @@ public void setup() throws IOException File indexFile = indexMergerV9.persist( incIndex, tmpDir, - new IndexSpec(), + IndexSpec.DEFAULT, null ); @@ -166,7 +166,7 @@ public void mergeV9(Blackhole blackhole) throws Exception rollup, schemaInfo.getAggsArray(), tmpFile, - new IndexSpec(), + IndexSpec.DEFAULT, null, -1 ); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexPersistBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexPersistBenchmark.java index 9b91a70293e5..5fff33d1fec4 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexPersistBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexPersistBenchmark.java @@ -189,7 +189,7 @@ public void persistV9(Blackhole blackhole) throws Exception File indexFile = INDEX_MERGER_V9.persist( incIndex, tmpDir, - new IndexSpec(), + IndexSpec.DEFAULT, null ); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/GroupByBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/GroupByBenchmark.java index 602126ba4a26..db572c8a4fe0 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/GroupByBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/GroupByBenchmark.java @@ -601,7 +601,7 @@ public void setup(GroupByBenchmark global) throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndex, new File(qIndexesDir, String.valueOf(i)), - new IndexSpec(), + IndexSpec.DEFAULT, null ); incIndex.close(); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/ScanBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/ScanBenchmark.java index b9ae2a40f662..45d89f14addf 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/ScanBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/ScanBenchmark.java @@ -340,7 +340,7 @@ public void setup(ScanBenchmark global) throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndex, new File(qIndexesDir, String.valueOf(i)), - new IndexSpec(), + IndexSpec.DEFAULT, null ); incIndex.close(); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SearchBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SearchBenchmark.java index a3de820c0682..e1d027581266 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SearchBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SearchBenchmark.java @@ -407,7 +407,7 @@ public void setup(SearchBenchmark global) throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndex, new File(qIndexesDir, String.valueOf(i)), - new IndexSpec(), + IndexSpec.DEFAULT, null ); incIndex.close(); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java index d32c7fac5589..9459b234303b 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java @@ -457,15 +457,7 @@ public void setup() final QueryableIndex index = segmentGenerator.generate( dataSegment, schemaInfo, - new IndexSpec( - null, - null, - encodingStrategy, - null, - null, - null, - null - ), + IndexSpec.builder().withStringDictionaryEncoding(encodingStrategy).build(), Granularities.NONE, rowsPerSegment ); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java index 1ba7c24c0daa..aeda68b25b05 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java @@ -322,15 +322,7 @@ public void setup() schemaInfo, dimsSpec, transformSpec, - new IndexSpec( - null, - null, - encodingStrategy, - null, - null, - null, - null - ), + IndexSpec.builder().withStringDictionaryEncoding(encodingStrategy).build(), Granularities.NONE, rowsPerSegment ); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/TimeseriesBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/TimeseriesBenchmark.java index 1f271e22274c..68625768967b 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/TimeseriesBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/TimeseriesBenchmark.java @@ -332,7 +332,7 @@ public void setup(TimeseriesBenchmark global) throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndex, new File(qIndexesDir, String.valueOf(i)), - new IndexSpec(), + IndexSpec.DEFAULT, null ); incIndex.close(); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/TopNBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/TopNBenchmark.java index 3eaf7dc85f48..7bced644ac71 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/TopNBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/TopNBenchmark.java @@ -308,7 +308,7 @@ public void setup(TopNBenchmark global) throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndex, new File(qIndexesDir, String.valueOf(i)), - new IndexSpec(), + IndexSpec.DEFAULT, null ); incIndex.close(); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/timecompare/TimeCompareBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/timecompare/TimeCompareBenchmark.java index 0208fe16f89e..cc5d12dfa670 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/timecompare/TimeCompareBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/timecompare/TimeCompareBenchmark.java @@ -336,7 +336,7 @@ public void setup() throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndexes.get(i), tmpDir, - new IndexSpec(), + IndexSpec.DEFAULT, null ); diff --git a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java index 02735cf00f10..6bdd3678bc89 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java +++ b/extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java @@ -51,7 +51,7 @@ public class K8sTestUtils { - private static final IndexSpec INDEX_SPEC = new IndexSpec(); + private static final IndexSpec INDEX_SPEC = IndexSpec.DEFAULT; /* diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfigTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfigTest.java index 5033929830a9..f0b489ec5972 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfigTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfigTest.java @@ -68,8 +68,8 @@ public void testSerdeWithDefaults() throws Exception Assert.assertNull(config.getMaxTotalRows()); Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod()); Assert.assertEquals(0, config.getMaxPendingPersists()); - Assert.assertEquals(new IndexSpec(), config.getIndexSpec()); - Assert.assertEquals(new IndexSpec(), config.getIndexSpecForIntermediatePersists()); + Assert.assertEquals(IndexSpec.DEFAULT, config.getIndexSpec()); + Assert.assertEquals(IndexSpec.DEFAULT, config.getIndexSpecForIntermediatePersists()); Assert.assertEquals(false, config.isReportParseExceptions()); Assert.assertEquals(0, config.getHandoffConditionTimeout()); } @@ -112,8 +112,14 @@ public void testSerdeWithNonDefaults() throws Exception Assert.assertEquals(100, config.getMaxPendingPersists()); Assert.assertEquals(true, config.isReportParseExceptions()); Assert.assertEquals(100, config.getHandoffConditionTimeout()); - Assert.assertEquals(new IndexSpec(null, null, CompressionStrategy.NONE, null), config.getIndexSpec()); - Assert.assertEquals(new IndexSpec(null, CompressionStrategy.UNCOMPRESSED, null, null), config.getIndexSpecForIntermediatePersists()); + Assert.assertEquals( + IndexSpec.builder().withMetricCompression(CompressionStrategy.NONE).build(), + config.getIndexSpec() + ); + Assert.assertEquals( + IndexSpec.builder().withDimensionCompression(CompressionStrategy.UNCOMPRESSED).build(), + config.getIndexSpecForIntermediatePersists() + ); } @Test @@ -128,8 +134,8 @@ public void testConvert() 10L, new Period("PT3S"), 4, - new IndexSpec(), - new IndexSpec(), + IndexSpec.DEFAULT, + IndexSpec.DEFAULT, true, 5L, null, @@ -156,7 +162,7 @@ public void testConvert() Assert.assertEquals(new Period("PT3S"), copy.getIntermediatePersistPeriod()); Assert.assertNull(copy.getBasePersistDirectory()); Assert.assertEquals(4, copy.getMaxPendingPersists()); - Assert.assertEquals(new IndexSpec(), copy.getIndexSpec()); + Assert.assertEquals(IndexSpec.DEFAULT, copy.getIndexSpec()); Assert.assertEquals(true, copy.isReportParseExceptions()); Assert.assertEquals(5L, copy.getHandoffConditionTimeout()); } @@ -174,8 +180,8 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException new Period("PT3S"), new File("/tmp/xxx"), 4, - new IndexSpec(), - new IndexSpec(), + IndexSpec.DEFAULT, + IndexSpec.DEFAULT, true, 5L, null, @@ -222,8 +228,8 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException 10L, new Period("PT3S"), 4, - new IndexSpec(), - new IndexSpec(), + IndexSpec.DEFAULT, + IndexSpec.DEFAULT, true, 5L, null, @@ -262,7 +268,12 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException public void testEqualsAndHashCode() { EqualsVerifier.forClass(KafkaIndexTaskTuningConfig.class) - .usingGetClass() - .verify(); + .withPrefabValues( + IndexSpec.class, + IndexSpec.DEFAULT, + IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build() + ) + .usingGetClass() + .verify(); } } diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfigTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfigTest.java index d0ad3cb08854..d9d572220b19 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfigTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfigTest.java @@ -63,8 +63,8 @@ public void testSerdeWithDefaults() throws Exception Assert.assertEquals(5_000_000, config.getMaxRowsPerSegment().intValue()); Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod()); Assert.assertEquals(0, config.getMaxPendingPersists()); - Assert.assertEquals(new IndexSpec(), config.getIndexSpec()); - Assert.assertEquals(new IndexSpec(), config.getIndexSpecForIntermediatePersists()); + Assert.assertEquals(IndexSpec.DEFAULT, config.getIndexSpec()); + Assert.assertEquals(IndexSpec.DEFAULT, config.getIndexSpecForIntermediatePersists()); Assert.assertEquals(false, config.isReportParseExceptions()); Assert.assertEquals(0, config.getHandoffConditionTimeout()); Assert.assertNull(config.getWorkerThreads()); @@ -122,8 +122,14 @@ public void testSerdeWithNonDefaults() throws Exception Assert.assertEquals(Duration.standardSeconds(15), config.getHttpTimeout()); Assert.assertEquals(Duration.standardSeconds(95), config.getShutdownTimeout()); Assert.assertEquals(Duration.standardSeconds(20), config.getOffsetFetchPeriod()); - Assert.assertEquals(new IndexSpec(null, null, CompressionStrategy.NONE, null), config.getIndexSpec()); - Assert.assertEquals(new IndexSpec(null, CompressionStrategy.UNCOMPRESSED, null, null), config.getIndexSpecForIntermediatePersists()); + Assert.assertEquals( + IndexSpec.builder().withMetricCompression(CompressionStrategy.NONE).build(), + config.getIndexSpec() + ); + Assert.assertEquals( + IndexSpec.builder().withDimensionCompression(CompressionStrategy.UNCOMPRESSED).build(), + config.getIndexSpecForIntermediatePersists() + ); } } diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfigTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfigTest.java index 2892ea51c372..ed497a64cfac 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfigTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfigTest.java @@ -27,6 +27,7 @@ import org.apache.druid.indexing.kinesis.test.TestModifiedKinesisIndexTaskTuningConfig; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; import org.apache.druid.segment.indexing.TuningConfig; import org.hamcrest.CoreMatchers; @@ -73,7 +74,7 @@ public void testSerdeWithDefaults() throws Exception Assert.assertEquals(5_000_000, config.getMaxRowsPerSegment().intValue()); Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod()); Assert.assertEquals(0, config.getMaxPendingPersists()); - Assert.assertEquals(new IndexSpec(), config.getIndexSpec()); + Assert.assertEquals(IndexSpec.DEFAULT, config.getIndexSpec()); Assert.assertFalse(config.isReportParseExceptions()); Assert.assertEquals(0, config.getHandoffConditionTimeout()); Assert.assertNull(config.getRecordBufferSizeConfigured()); @@ -146,8 +147,8 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException new Period("PT3S"), new File("/tmp/xxx"), 4, - new IndexSpec(), - new IndexSpec(), + IndexSpec.DEFAULT, + IndexSpec.DEFAULT, true, 5L, true, @@ -205,8 +206,8 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException new Period("PT3S"), new File("/tmp/xxx"), 4, - new IndexSpec(), - new IndexSpec(), + IndexSpec.DEFAULT, + IndexSpec.DEFAULT, true, 5L, true, @@ -289,8 +290,8 @@ public void testConvert() 100L, new Period("PT3S"), 4, - new IndexSpec(), - new IndexSpec(), + IndexSpec.DEFAULT, + IndexSpec.DEFAULT, true, 5L, true, @@ -325,7 +326,7 @@ public void testConvert() Assert.assertEquals(new Period("PT3S"), copy.getIntermediatePersistPeriod()); Assert.assertNull(copy.getBasePersistDirectory()); Assert.assertEquals(4, copy.getMaxPendingPersists()); - Assert.assertEquals(new IndexSpec(), copy.getIndexSpec()); + Assert.assertEquals(IndexSpec.DEFAULT, copy.getIndexSpec()); Assert.assertTrue(copy.isReportParseExceptions()); Assert.assertEquals(5L, copy.getHandoffConditionTimeout()); Assert.assertEquals(1000, (int) copy.getRecordBufferSizeConfigured()); @@ -342,7 +343,12 @@ public void testConvert() public void testEqualsAndHashCode() { EqualsVerifier.forClass(KinesisIndexTaskTuningConfig.class) - .usingGetClass() - .verify(); + .withPrefabValues( + IndexSpec.class, + IndexSpec.DEFAULT, + IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build() + ) + .usingGetClass() + .verify(); } } diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfigTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfigTest.java index 8f7440f2d793..320a978f1251 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfigTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfigTest.java @@ -62,7 +62,7 @@ public void testSerdeWithDefaults() throws Exception Assert.assertEquals(5_000_000, config.getMaxRowsPerSegment().intValue()); Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod()); Assert.assertEquals(0, config.getMaxPendingPersists()); - Assert.assertEquals(new IndexSpec(), config.getIndexSpec()); + Assert.assertEquals(IndexSpec.DEFAULT, config.getIndexSpec()); Assert.assertEquals(false, config.isReportParseExceptions()); Assert.assertEquals(0, config.getHandoffConditionTimeout()); Assert.assertNull(config.getWorkerThreads()); diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQTuningConfig.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQTuningConfig.java index f97e250e1ce8..f36cbec2b399 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQTuningConfig.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQTuningConfig.java @@ -122,7 +122,7 @@ public int getRowsPerSegment() public IndexSpec getIndexSpec() { - return indexSpec != null ? indexSpec : new IndexSpec(); + return indexSpec != null ? indexSpec : IndexSpec.DEFAULT; } @Override diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/SegmentGeneratorFrameProcessorFactory.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/SegmentGeneratorFrameProcessorFactory.java index f7d405b8a8ac..81d91c28461d 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/SegmentGeneratorFrameProcessorFactory.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/SegmentGeneratorFrameProcessorFactory.java @@ -302,13 +302,13 @@ public IndexSpec getIndexSpec() public IndexSpec getIndexSpecForIntermediatePersists() { // Disable compression for intermediate persists to reduce direct memory usage. - return new IndexSpec( - null, - CompressionStrategy.UNCOMPRESSED, // Dimensions don't support NONE, so use UNCOMPRESSED - CompressionStrategy.NONE, // NONE is more efficient than UNCOMPRESSED - CompressionFactory.LongEncodingStrategy.LONGS, - null - ); + return IndexSpec.builder() + // Dimensions don't support NONE, so use UNCOMPRESSED + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + // NONE is more efficient than UNCOMPRESSED + .withMetricCompression(CompressionStrategy.NONE) + .withLongEncoding(CompressionFactory.LongEncodingStrategy.LONGS) + .build(); } @Override diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQSpecTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQSpecTest.java index a8f806fb1ca2..5d1e386f9a53 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQSpecTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQSpecTest.java @@ -20,6 +20,8 @@ package org.apache.druid.msq.indexing; import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.data.CompressionStrategy; import org.junit.Test; public class MSQSpecTest @@ -30,6 +32,11 @@ public void testEquals() { EqualsVerifier.forClass(MSQSpec.class) .withNonnullFields("query", "destination", "tuningConfig") + .withPrefabValues( + IndexSpec.class, + IndexSpec.DEFAULT, + IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build() + ) .usingGetClass() .verify(); } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQTuningConfigTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQTuningConfigTest.java index 45bc8ac46af5..44d22b3d50fa 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQTuningConfigTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQTuningConfigTest.java @@ -24,6 +24,7 @@ import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.StringEncodingStrategy; +import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.FrontCodedIndexed; import org.junit.Assert; import org.junit.Test; @@ -52,15 +53,11 @@ public void testSerdeNonDefault() throws Exception 2, 3, 4, - new IndexSpec( - null, - null, - new StringEncodingStrategy.FrontCoded(null, FrontCodedIndexed.V1), - null, - null, - null, - null - ) + IndexSpec.builder() + .withStringDictionaryEncoding( + new StringEncodingStrategy.FrontCoded(null, FrontCodedIndexed.V1) + ) + .build() ); Assert.assertEquals(config, mapper.readValue(mapper.writeValueAsString(config), MSQTuningConfig.class)); @@ -69,6 +66,13 @@ public void testSerdeNonDefault() throws Exception @Test public void testEquals() { - EqualsVerifier.forClass(MSQTuningConfig.class).usingGetClass().verify(); + EqualsVerifier.forClass(MSQTuningConfig.class) + .withPrefabValues( + IndexSpec.class, + IndexSpec.DEFAULT, + IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build() + ) + .usingGetClass() + .verify(); } } diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MultiStageQueryContextTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MultiStageQueryContextTest.java index c118a40e89aa..a604f6626af7 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MultiStageQueryContextTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/util/MultiStageQueryContextTest.java @@ -204,16 +204,20 @@ public void testGetIndexSpec() { Assert.assertNull(decodeIndexSpec(null)); - Assert.assertEquals(new IndexSpec(), decodeIndexSpec("{}")); - Assert.assertEquals(new IndexSpec(), decodeIndexSpec(Collections.emptyMap())); + Assert.assertEquals(IndexSpec.DEFAULT, decodeIndexSpec("{}")); + Assert.assertEquals(IndexSpec.DEFAULT, decodeIndexSpec(Collections.emptyMap())); Assert.assertEquals( - new IndexSpec(null, null, new StringEncodingStrategy.FrontCoded(null, null), null, null, null, null), + IndexSpec.builder() + .withStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(null, null)) + .build(), decodeIndexSpec("{\"stringDictionaryEncoding\":{\"type\":\"frontCoded\"}}") ); Assert.assertEquals( - new IndexSpec(null, null, new StringEncodingStrategy.FrontCoded(null), null, null, null, null), + IndexSpec.builder() + .withStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(null)) + .build(), decodeIndexSpec(ImmutableMap.of("stringDictionaryEncoding", ImmutableMap.of("type", "frontCoded"))) ); diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/HadoopTuningConfig.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/HadoopTuningConfig.java index 88b8fbdc161e..7371e70653f2 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/HadoopTuningConfig.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/HadoopTuningConfig.java @@ -43,7 +43,7 @@ public class HadoopTuningConfig implements TuningConfig { private static final DimensionBasedPartitionsSpec DEFAULT_PARTITIONS_SPEC = HashedPartitionsSpec.defaultSpec(); private static final Map> DEFAULT_SHARD_SPECS = ImmutableMap.of(); - private static final IndexSpec DEFAULT_INDEX_SPEC = new IndexSpec(); + private static final IndexSpec DEFAULT_INDEX_SPEC = IndexSpec.DEFAULT; private static final boolean DEFAULT_USE_COMBINER = false; private static final int DEFAULT_NUM_BACKGROUND_PERSIST_THREADS = 0; diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopTuningConfigTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopTuningConfigTest.java index 0bac6b2206ba..7bd993d2382e 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopTuningConfigTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopTuningConfigTest.java @@ -73,8 +73,8 @@ public void testSerde() throws Exception Assert.assertEquals(new OnheapIncrementalIndex.Spec(), actual.getAppendableIndexSpec()); Assert.assertNotNull(actual.getPartitionsSpec()); Assert.assertEquals(ImmutableMap.>of(), actual.getShardSpecs()); - Assert.assertEquals(new IndexSpec(), actual.getIndexSpec()); - Assert.assertEquals(new IndexSpec(), actual.getIndexSpecForIntermediatePersists()); + Assert.assertEquals(IndexSpec.DEFAULT, actual.getIndexSpec()); + Assert.assertEquals(IndexSpec.DEFAULT, actual.getIndexSpecForIntermediatePersists()); Assert.assertEquals(100, actual.getMaxRowsInMemory()); Assert.assertEquals(true, actual.isLeaveIntermediate()); Assert.assertEquals(true, actual.isCleanupOnFailure()); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/index/RealtimeAppenderatorTuningConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/index/RealtimeAppenderatorTuningConfig.java index d91daa6d92a6..95154acf1590 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/index/RealtimeAppenderatorTuningConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/index/RealtimeAppenderatorTuningConfig.java @@ -42,7 +42,7 @@ public class RealtimeAppenderatorTuningConfig implements AppenderatorConfig private static final Period DEFAULT_INTERMEDIATE_PERSIST_PERIOD = new Period("PT10M"); private static final int DEFAULT_MAX_PENDING_PERSISTS = 0; private static final ShardSpec DEFAULT_SHARD_SPEC = new NumberedShardSpec(0, 1); - private static final IndexSpec DEFAULT_INDEX_SPEC = new IndexSpec(); + private static final IndexSpec DEFAULT_INDEX_SPEC = IndexSpec.DEFAULT; private static final Boolean DEFAULT_REPORT_PARSE_EXCEPTIONS = Boolean.FALSE; private static final long DEFAULT_HANDOFF_CONDITION_TIMEOUT = 0; private static final long DEFAULT_ALERT_TIMEOUT = 0; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java index 737cec3409fe..5e3d7b65fbfb 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/IndexTask.java @@ -1236,7 +1236,7 @@ public boolean isDropExisting() public static class IndexTuningConfig implements AppenderatorConfig { - private static final IndexSpec DEFAULT_INDEX_SPEC = new IndexSpec(); + private static final IndexSpec DEFAULT_INDEX_SPEC = IndexSpec.DEFAULT; private static final int DEFAULT_MAX_PENDING_PERSISTS = 0; private static final boolean DEFAULT_GUARANTEE_ROLLUP = false; private static final boolean DEFAULT_REPORT_PARSE_EXCEPTIONS = false; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/TestIndexTask.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/TestIndexTask.java index 0dbc6f718c8a..ef9a67562eaf 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/TestIndexTask.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/TestIndexTask.java @@ -77,7 +77,7 @@ public TestIndexTask( null, null, new DynamicPartitionsSpec(10000, null), - new IndexSpec(), + IndexSpec.DEFAULT, null, 3, false, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AppenderatorsTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AppenderatorsTest.java index c7b312f190b4..62258ceb48dc 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AppenderatorsTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/AppenderatorsTest.java @@ -169,7 +169,7 @@ public AppenderatorTester( maxRowsInMemory, maxSizeInBytes == 0L ? getDefaultMaxBytesInMemory() : maxSizeInBytes, skipBytesInMemoryOverheadCheck, - new IndexSpec(), + IndexSpec.DEFAULT, 0, false, 0L, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/BatchAppenderatorsTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/BatchAppenderatorsTest.java index 313a99682205..d0ba1de3ce16 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/BatchAppenderatorsTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/BatchAppenderatorsTest.java @@ -174,7 +174,7 @@ public AppenderatorTester( maxRowsInMemory, maxSizeInBytes == 0L ? getDefaultMaxBytesInMemory() : maxSizeInBytes, skipBytesInMemoryOverheadCheck, - new IndexSpec(), + IndexSpec.DEFAULT, 0, false, 0L, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java index c6b6d6fdf0a4..e1dff0e42989 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java @@ -54,7 +54,6 @@ import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.data.BitmapSerde.DefaultBitmapSerdeFactory; import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; @@ -101,18 +100,16 @@ public void testClientCompactionTaskQueryToCompactionTask() throws IOException null, new SegmentsSplitHintSpec(new HumanReadableBytes(100000L), 10), new DynamicPartitionsSpec(100, 30000L), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.UNCOMPRESSED, - LongEncodingStrategy.AUTO - ), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.UNCOMPRESSED) + .withLongEncoding(LongEncodingStrategy.AUTO) + .build(), 2, 1000L, TmpFileSegmentWriteOutMediumFactory.instance(), @@ -260,18 +257,16 @@ public void testCompactionTaskToClientCompactionTaskQuery() throws IOException null, new SegmentsSplitHintSpec(new HumanReadableBytes(100000L), 10), new DynamicPartitionsSpec(100, 30000L), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.UNCOMPRESSED, - LongEncodingStrategy.AUTO - ), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.UNCOMPRESSED) + .withLongEncoding(LongEncodingStrategy.AUTO) + .build(), 2, null, null, @@ -322,18 +317,16 @@ public void testCompactionTaskToClientCompactionTaskQuery() throws IOException 30000L, new SegmentsSplitHintSpec(new HumanReadableBytes(100000L), 10), new DynamicPartitionsSpec(100, 30000L), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.UNCOMPRESSED, - LongEncodingStrategy.AUTO - ), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.UNCOMPRESSED) + .withLongEncoding(LongEncodingStrategy.AUTO) + .build(), 2, 1000L, TmpFileSegmentWriteOutMediumFactory.instance(), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java index e18f852e367d..9a345f122348 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskRunTest.java @@ -210,7 +210,7 @@ public static CompactionState getDefaultCompactionState( new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))), ImmutableList.of(expectedLongSumMetric), null, - mapper.readValue(mapper.writeValueAsString(new IndexSpec()), Map.class), + IndexSpec.DEFAULT.asMap(mapper), mapper.readValue( mapper.writeValueAsString( new UniformGranularitySpec( @@ -773,7 +773,7 @@ public void testCompactionWithFilterInTransformSpec() throws Exception new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))), ImmutableList.of(expectedLongSumMetric), getObjectMapper().readValue(getObjectMapper().writeValueAsString(compactionTask.getTransformSpec()), Map.class), - mapper.readValue(mapper.writeValueAsString(new IndexSpec()), Map.class), + IndexSpec.DEFAULT.asMap(mapper), mapper.readValue( mapper.writeValueAsString( new UniformGranularitySpec( @@ -837,7 +837,7 @@ public void testCompactionWithNewMetricInMetricsSpec() throws Exception new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))), ImmutableList.of(expectedCountMetric, expectedLongSumMetric), getObjectMapper().readValue(getObjectMapper().writeValueAsString(compactionTask.getTransformSpec()), Map.class), - mapper.readValue(mapper.writeValueAsString(new IndexSpec()), Map.class), + IndexSpec.DEFAULT.asMap(mapper), mapper.readValue( mapper.writeValueAsString( new UniformGranularitySpec( diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index 676fd86eed75..ae546db2a226 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -325,12 +325,12 @@ private static CompactionTask.CompactionTuningConfig createTuningConfig() null, null, null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -625,12 +625,12 @@ public void testSerdeWithOldTuningConfigSuccessfullyDeserializeToNewOne() throws null, null, null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -707,12 +707,12 @@ public void testGetTuningConfigWithIndexTuningConfig() null, null, null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -738,12 +738,12 @@ public void testGetTuningConfigWithIndexTuningConfig() null, null, null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -783,12 +783,12 @@ public void testGetTuningConfigWithParallelIndexTuningConfig() null, null, null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -822,12 +822,12 @@ public void testGetTuningConfigWithParallelIndexTuningConfig() null, null, null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -972,12 +972,12 @@ public void testCreateIngestionSchemaWithTargetPartitionSize() throws IOExceptio null, null, null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -1048,12 +1048,12 @@ public void testCreateIngestionSchemaWithMaxTotalRows() throws IOException null, null, null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, false, @@ -1124,12 +1124,12 @@ public void testCreateIngestionSchemaWithNumShards() throws IOException null, null, new HashedPartitionsSpec(null, 3, null), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -1784,12 +1784,12 @@ private void assertIngestionSchema( null, null, new HashedPartitionsSpec(5000000, null, null), // automatically computed targetPartitionSize - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTuningConfigTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTuningConfigTest.java index 2370d4c94ac5..ace69c69433b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTuningConfigTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTuningConfigTest.java @@ -78,13 +78,13 @@ public void testSerdeWithNonZeroAwaitSegmentAvailabilityTimeoutMillis() null, null, new DynamicPartitionsSpec(100, 100L), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, false, true, @@ -120,13 +120,13 @@ public void testSerdeWithZeroAwaitSegmentAvailabilityTimeoutMillis() null, null, new DynamicPartitionsSpec(100, 100L), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, false, true, @@ -163,13 +163,13 @@ public void testSerdeWithNullAwaitSegmentAvailabilityTimeoutMillis() null, null, new DynamicPartitionsSpec(100, 100L), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, false, true, @@ -196,6 +196,11 @@ public void testSerdeWithNullAwaitSegmentAvailabilityTimeoutMillis() public void testEqualsAndHashCode() { EqualsVerifier.forClass(CompactionTask.CompactionTuningConfig.class) + .withPrefabValues( + IndexSpec.class, + IndexSpec.DEFAULT, + IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build() + ) .usingGetClass() .verify(); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskSerdeTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskSerdeTest.java index 29faed025b94..04075ea2b157 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskSerdeTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskSerdeTest.java @@ -68,12 +68,12 @@ public void testSerdeTuningConfigWithDynamicPartitionsSpec() throws IOException null, null, new DynamicPartitionsSpec(1000, 2000L), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, false, @@ -105,12 +105,12 @@ public void testSerdeTuningConfigWithHashedPartitionsSpec() throws IOException null, null, new HashedPartitionsSpec(null, 10, ImmutableList.of("dim1", "dim2")), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -142,12 +142,12 @@ public void testSerdeTuningConfigWithDeprecatedDynamicPartitionsSpec() throws IO null, null, null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, false, @@ -179,12 +179,12 @@ public void testSerdeTuningConfigWithDeprecatedHashedPartitionsSpec() throws IOE 10, ImmutableList.of("dim1", "dim2"), null, - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, false, @@ -218,12 +218,12 @@ public void testForceGuaranteedRollupWithDynamicPartitionsSpec() null, null, new DynamicPartitionsSpec(1000, 2000L), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, true, @@ -256,12 +256,12 @@ public void testBestEffortRollupWithHashedPartitionsSpec() null, null, new HashedPartitionsSpec(null, 10, ImmutableList.of("dim1", "dim2")), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), null, null, false, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java index e90b40bf21e9..459c29db37ac 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java @@ -73,6 +73,7 @@ import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.QueryableIndexStorageAdapter; import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.handoff.SegmentHandoffNotifier; import org.apache.druid.segment.handoff.SegmentHandoffNotifierFactory; import org.apache.druid.segment.incremental.RowIngestionMeters; @@ -171,7 +172,7 @@ public static Iterable constructorFeeder() ); } - private static final IndexSpec INDEX_SPEC = new IndexSpec(); + private static final IndexSpec INDEX_SPEC = IndexSpec.DEFAULT; private final ObjectMapper jsonMapper; private final IndexIO indexIO; private final RowIngestionMetersFactory rowIngestionMetersFactory; @@ -2983,7 +2984,12 @@ public void emit(Event event) public void testEqualsAndHashCode() { EqualsVerifier.forClass(IndexTuningConfig.class) - .usingGetClass() - .verify(); + .withPrefabValues( + IndexSpec.class, + IndexSpec.DEFAULT, + IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build() + ) + .usingGetClass() + .verify(); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskSerdeTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskSerdeTest.java index 7bc6548dc209..d3deefc9ae78 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskSerdeTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/TaskSerdeTest.java @@ -65,7 +65,7 @@ public class TaskSerdeTest { private final ObjectMapper jsonMapper; - private final IndexSpec indexSpec = new IndexSpec(); + private final IndexSpec indexSpec = IndexSpec.DEFAULT; @Rule public ExpectedException thrown = ExpectedException.none(); @@ -106,7 +106,7 @@ public void testIndexTaskTuningConfigDefaults() throws Exception ); Assert.assertFalse(tuningConfig.isReportParseExceptions()); - Assert.assertEquals(new IndexSpec(), tuningConfig.getIndexSpec()); + Assert.assertEquals(IndexSpec.DEFAULT, tuningConfig.getIndexSpec()); Assert.assertEquals(new Period(Integer.MAX_VALUE), tuningConfig.getIntermediatePersistPeriod()); Assert.assertEquals(0, tuningConfig.getMaxPendingPersists()); Assert.assertEquals(1000000, tuningConfig.getMaxRowsInMemory()); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java index 02966e623bf7..2518c1053d16 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTaskTest.java @@ -246,13 +246,13 @@ public void testFailToConstructWhenBothAppendToExistingAndForceGuaranteedRollupA null, null, new HashedPartitionsSpec(null, 10, null), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, forceGuaranteedRollup, true, @@ -318,13 +318,13 @@ public void testFailToConstructWhenBothInputSourceAndParserAreSet() null, null, new HashedPartitionsSpec(null, 10, null), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, true, true, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTuningConfigTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTuningConfigTest.java index 6bbe5fe2f976..380f826c18f8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTuningConfigTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTuningConfigTest.java @@ -78,13 +78,13 @@ public void testSerdeWithMaxRowsPerSegment() null, null, new DynamicPartitionsSpec(100, 100L), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, false, true, @@ -125,13 +125,13 @@ public void testSerdeWithMaxNumConcurrentSubTasks() throws IOException null, null, new DynamicPartitionsSpec(100, 100L), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, false, true, @@ -172,13 +172,13 @@ public void testSerdeWithMaxNumSubTasks() throws IOException null, null, new DynamicPartitionsSpec(100, 100L), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, false, true, @@ -221,13 +221,13 @@ public void testSerdeWithMaxNumSubTasksAndMaxNumConcurrentSubTasks() null, null, new DynamicPartitionsSpec(100, 100L), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, false, true, @@ -267,13 +267,13 @@ public void testConstructorWithHashedPartitionsSpecAndNonForceGuaranteedRollupFa null, null, new HashedPartitionsSpec(null, 10, null), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, forceGuaranteedRollup, true, @@ -313,13 +313,13 @@ public void testConstructorWithSingleDimensionPartitionsSpecAndNonForceGuarantee null, null, new SingleDimensionPartitionsSpec(100, null, "dim", false), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, forceGuaranteedRollup, true, @@ -359,13 +359,13 @@ public void testConstructorWithDynamicPartitionsSpecAndForceGuaranteedRollupFail null, null, new DynamicPartitionsSpec(100, null), - new IndexSpec( - RoaringBitmapSerdeFactory.getInstance(), - CompressionStrategy.UNCOMPRESSED, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec(), + IndexSpec.builder() + .withBitmapSerdeFactory(RoaringBitmapSerdeFactory.getInstance()) + .withDimensionCompression(CompressionStrategy.UNCOMPRESSED) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.DEFAULT, 1, forceGuaranteedRollup, true, @@ -392,7 +392,12 @@ public void testConstructorWithDynamicPartitionsSpecAndForceGuaranteedRollupFail public void testEqualsAndHashCode() { EqualsVerifier.forClass(ParallelIndexTuningConfig.class) - .usingGetClass() - .verify(); + .usingGetClass() + .withPrefabValues( + IndexSpec.class, + IndexSpec.DEFAULT, + IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build() + ) + .verify(); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/input/DruidSegmentReaderTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/input/DruidSegmentReaderTest.java index e65b7c8c71a1..e2bae93e6751 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/input/DruidSegmentReaderTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/input/DruidSegmentReaderTest.java @@ -793,7 +793,7 @@ private void persistSegment(List rows) throws IOException ).persist( incrementalIndex, segmentDirectory, - new IndexSpec(), + IndexSpec.DEFAULT, null ); segmentSize = FileUtils.getFileSize(segmentDirectory); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java index 585d52539d42..6b5927bf1791 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java @@ -415,7 +415,7 @@ public void setUp() throws Exception // initialize variables announcedSinks = 0; pushedSegments = 0; - indexSpec = new IndexSpec(); + indexSpec = IndexSpec.DEFAULT; emitter = newMockEmitter(); EmittingLogger.registerEmitter(emitter); mapper = TEST_UTILS.getTestObjectMapper(); diff --git a/processing/src/main/java/org/apache/druid/segment/IndexSpec.java b/processing/src/main/java/org/apache/druid/segment/IndexSpec.java index d216659e543f..59ad870f3fad 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexSpec.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexSpec.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; import org.apache.druid.segment.column.StringEncodingStrategy; import org.apache.druid.segment.data.BitmapSerde; import org.apache.druid.segment.data.BitmapSerdeFactory; @@ -44,6 +43,13 @@ */ public class IndexSpec { + public static IndexSpec DEFAULT = IndexSpec.builder().build(); + + public static Builder builder() + { + return new Builder(); + } + private final BitmapSerdeFactory bitmapSerdeFactory; private final CompressionStrategy dimensionCompression; private final StringEncodingStrategy stringDictionaryEncoding; @@ -56,37 +62,6 @@ public class IndexSpec @Nullable private final SegmentizerFactory segmentLoader; - /** - * Creates an IndexSpec with default parameters - */ - public IndexSpec() - { - this(null, null, null, null, null, null, null); - } - - @VisibleForTesting - public IndexSpec( - @Nullable BitmapSerdeFactory bitmapSerdeFactory, - @Nullable CompressionStrategy dimensionCompression, - @Nullable CompressionStrategy metricCompression, - @Nullable CompressionFactory.LongEncodingStrategy longEncoding - ) - { - this(bitmapSerdeFactory, dimensionCompression, null, metricCompression, longEncoding, null, null); - } - - @VisibleForTesting - public IndexSpec( - @Nullable BitmapSerdeFactory bitmapSerdeFactory, - @Nullable CompressionStrategy dimensionCompression, - @Nullable CompressionStrategy metricCompression, - @Nullable CompressionFactory.LongEncodingStrategy longEncoding, - @Nullable SegmentizerFactory segmentLoader - ) - { - this(bitmapSerdeFactory, dimensionCompression, null, metricCompression, longEncoding, null, segmentLoader); - } - /** * Creates an IndexSpec with the given storage format settings. * @@ -232,4 +207,76 @@ public String toString() ", segmentLoader=" + segmentLoader + '}'; } + + public static class Builder + { + @Nullable + private BitmapSerdeFactory bitmapSerdeFactory; + @Nullable + private CompressionStrategy dimensionCompression; + @Nullable + private StringEncodingStrategy stringDictionaryEncoding; + @Nullable + private CompressionStrategy metricCompression; + @Nullable + private CompressionFactory.LongEncodingStrategy longEncoding; + @Nullable + private CompressionStrategy jsonCompression; + @Nullable + private SegmentizerFactory segmentLoader; + + public Builder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFactory) + { + this.bitmapSerdeFactory = bitmapSerdeFactory; + return this; + } + + public Builder withDimensionCompression(CompressionStrategy dimensionCompression) + { + this.dimensionCompression = dimensionCompression; + return this; + } + public Builder withStringDictionaryEncoding(StringEncodingStrategy stringDictionaryEncoding) + { + this.stringDictionaryEncoding = stringDictionaryEncoding; + return this; + } + + public Builder withMetricCompression(CompressionStrategy metricCompression) + { + this.metricCompression = metricCompression; + return this; + } + + public Builder withLongEncoding(CompressionFactory.LongEncodingStrategy longEncoding) + { + this.longEncoding = longEncoding; + return this; + } + + public Builder withJsonCompression(CompressionStrategy jsonCompression) + { + this.jsonCompression = jsonCompression; + return this; + } + + public Builder withSegmentLoader(SegmentizerFactory segmentLoader) + { + this.segmentLoader = segmentLoader; + return this; + } + + public IndexSpec build() + { + return new IndexSpec( + bitmapSerdeFactory, + dimensionCompression, + stringDictionaryEncoding, + metricCompression, + longEncoding, + jsonCompression, + segmentLoader + ); + } + } } diff --git a/processing/src/main/java/org/apache/druid/segment/data/VSizeLongSerde.java b/processing/src/main/java/org/apache/druid/segment/data/VSizeLongSerde.java index ff0316d0f153..d400a13af2f0 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/VSizeLongSerde.java +++ b/processing/src/main/java/org/apache/druid/segment/data/VSizeLongSerde.java @@ -521,14 +521,14 @@ public void getDelta(long[] out, int outPosition, int startIndex, int length, lo } for ( ; i + Byte.SIZE < length; index += Byte.SIZE) { final byte unpack = buffer.get(offset + (index >> 3)); - out[outPosition + i++] = base + (unpack >> 7) & 1; - out[outPosition + i++] = base + (unpack >> 6) & 1; - out[outPosition + i++] = base + (unpack >> 5) & 1; - out[outPosition + i++] = base + (unpack >> 4) & 1; - out[outPosition + i++] = base + (unpack >> 3) & 1; - out[outPosition + i++] = base + (unpack >> 2) & 1; - out[outPosition + i++] = base + (unpack >> 1) & 1; - out[outPosition + i++] = base + unpack & 1; + out[outPosition + i++] = base + ((unpack >> 7) & 1); + out[outPosition + i++] = base + ((unpack >> 6) & 1); + out[outPosition + i++] = base + ((unpack >> 5) & 1); + out[outPosition + i++] = base + ((unpack >> 4) & 1); + out[outPosition + i++] = base + ((unpack >> 3) & 1); + out[outPosition + i++] = base + ((unpack >> 2) & 1); + out[outPosition + i++] = base + ((unpack >> 1) & 1); + out[outPosition + i++] = base + (unpack & 1); } while (i < length) { out[outPosition + i++] = base + get(index++); @@ -592,14 +592,14 @@ public void getDelta(long[] out, int outPosition, int startIndex, int length, lo } for ( ; i + 8 < length; index += 8) { final short unpack = buffer.getShort(offset + (index >> 2)); - out[outPosition + i++] = base + (unpack >> 14) & 3; - out[outPosition + i++] = base + (unpack >> 12) & 3; - out[outPosition + i++] = base + (unpack >> 10) & 3; - out[outPosition + i++] = base + (unpack >> 8) & 3; - out[outPosition + i++] = base + (unpack >> 6) & 3; - out[outPosition + i++] = base + (unpack >> 4) & 3; - out[outPosition + i++] = base + (unpack >> 2) & 3; - out[outPosition + i++] = base + unpack & 3; + out[outPosition + i++] = base + ((unpack >> 14) & 3); + out[outPosition + i++] = base + ((unpack >> 12) & 3); + out[outPosition + i++] = base + ((unpack >> 10) & 3); + out[outPosition + i++] = base + ((unpack >> 8) & 3); + out[outPosition + i++] = base + ((unpack >> 6) & 3); + out[outPosition + i++] = base + ((unpack >> 4) & 3); + out[outPosition + i++] = base + ((unpack >> 2) & 3); + out[outPosition + i++] = base + (unpack & 3); } while (i < length) { out[outPosition + i++] = base + get(index++); @@ -659,18 +659,18 @@ public void getDelta(long[] out, int outPosition, int startIndex, int length, lo // byte align while ((index & 0x1) != 0 && i < length) { - out[outPosition + i++] = base + get(index++) & 0xF; + out[outPosition + i++] = base + (get(index++) & 0xF); } for ( ; i + 8 < length; index += 8) { final int unpack = buffer.getInt(offset + (index >> 1)); - out[outPosition + i++] = base + (unpack >> 28) & 0xF; - out[outPosition + i++] = base + (unpack >> 24) & 0xF; - out[outPosition + i++] = base + (unpack >> 20) & 0xF; - out[outPosition + i++] = base + (unpack >> 16) & 0xF; - out[outPosition + i++] = base + (unpack >> 12) & 0xF; - out[outPosition + i++] = base + (unpack >> 8) & 0xF; - out[outPosition + i++] = base + (unpack >> 4) & 0xF; - out[outPosition + i++] = base + unpack & 0xF; + out[outPosition + i++] = base + ((unpack >> 28) & 0xF); + out[outPosition + i++] = base + ((unpack >> 24) & 0xF); + out[outPosition + i++] = base + ((unpack >> 20) & 0xF); + out[outPosition + i++] = base + ((unpack >> 16) & 0xF); + out[outPosition + i++] = base + ((unpack >> 12) & 0xF); + out[outPosition + i++] = base + ((unpack >> 8) & 0xF); + out[outPosition + i++] = base + ((unpack >> 4) & 0xF); + out[outPosition + i++] = base + (unpack & 0xF); } while (i < length) { out[outPosition + i++] = base + get(index++); @@ -725,7 +725,7 @@ public long get(int index) public void getDelta(long[] out, int outPosition, int startIndex, int length, long base) { for (int i = 0, indexOffset = startIndex; i < length; i++, indexOffset++) { - out[outPosition + i] = base + buffer.get(offset + indexOffset) & 0xFF; + out[outPosition + i] = base + (buffer.get(offset + indexOffset) & 0xFF); } } @@ -795,7 +795,7 @@ public void getDelta(long[] out, int outPosition, int startIndex, int length, lo int index = startIndex; // every other value is byte aligned if ((index & 0x1) != 0) { - out[outPosition + i++] = get(index++); + out[outPosition + i++] = base + get(index++); } final int unpackSize = Long.BYTES + Integer.BYTES; for (int indexOffset = (index * 3) >> 1; i + 8 < length; indexOffset += unpackSize) { @@ -838,7 +838,7 @@ public long get(int index) public void getDelta(long[] out, int outPosition, int startIndex, int length, long base) { for (int i = 0, indexOffset = (startIndex << 1); i < length; i++, indexOffset += Short.BYTES) { - out[outPosition + i] = base + buffer.getShort(offset + indexOffset) & 0xFFFF; + out[outPosition + i] = base + (buffer.getShort(offset + indexOffset) & 0xFFFF); } } @@ -851,7 +851,7 @@ public int getDelta(long[] out, int outPosition, int[] indexes, int length, int return i; } - out[outPosition + i] = base + buffer.getShort(offset + (index << 1)) & 0xFFFF; + out[outPosition + i] = base + (buffer.getShort(offset + (index << 1)) & 0xFFFF); } return length; @@ -884,7 +884,7 @@ public void getDelta(long[] out, int outPosition, int startIndex, int length, lo int index = startIndex; // every other value is byte aligned if ((index & 0x1) != 0) { - out[outPosition + i++] = get(index++); + out[outPosition + i++] = base + get(index++); } final int unpackSize = Long.BYTES + Long.BYTES + Integer.BYTES; for (int indexOffset = (index * 5) >> 1; i + 8 < length; indexOffset += unpackSize) { @@ -970,7 +970,7 @@ public long get(int index) public void getDelta(long[] out, int outPosition, int startIndex, int length, long base) { for (int i = 0, indexOffset = (startIndex << 2); i < length; i++, indexOffset += Integer.BYTES) { - out[outPosition + i] = base + buffer.getInt(offset + indexOffset) & 0xFFFFFFFFL; + out[outPosition + i] = base + (buffer.getInt(offset + indexOffset) & 0xFFFFFFFFL); } } } diff --git a/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java b/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java index 9dc8ee684756..f71a6511aa0a 100644 --- a/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java +++ b/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java @@ -358,7 +358,7 @@ private static QueryableIndex buildIndex(String storeDoubleAsFloat) throws IOExc File someTmpFile = File.createTempFile("billy", "yay"); someTmpFile.delete(); FileUtils.mkdirp(someTmpFile); - INDEX_MERGER_V9.persist(index, someTmpFile, new IndexSpec(), null); + INDEX_MERGER_V9.persist(index, someTmpFile, IndexSpec.DEFAULT, null); someTmpFile.delete(); return INDEX_IO.loadIndex(someTmpFile); } diff --git a/processing/src/test/java/org/apache/druid/query/MultiValuedDimensionTest.java b/processing/src/test/java/org/apache/druid/query/MultiValuedDimensionTest.java index dafa7ff9a82d..60d826abb9df 100644 --- a/processing/src/test/java/org/apache/druid/query/MultiValuedDimensionTest.java +++ b/processing/src/test/java/org/apache/druid/query/MultiValuedDimensionTest.java @@ -174,7 +174,7 @@ public void setup() throws Exception persistedSegmentDir = FileUtils.createTempDir(); TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory) - .persist(incrementalIndex, persistedSegmentDir, new IndexSpec(), null); + .persist(incrementalIndex, persistedSegmentDir, IndexSpec.DEFAULT, null); queryableIndex = TestHelper.getTestIndexIO().loadIndex(persistedSegmentDir); @@ -207,7 +207,7 @@ public void setup() throws Exception } persistedSegmentDirNullSampler = FileUtils.createTempDir(); TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory) - .persist(incrementalIndexNullSampler, persistedSegmentDirNullSampler, new IndexSpec(), null); + .persist(incrementalIndexNullSampler, persistedSegmentDirNullSampler, IndexSpec.DEFAULT, null); queryableIndexNullSampler = TestHelper.getTestIndexIO().loadIndex(persistedSegmentDirNullSampler); } diff --git a/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java b/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java index a4801be561bb..1d8b9a0e6554 100644 --- a/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java +++ b/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java @@ -210,7 +210,7 @@ public static List createSimpleNestedTestDataTsvSegments( COUNT, granularity, rollup, - new IndexSpec() + IndexSpec.DEFAULT ); } @@ -236,7 +236,7 @@ public static List createSimpleNestedTestDataSegments( SIMPLE_DATA_FILE, Granularities.NONE, true, - new IndexSpec() + IndexSpec.DEFAULT ); } @@ -322,7 +322,7 @@ public static List createSegmentsWithConcatenatedJsonInput( COUNT, granularity, rollup, - new IndexSpec() + IndexSpec.DEFAULT ); } @@ -505,7 +505,7 @@ public List apply(TemporaryFolder tempFolder, Closer closer) tempFolder, closer, jsonInputFile, - new IndexSpec() + IndexSpec.DEFAULT ) ) .add(NestedDataTestUtils.createIncrementalIndexForJsonInput(tempFolder, jsonInputFile)) @@ -556,7 +556,7 @@ public List apply(TemporaryFolder tempFolder, Closer closer) tempFolder, closer, jsonInputFile, - new IndexSpec() + IndexSpec.DEFAULT ) ) .addAll( @@ -564,7 +564,7 @@ public List apply(TemporaryFolder tempFolder, Closer closer) tempFolder, closer, jsonInputFile, - new IndexSpec() + IndexSpec.DEFAULT ) ) .build(); @@ -592,15 +592,11 @@ public List apply(TemporaryFolder tempFolder, Closer closer) tempFolder, closer, jsonInputFile, - new IndexSpec( - null, - null, - new StringEncodingStrategy.FrontCoded(4, (byte) 0x01), - null, - null, - null, - null - ) + IndexSpec.builder() + .withStringDictionaryEncoding( + new StringEncodingStrategy.FrontCoded(4, (byte) 0x01) + ) + .build() ) ) .addAll( @@ -608,15 +604,11 @@ public List apply(TemporaryFolder tempFolder, Closer closer) tempFolder, closer, jsonInputFile, - new IndexSpec( - null, - null, - new StringEncodingStrategy.FrontCoded(4, (byte) 0x00), - null, - null, - null, - null - ) + IndexSpec.builder() + .withStringDictionaryEncoding( + new StringEncodingStrategy.FrontCoded(4, (byte) 0x00) + ) + .build() ) ) .build(); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java index 65a59fc528e3..236159e25719 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java @@ -543,7 +543,7 @@ public void createIndex( if (!index.canAppendRow()) { File tmp = tempFolder.newFolder(); toMerge.add(tmp); - indexMerger.persist(index, tmp, new IndexSpec(), null); + indexMerger.persist(index, tmp, IndexSpec.DEFAULT, null); index.close(); index = new OnheapIncrementalIndex.Builder() .setIndexSchema( @@ -572,19 +572,19 @@ public void createIndex( if (toMerge.size() > 0) { File tmp = tempFolder.newFolder(); toMerge.add(tmp); - indexMerger.persist(index, tmp, new IndexSpec(), null); + indexMerger.persist(index, tmp, IndexSpec.DEFAULT, null); List indexes = new ArrayList<>(toMerge.size()); for (File file : toMerge) { indexes.add(indexIO.loadIndex(file)); } - indexMerger.mergeQueryableIndex(indexes, rollup, metrics, outDir, new IndexSpec(), null, -1); + indexMerger.mergeQueryableIndex(indexes, rollup, metrics, outDir, IndexSpec.DEFAULT, null, -1); for (QueryableIndex qi : indexes) { qi.close(); } } else { - indexMerger.persist(index, outDir, new IndexSpec(), null); + indexMerger.persist(index, outDir, IndexSpec.DEFAULT, null); } } finally { @@ -679,7 +679,7 @@ public Segment persistIncrementalIndex( if (outDir == null) { outDir = tempFolder.newFolder(); } - indexMerger.persist(index, outDir, new IndexSpec(), null); + indexMerger.persist(index, outDir, IndexSpec.DEFAULT, null); return new QueryableIndexSegment(indexIO.loadIndex(outDir), SegmentId.dummy("")); } diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java index 88f3ea260bbf..3378a494b29d 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java @@ -209,7 +209,7 @@ public void setup() throws Exception final File fileA = INDEX_MERGER_V9.persist( indexA, new File(tmpDir, "A"), - new IndexSpec(), + IndexSpec.DEFAULT, OffHeapMemorySegmentWriteOutMediumFactory.instance() ); QueryableIndex qindexA = INDEX_IO.loadIndex(fileA); @@ -251,7 +251,7 @@ public void setup() throws Exception final File fileB = INDEX_MERGER_V9.persist( indexB, new File(tmpDir, "B"), - new IndexSpec(), + IndexSpec.DEFAULT, OffHeapMemorySegmentWriteOutMediumFactory.instance() ); QueryableIndex qindexB = INDEX_IO.loadIndex(fileB); diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java index 25b76e9f52a1..831fe8e391e3 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java @@ -206,7 +206,7 @@ public void setup() throws Exception final File fileA = INDEX_MERGER_V9.persist( indexA, new File(tmpDir, "A"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexA = INDEX_IO.loadIndex(fileA); @@ -242,7 +242,7 @@ public void setup() throws Exception final File fileB = INDEX_MERGER_V9.persist( indexB, new File(tmpDir, "B"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexB = INDEX_IO.loadIndex(fileB); @@ -277,7 +277,7 @@ public void setup() throws Exception final File fileC = INDEX_MERGER_V9.persist( indexC, new File(tmpDir, "C"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexC = INDEX_IO.loadIndex(fileC); @@ -313,7 +313,7 @@ public void setup() throws Exception final File fileD = INDEX_MERGER_V9.persist( indexD, new File(tmpDir, "D"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexD = INDEX_IO.loadIndex(fileD); @@ -428,7 +428,7 @@ public void setup() throws Exception final File fileE = INDEX_MERGER_V9.persist( indexE, new File(tmpDir, "E"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexE = INDEX_IO.loadIndex(fileE); @@ -523,7 +523,7 @@ public void setup() throws Exception final File fileF = INDEX_MERGER_V9.persist( indexF, new File(tmpDir, "F"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexF = INDEX_IO.loadIndex(fileF); diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByMultiSegmentTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByMultiSegmentTest.java index ef68abbbbb70..2098a2fffe9b 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByMultiSegmentTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByMultiSegmentTest.java @@ -167,7 +167,7 @@ public void setup() throws Exception final File fileA = INDEX_MERGER_V9.persist( indexA, new File(tmpDir, "A"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexA = INDEX_IO.loadIndex(fileA); @@ -189,7 +189,7 @@ public void setup() throws Exception final File fileB = INDEX_MERGER_V9.persist( indexB, new File(tmpDir, "B"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexB = INDEX_IO.loadIndex(fileB); diff --git a/processing/src/test/java/org/apache/druid/query/groupby/NestedQueryPushDownTest.java b/processing/src/test/java/org/apache/druid/query/groupby/NestedQueryPushDownTest.java index f96805506377..f04dacff2c85 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/NestedQueryPushDownTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/NestedQueryPushDownTest.java @@ -191,7 +191,7 @@ public void setup() throws Exception final File fileA = INDEX_MERGER_V9.persist( indexA, new File(tmpDir, "A"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexA = INDEX_IO.loadIndex(fileA); @@ -235,7 +235,7 @@ public void setup() throws Exception final File fileB = INDEX_MERGER_V9.persist( indexB, new File(tmpDir, "B"), - new IndexSpec(), + IndexSpec.DEFAULT, null ); QueryableIndex qindexB = INDEX_IO.loadIndex(fileB); diff --git a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java index ff469de7bcb2..87798ebc18fe 100644 --- a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java @@ -137,7 +137,7 @@ public void testIngestAndScanSegmentsRollup() throws Exception NestedDataTestUtils.COUNT, Granularities.YEAR, true, - new IndexSpec() + IndexSpec.DEFAULT ) ).build(); @@ -340,7 +340,7 @@ public void testIngestWithMergesAndScanSegments() throws Exception NestedDataTestUtils.SIMPLE_DATA_FILE, Granularities.HOUR, true, - new IndexSpec() + IndexSpec.DEFAULT ); final Sequence seq = helper.runQueryOnSegmentsObjs(segs, scanQuery); @@ -524,7 +524,7 @@ public void testIngestAndScanSegmentsRealtimeSchemaDiscovery() throws Exception NestedDataTestUtils.COUNT, Granularities.DAY, true, - new IndexSpec() + IndexSpec.DEFAULT ); @@ -584,7 +584,7 @@ public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryArrayTypes() throws aggs, Granularities.NONE, true, - new IndexSpec() + IndexSpec.DEFAULT ); diff --git a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java index 8e452c4431cc..7715a7eeef66 100644 --- a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java @@ -82,7 +82,7 @@ public void testDefaultSegmentizerPersist() throws IOException data, Intervals.of("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"), segment, - new IndexSpec(), + IndexSpec.DEFAULT, null ); @@ -101,13 +101,7 @@ public void testCustomSegmentizerPersist() throws IOException data, Intervals.of("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"), segment, - new IndexSpec( - null, - null, - null, - null, - new CustomSegmentizerFactory() - ), + IndexSpec.builder().withSegmentLoader(new CustomSegmentizerFactory()).build(), null ); diff --git a/processing/src/test/java/org/apache/druid/segment/EmptyIndexTest.java b/processing/src/test/java/org/apache/druid/segment/EmptyIndexTest.java index 1df510303c21..c9026a177c2c 100644 --- a/processing/src/test/java/org/apache/druid/segment/EmptyIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/EmptyIndexTest.java @@ -90,7 +90,7 @@ public void testEmptyIndex() throws Exception new AggregatorFactory[0], tmpDir, DimensionsSpec.EMPTY, - new IndexSpec(), + IndexSpec.DEFAULT, -1 ); diff --git a/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java b/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java index 95a863dbb017..0fbd20a9a86e 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java @@ -73,7 +73,7 @@ public class IndexBuilder private SegmentWriteOutMediumFactory segmentWriteOutMediumFactory = OffHeapMemorySegmentWriteOutMediumFactory.instance(); private IndexMerger indexMerger; private File tmpDir; - private IndexSpec indexSpec = new IndexSpec(); + private IndexSpec indexSpec = IndexSpec.DEFAULT; private int maxRows = DEFAULT_MAX_ROWS; private int intermediatePersistSize = ROWS_PER_INDEX_FOR_MERGING; private IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() diff --git a/processing/src/test/java/org/apache/druid/segment/IndexIONullColumnsCompatibilityTest.java b/processing/src/test/java/org/apache/druid/segment/IndexIONullColumnsCompatibilityTest.java index e5f555a31029..b11417dc361e 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexIONullColumnsCompatibilityTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexIONullColumnsCompatibilityTest.java @@ -100,7 +100,7 @@ public void setup() throws IOException segmentDir = indexMerger.persist( incrementalIndex, temporaryFolder.newFolder(), - new IndexSpec(), + IndexSpec.DEFAULT, OffHeapMemorySegmentWriteOutMediumFactory.instance() ); } diff --git a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java index 8c1495c85dbe..03fac9ebcfdc 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java @@ -66,12 +66,12 @@ public class IndexIOTest extends InitializedNullHandlingTest { private static Interval DEFAULT_INTERVAL = Intervals.of("1970-01-01/2000-01-01"); - private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( - new ConciseBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.LZ4, - CompressionFactory.LongEncodingStrategy.LONGS - ); + private static final IndexSpec INDEX_SPEC = IndexSpec.builder() + .withBitmapSerdeFactory(new ConciseBitmapSerdeFactory()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZ4) + .withLongEncoding(CompressionFactory.LongEncodingStrategy.LONGS) + .build(); static { NullHandling.initializeForTests(); diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java index 44948bbbd89c..6568da52c4fb 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerNullHandlingTest.java @@ -74,7 +74,7 @@ public void setUp() { indexMerger = TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()); indexIO = TestHelper.getTestIndexIO(); - indexSpec = new IndexSpec(); + indexSpec = IndexSpec.DEFAULT; } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java index 82d6a7f62536..fd99ae96e944 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java @@ -58,7 +58,7 @@ public void setUp() indexMerger = TestHelper .getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()); indexIO = TestHelper.getTestIndexIO(); - indexSpec = new IndexSpec(); + indexSpec = IndexSpec.DEFAULT; } private void testStringFirstLastRollup( diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java index 3d0713ec8fd4..5a80e18a69e7 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java @@ -114,25 +114,6 @@ public Object[] apply(List input) ); } - static IndexSpec makeIndexSpec( - BitmapSerdeFactory bitmapSerdeFactory, - CompressionStrategy compressionStrategy, - CompressionStrategy dimCompressionStrategy, - CompressionFactory.LongEncodingStrategy longEncodingStrategy - ) - { - if (bitmapSerdeFactory != null || compressionStrategy != null) { - return new IndexSpec( - bitmapSerdeFactory, - dimCompressionStrategy, - compressionStrategy, - longEncodingStrategy - ); - } else { - return new IndexSpec(); - } - } - static BitmapValues getBitmapIndex(QueryableIndexIndexableAdapter adapter, String dimension, String value) { final ColumnHolder columnHolder = adapter.getQueryableIndex().getColumnHolder(dimension); @@ -170,12 +151,12 @@ protected IndexMergerTestBase( CompressionFactory.LongEncodingStrategy longEncodingStrategy ) { - this.indexSpec = makeIndexSpec( - bitmapSerdeFactory != null ? bitmapSerdeFactory : new ConciseBitmapSerdeFactory(), - compressionStrategy, - dimCompressionStrategy, - longEncodingStrategy - ); + this.indexSpec = IndexSpec.builder() + .withBitmapSerdeFactory(bitmapSerdeFactory != null ? bitmapSerdeFactory : new ConciseBitmapSerdeFactory()) + .withDimensionCompression(dimCompressionStrategy) + .withMetricCompression(compressionStrategy) + .withLongEncoding(longEncodingStrategy) + .build(); this.indexIO = TestHelper.getTestIndexIO(); this.useBitmapIndexes = bitmapSerdeFactory != null; } @@ -516,18 +497,20 @@ public void testMergeSpecChange() throws Exception Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(3, index1.getColumnNames().size()); - IndexSpec newSpec = new IndexSpec( - indexSpec.getBitmapSerdeFactory(), - CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? - CompressionStrategy.LZF : - CompressionStrategy.LZ4, - CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? - CompressionStrategy.LZF : - CompressionStrategy.LZ4, - CompressionFactory.LongEncodingStrategy.LONGS.equals(indexSpec.getLongEncoding()) ? - CompressionFactory.LongEncodingStrategy.AUTO : - CompressionFactory.LongEncodingStrategy.LONGS - ); + IndexSpec.Builder builder = IndexSpec.builder().withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()); + if (CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression())) { + builder.withDimensionCompression(CompressionStrategy.LZF) + .withMetricCompression(CompressionStrategy.LZF); + } else { + builder.withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZ4); + } + if (CompressionFactory.LongEncodingStrategy.LONGS.equals(indexSpec.getLongEncoding())) { + builder.withLongEncoding(CompressionFactory.LongEncodingStrategy.AUTO); + } else { + builder.withLongEncoding(CompressionFactory.LongEncodingStrategy.LONGS); + } + IndexSpec newSpec = builder.build(); AggregatorFactory[] mergedAggregators = new AggregatorFactory[]{new CountAggregatorFactory("count")}; QueryableIndex merged = closer.closeLater( diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerV9WithSpatialIndexTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerV9WithSpatialIndexTest.java index 106e21bfe267..71a8bcefc60d 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerV9WithSpatialIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerV9WithSpatialIndexTest.java @@ -89,7 +89,7 @@ public static Collection constructorFeeder() throws IOException IndexMergerV9 indexMergerV9 = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); IndexIO indexIO = TestHelper.getTestIndexIO(); - final IndexSpec indexSpec = new IndexSpec(); + final IndexSpec indexSpec = IndexSpec.DEFAULT; final IncrementalIndex rtIndex = makeIncrementalIndex(); final QueryableIndex mMappedTestIndex = makeQueryableIndex(indexSpec, indexMergerV9, indexIO); final QueryableIndex mergedRealtimeIndex = makeMergedQueryableIndex(indexSpec, indexMergerV9, indexIO); diff --git a/processing/src/test/java/org/apache/druid/segment/IndexSpecTest.java b/processing/src/test/java/org/apache/druid/segment/IndexSpecTest.java index 50f1c6f0ede7..261281cec0b1 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexSpecTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexSpecTest.java @@ -66,17 +66,17 @@ public void testSerdeUncompressed() throws Exception @Test public void testDefaults() { - final IndexSpec spec = new IndexSpec(); + final IndexSpec spec = IndexSpec.DEFAULT; Assert.assertEquals(CompressionStrategy.LZ4, spec.getDimensionCompression()); Assert.assertEquals(CompressionStrategy.LZ4, spec.getMetricCompression()); - Assert.assertEquals(CompressionFactory.LongEncodingStrategy.LONGS, spec.getLongEncoding()); + Assert.assertEquals(LongEncodingStrategy.LONGS, spec.getLongEncoding()); } @Test public void testAsMap() { final ObjectMapper objectMapper = new DefaultObjectMapper(); - final IndexSpec spec = new IndexSpec(); + final IndexSpec spec = IndexSpec.DEFAULT; final Map map = spec.asMap(objectMapper); Assert.assertEquals( spec.getBitmapSerdeFactory(), @@ -99,6 +99,15 @@ public void testAsMap() @Test public void testEquals() { - EqualsVerifier.forClass(IndexSpec.class).usingGetClass().verify(); + EqualsVerifier.forClass(IndexSpec.class) + .withPrefabValues( + IndexSpec.class, + IndexSpec.DEFAULT, + IndexSpec.builder() + .withJsonCompression(CompressionStrategy.ZSTD) + .build() + ) + .usingGetClass() + .verify(); } } diff --git a/processing/src/test/java/org/apache/druid/segment/QueryableIndexIndexableAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/QueryableIndexIndexableAdapterTest.java index 6b9117c3cb30..da99dd763eba 100644 --- a/processing/src/test/java/org/apache/druid/segment/QueryableIndexIndexableAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/QueryableIndexIndexableAdapterTest.java @@ -43,13 +43,12 @@ @RunWith(Parameterized.class) public class QueryableIndexIndexableAdapterTest { - private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( - new ConciseBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.LZ4, - CompressionFactory.LongEncodingStrategy.LONGS - ); - + private static final IndexSpec INDEX_SPEC = IndexSpec.builder() + .withBitmapSerdeFactory(new ConciseBitmapSerdeFactory()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZ4) + .withLongEncoding(CompressionFactory.LongEncodingStrategy.LONGS) + .build(); @Parameterized.Parameters public static Collection constructorFeeder() diff --git a/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java b/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java index a44a04cc6ef1..f75472c9ea51 100644 --- a/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/SchemalessIndexTest.java @@ -76,7 +76,7 @@ public class SchemalessIndexTest new CountAggregatorFactory("count") }; - private static final IndexSpec INDEX_SPEC = new IndexSpec(); + private static final IndexSpec INDEX_SPEC = IndexSpec.DEFAULT; private static final List> EVENTS = new ArrayList<>(); diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index 7415fbc63c38..83b1e7fa6363 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -146,7 +146,7 @@ public class TestIndex new DoubleMaxAggregatorFactory(DOUBLE_METRICS[2], VIRTUAL_COLUMNS.getVirtualColumns()[0].getOutputName()), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; - public static final IndexSpec INDEX_SPEC = new IndexSpec(); + public static final IndexSpec INDEX_SPEC = IndexSpec.DEFAULT; public static final IndexMerger INDEX_MERGER = TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()); @@ -215,15 +215,11 @@ public class TestIndex private static Supplier frontCodedMmappedIndex = Suppliers.memoize( () -> persistRealtimeAndLoadMMapped( realtimeIndex.get(), - new IndexSpec( - null, - null, - new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1), - null, - null, - null, - null - ) + IndexSpec.builder() + .withStringDictionaryEncoding( + new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1) + ) + .build() ) ); diff --git a/processing/src/test/java/org/apache/druid/segment/data/VSizeLongSerdeTest.java b/processing/src/test/java/org/apache/druid/segment/data/VSizeLongSerdeTest.java index 6168769efd17..dfd05be6fd07 100644 --- a/processing/src/test/java/org/apache/druid/segment/data/VSizeLongSerdeTest.java +++ b/processing/src/test/java/org/apache/druid/segment/data/VSizeLongSerdeTest.java @@ -247,6 +247,22 @@ private static void testContiguousGetSingleRow( values[i], out[outPosition] ); + + int delta = 100_000; + deserializer.getDelta(out, outPosition, i, 1, delta); + Assert.assertEquals( + StringUtils.format("Deserializer (testContiguousGetSingleRow, numBits = %d, position = %d)", numBits, i), + values[i] + delta, + out[outPosition] + ); + + deserializer.getDelta(out, outPosition, i, 1, -delta); + + Assert.assertEquals( + StringUtils.format("Deserializer (testContiguousGetSingleRow, numBits = %d, position = %d)", numBits, i), + values[i] - delta, + out[outPosition] + ); } } @@ -266,6 +282,27 @@ private static void testContiguousGetWholeRegion( values, Arrays.stream(out).skip(outPosition).toArray() ); + + final long[] valuesPlus = new long[values.length]; + final long[] valuesMinus = new long[values.length]; + final int delta = 100_000; + for (int i = 0; i < values.length; i++) { + valuesPlus[i] = values[i] + delta; + valuesMinus[i] = values[i] - delta; + } + deserializer.getDelta(out, outPosition, 0, values.length, delta); + Assert.assertArrayEquals( + StringUtils.format("Deserializer (testContiguousGetWholeRegion, numBits = %d)", numBits), + valuesPlus, + Arrays.stream(out).skip(outPosition).toArray() + ); + + deserializer.getDelta(out, outPosition, 0, values.length, -delta); + Assert.assertArrayEquals( + StringUtils.format("Deserializer (testContiguousGetWholeRegion, numBits = %d)", numBits), + valuesMinus, + Arrays.stream(out).skip(outPosition).toArray() + ); } private static void testNoncontiguousGetSingleRow( @@ -291,6 +328,23 @@ private static void testNoncontiguousGetSingleRow( values[i], out[outPosition] ); + + int delta = 100_000; + deserializer.getDelta(out, outPosition, indexes, 1, indexOffset, values.length, delta); + + Assert.assertEquals( + StringUtils.format("Deserializer (testNoncontiguousGetSingleRow, numBits = %d, position = %d)", numBits, i), + values[i] + delta, + out[outPosition] + ); + + deserializer.getDelta(out, outPosition, indexes, 1, indexOffset, values.length, -delta); + + Assert.assertEquals( + StringUtils.format("Deserializer (testNoncontiguousGetSingleRow, numBits = %d, position = %d)", numBits, i), + values[i] - delta, + out[outPosition] + ); } } @@ -304,17 +358,24 @@ private static void testNoncontiguousGetEveryOtherValue( final int outPosition = 1; final long[] out = new long[values.length + outPosition]; final long[] expectedOut = new long[values.length + outPosition]; + final long[] expectedOutDeltaPlus = new long[values.length + outPosition]; + final long[] expectedOutDeltaMinus = new long[values.length + outPosition]; final int[] indexes = new int[values.length + outPosition]; Arrays.fill(out, -1); Arrays.fill(expectedOut, -1); + Arrays.fill(expectedOutDeltaPlus, -1); + Arrays.fill(expectedOutDeltaMinus, -1); Arrays.fill(indexes, -1); + final int delta = 100_000; int cnt = 0; for (int i = 0; i < values.length; i++) { if (i % 2 == 0) { indexes[outPosition + i / 2] = i + indexOffset; expectedOut[outPosition + i / 2] = values[i]; + expectedOutDeltaPlus[outPosition + i / 2] = values[i] + delta; + expectedOutDeltaMinus[outPosition + i / 2] = values[i] - delta; cnt++; } } @@ -326,6 +387,22 @@ private static void testNoncontiguousGetEveryOtherValue( expectedOut, out ); + + deserializer.getDelta(out, outPosition, indexes, cnt, indexOffset, values.length, delta); + + Assert.assertArrayEquals( + StringUtils.format("Deserializer (testNoncontiguousGetEveryOtherValue, numBits = %d)", numBits), + expectedOutDeltaPlus, + out + ); + + deserializer.getDelta(out, outPosition, indexes, cnt, indexOffset, values.length, -delta); + + Assert.assertArrayEquals( + StringUtils.format("Deserializer (testNoncontiguousGetEveryOtherValue, numBits = %d)", numBits), + expectedOutDeltaMinus, + out + ); } private static void testNoncontiguousGetEveryOtherValueWithLimit( @@ -338,11 +415,16 @@ private static void testNoncontiguousGetEveryOtherValueWithLimit( final int outPosition = 1; final long[] out = new long[values.length + outPosition]; final long[] expectedOut = new long[values.length + outPosition]; + final long[] expectedOutDeltaPlus = new long[values.length + outPosition]; + final long[] expectedOutDeltaMinus = new long[values.length + outPosition]; final int[] indexes = new int[values.length + outPosition]; final int limit = values.length - 2; // Don't do the last value + final int delta = 100_000; Arrays.fill(out, -1); Arrays.fill(expectedOut, -1); + Arrays.fill(expectedOutDeltaPlus, -1); + Arrays.fill(expectedOutDeltaMinus, -1); Arrays.fill(indexes, -1); int cnt = 0; @@ -352,13 +434,15 @@ private static void testNoncontiguousGetEveryOtherValueWithLimit( if (i < limit) { expectedOut[outPosition + i / 2] = values[i]; + expectedOutDeltaPlus[outPosition + i / 2] = values[i] + delta; + expectedOutDeltaMinus[outPosition + i / 2] = values[i] - delta; } cnt++; } } - final int ret = deserializer.getDelta(out, outPosition, indexes, cnt, indexOffset, limit, 0); + int ret = deserializer.getDelta(out, outPosition, indexes, cnt, indexOffset, limit, 0); Assert.assertArrayEquals( StringUtils.format("Deserializer (testNoncontiguousGetEveryOtherValue, numBits = %d)", numBits), @@ -367,5 +451,25 @@ private static void testNoncontiguousGetEveryOtherValueWithLimit( ); Assert.assertEquals(Math.max(0, cnt - 1), ret); + + ret = deserializer.getDelta(out, outPosition, indexes, cnt, indexOffset, limit, delta); + + Assert.assertArrayEquals( + StringUtils.format("Deserializer (testNoncontiguousGetEveryOtherValue, numBits = %d)", numBits), + expectedOutDeltaPlus, + out + ); + + Assert.assertEquals(Math.max(0, cnt - 1), ret); + + ret = deserializer.getDelta(out, outPosition, indexes, cnt, indexOffset, limit, -delta); + + Assert.assertArrayEquals( + StringUtils.format("Deserializer (testNoncontiguousGetEveryOtherValue, numBits = %d)", numBits), + expectedOutDeltaMinus, + out + ); + + Assert.assertEquals(Math.max(0, cnt - 1), ret); } } diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java index 99ab89103125..017ec103f08b 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java @@ -375,15 +375,10 @@ public static Collection makeConstructors() .create() .schema(DEFAULT_INDEX_SCHEMA) .indexSpec( - new IndexSpec( - bitmapSerdeFactoryEntry.getValue(), - null, - encodingStrategy, - null, - null, - null, - null - ) + IndexSpec.builder() + .withBitmapSerdeFactory(bitmapSerdeFactoryEntry.getValue()) + .withStringDictionaryEncoding(encodingStrategy) + .build() ) .segmentWriteOutMediumFactory(segmentWriteOutMediumFactoryEntry.getValue()); constructors.add(new Object[]{testName, indexBuilder, finisherEntry.getValue(), cnf, optimize}); diff --git a/processing/src/test/java/org/apache/druid/segment/filter/SpatialFilterBonusTest.java b/processing/src/test/java/org/apache/druid/segment/filter/SpatialFilterBonusTest.java index cf60399f7996..cbe2dd39e11f 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/SpatialFilterBonusTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/SpatialFilterBonusTest.java @@ -103,7 +103,7 @@ public static Collection constructorFeeder() throws IOException for (SegmentWriteOutMediumFactory segmentWriteOutMediumFactory : SegmentWriteOutMediumFactory.builtInFactories()) { IndexMerger indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); IndexIO indexIO = TestHelper.getTestIndexIO(); - final IndexSpec indexSpec = new IndexSpec(); + final IndexSpec indexSpec = IndexSpec.DEFAULT; final IncrementalIndex rtIndex = makeIncrementalIndex(); final QueryableIndex mMappedTestIndex = makeQueryableIndex(indexSpec, indexMerger, indexIO); final QueryableIndex mergedRealtimeIndex = makeMergedQueryableIndex(indexSpec, indexMerger, indexIO); diff --git a/processing/src/test/java/org/apache/druid/segment/filter/SpatialFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/SpatialFilterTest.java index 4310e4ff6afb..b5becd7563bd 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/SpatialFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/SpatialFilterTest.java @@ -96,16 +96,11 @@ public class SpatialFilterTest extends InitializedNullHandlingTest @Parameterized.Parameters public static Collection constructorFeeder() throws IOException { - final IndexSpec indexSpec = new IndexSpec(); - final IndexSpec frontCodedIndexSpec = new IndexSpec( - null, - null, - new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1), - null, - null, - null, - null - ); + final IndexSpec indexSpec = IndexSpec.DEFAULT; + final IndexSpec frontCodedIndexSpec = + IndexSpec.builder() + .withStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)) + .build(); final IncrementalIndex rtIndex = makeIncrementalIndex(); final QueryableIndex mMappedTestIndex = makeQueryableIndex(indexSpec); final QueryableIndex mergedRealtimeIndex = makeMergedQueryableIndex(indexSpec); diff --git a/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java b/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java index 9f3851bd64c3..5c03410ad1d3 100644 --- a/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java +++ b/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java @@ -112,7 +112,7 @@ public QueryableIndex generate( final int numRows ) { - return generate(dataSegment, schemaInfo, schemaInfo.getDimensionsSpec(), TransformSpec.NONE, new IndexSpec(), granularity, numRows); + return generate(dataSegment, schemaInfo, schemaInfo.getDimensionsSpec(), TransformSpec.NONE, IndexSpec.DEFAULT, granularity, numRows); } public QueryableIndex generate( diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexAdapterTest.java index 096d5a0aef13..57718db098d8 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexAdapterTest.java @@ -47,12 +47,13 @@ @RunWith(Parameterized.class) public class IncrementalIndexAdapterTest extends InitializedNullHandlingTest { - private static final IndexSpec INDEX_SPEC = new IndexSpec( - new ConciseBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.LZ4, - CompressionFactory.LongEncodingStrategy.LONGS - ); + private static final IndexSpec INDEX_SPEC = + IndexSpec.builder() + .withBitmapSerdeFactory(new ConciseBitmapSerdeFactory()) + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZ4) + .withLongEncoding(CompressionFactory.LongEncodingStrategy.LONGS) + .build(); public final IncrementalIndexCreator indexCreator; diff --git a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java index 93f8c488fe5f..8b11984354ae 100644 --- a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java +++ b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java @@ -120,7 +120,7 @@ public void setup() throws IOException, SegmentLoadingException data, testInterval, segment, - new IndexSpec(), + IndexSpec.DEFAULT, null ); File factoryJson = new File(persisted, "factory.json"); diff --git a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java index 9e4b2ea278c3..3f4f5ae9db7e 100644 --- a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java @@ -91,13 +91,7 @@ public void testSegmentizer() throws IOException, SegmentLoadingException data, testInterval, segment, - new IndexSpec( - null, - null, - null, - null, - expectedFactory - ), + IndexSpec.builder().withSegmentLoader(expectedFactory).build(), null ); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java index b771d4411a1e..aa01998bb259 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java @@ -195,7 +195,7 @@ private SmooshedFileMapper smooshify( try (final FileSmoosher smoosher = new FileSmoosher(tmpFile)) { NestedDataColumnSerializer serializer = new NestedDataColumnSerializer( fileNameBase, - new IndexSpec(), + IndexSpec.DEFAULT, writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), closer ); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java index f8d9dc4033d3..a3f523bb9eab 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java @@ -167,7 +167,7 @@ private SmooshedFileMapper smooshify( try (final FileSmoosher smoosher = new FileSmoosher(tmpFile)) { NestedDataColumnSerializerV4 serializer = new NestedDataColumnSerializerV4( fileNameBase, - new IndexSpec(), + IndexSpec.DEFAULT, writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), closer ); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java index 2917b7eb4f8f..282b403ff9be 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java @@ -338,7 +338,7 @@ private ColumnSelectorFactory getNumericColumnSelectorFactory(VirtualColumns vir NestedDataTestUtils.COUNT, Granularities.NONE, true, - new IndexSpec() + IndexSpec.DEFAULT ); Assert.assertEquals(1, segments.size()); StorageAdapter storageAdapter = segments.get(0).asStorageAdapter(); @@ -369,7 +369,7 @@ private VectorColumnSelectorFactory getVectorColumnSelectorFactory(VirtualColumn NestedDataTestUtils.COUNT, Granularities.NONE, true, - new IndexSpec() + IndexSpec.DEFAULT ); Assert.assertEquals(1, segments.size()); StorageAdapter storageAdapter = segments.get(0).asStorageAdapter(); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java index 9d25216936f7..51d18676ebff 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java @@ -122,7 +122,7 @@ private SmooshedFileMapper smooshify( try (final FileSmoosher smoosher = new FileSmoosher(tmpFile)) { ScalarDoubleColumnSerializer serializer = new ScalarDoubleColumnSerializer( fileNameBase, - new IndexSpec(), + IndexSpec.DEFAULT, writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), closer ); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java index 16c2cfbd0114..a1e13cbf3ab2 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java @@ -122,7 +122,7 @@ private SmooshedFileMapper smooshify( try (final FileSmoosher smoosher = new FileSmoosher(tmpFile)) { ScalarLongColumnSerializer serializer = new ScalarLongColumnSerializer( fileNameBase, - new IndexSpec(), + IndexSpec.DEFAULT, writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), closer ); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java index 926b2a8f7960..430d786d38b2 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java @@ -125,7 +125,7 @@ private SmooshedFileMapper smooshify( try (final FileSmoosher smoosher = new FileSmoosher(tmpFile)) { ScalarStringColumnSerializer serializer = new ScalarStringColumnSerializer( fileNameBase, - new IndexSpec(), + IndexSpec.DEFAULT, writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), closer ); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/VariantArrayColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/VariantArrayColumnSupplierTest.java index 77658495c47a..951d59f456b9 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/VariantArrayColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/VariantArrayColumnSupplierTest.java @@ -119,7 +119,7 @@ private SmooshedFileMapper smooshify( try (final FileSmoosher smoosher = new FileSmoosher(tmpFile)) { VariantArrayColumnSerializer serializer = new VariantArrayColumnSerializer( fileNameBase, - new IndexSpec(), + IndexSpec.DEFAULT, writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), closer ); diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java index 8e0055d44cf3..0a667ee081d7 100644 --- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java @@ -32,10 +32,14 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.Cursor; import org.apache.druid.segment.DeprecatedQueryableIndexColumnSelector; +import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.QueryableIndexStorageAdapter; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.StringEncodingStrategy; +import org.apache.druid.segment.data.CompressionFactory; +import org.apache.druid.segment.data.FrontCodedIndexed; import org.apache.druid.segment.generator.GeneratorBasicSchemas; import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.generator.SegmentGenerator; @@ -46,6 +50,7 @@ import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; +import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -58,7 +63,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; @RunWith(Parameterized.class) public class ExpressionVectorSelectorsTest extends InitializedNullHandlingTest @@ -98,6 +102,7 @@ public class ExpressionVectorSelectorsTest extends InitializedNullHandlingTest private static final int ROWS_PER_SEGMENT = 10_000; private static QueryableIndex INDEX; + private static QueryableIndex INDEX_OTHER_ENCODINGS; private static Closer CLOSER; @BeforeClass @@ -115,10 +120,33 @@ public static void setupClass() .size(0) .build(); + + final SegmentGenerator segmentGenerator = CLOSER.register(new SegmentGenerator()); + INDEX = CLOSER.register( segmentGenerator.generate(dataSegment, schemaInfo, Granularities.HOUR, ROWS_PER_SEGMENT) ); + + + final SegmentGenerator otherGenerator = CLOSER.register(new SegmentGenerator()); + final DataSegment otherSegment = DataSegment.builder() + .dataSource("foo") + .interval(schemaInfo.getDataInterval()) + .version("2") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(); + IndexSpec otherEncodings = IndexSpec.builder() + .withStringDictionaryEncoding( + new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1) + ) + .withLongEncoding(CompressionFactory.LongEncodingStrategy.AUTO) + .build(); + + INDEX_OTHER_ENCODINGS = CLOSER.register( + otherGenerator.generate(otherSegment, schemaInfo, otherEncodings, Granularities.HOUR, ROWS_PER_SEGMENT) + ); } @AfterClass @@ -127,34 +155,57 @@ public static void teardownClass() throws IOException CLOSER.close(); } - @Parameterized.Parameters(name = "expression = {0}") + @Parameterized.Parameters(name = "expression = {0}, encoding = {1}") public static Iterable constructorFeeder() { - return EXPRESSIONS.stream().map(x -> new Object[]{x}).collect(Collectors.toList()); + List params = new ArrayList<>(); + for (String encoding : new String[]{"default", "front-coded-and-auto-longs"}) { + for (String expression : EXPRESSIONS) { + params.add(new Object[]{expression, encoding}); + } + } + return params; } + private String encoding; private ExpressionType outputType; private String expression; - public ExpressionVectorSelectorsTest(String expression) + private QueryableIndex queryableIndexToUse; + private Closer perTestCloser = Closer.create(); + + public ExpressionVectorSelectorsTest(String expression, String encoding) { this.expression = expression; + this.encoding = encoding; + if ("front-coded-and-auto-longs".equals(encoding)) { + this.queryableIndexToUse = INDEX_OTHER_ENCODINGS; + } else { + this.queryableIndexToUse = INDEX; + } } @Before public void setup() { Expr parsed = Parser.parse(expression, ExprMacroTable.nil()); - outputType = parsed.getOutputType(new DeprecatedQueryableIndexColumnSelector(INDEX)); + outputType = parsed.getOutputType(new DeprecatedQueryableIndexColumnSelector(queryableIndexToUse)); if (outputType == null) { outputType = ExpressionType.STRING; } } + @After + public void teardown() throws IOException + { + perTestCloser.close(); + } + + @Test public void sanityTestVectorizedExpressionSelector() { - sanityTestVectorizedExpressionSelectors(expression, outputType, INDEX, CLOSER, ROWS_PER_SEGMENT); + sanityTestVectorizedExpressionSelectors(expression, outputType, queryableIndexToUse, perTestCloser, ROWS_PER_SEGMENT); } public static void sanityTestVectorizedExpressionSelectors( diff --git a/server/src/main/java/org/apache/druid/segment/indexing/RealtimeTuningConfig.java b/server/src/main/java/org/apache/druid/segment/indexing/RealtimeTuningConfig.java index 55ae04bd7aec..b4cda686816d 100644 --- a/server/src/main/java/org/apache/druid/segment/indexing/RealtimeTuningConfig.java +++ b/server/src/main/java/org/apache/druid/segment/indexing/RealtimeTuningConfig.java @@ -49,7 +49,7 @@ public class RealtimeTuningConfig implements AppenderatorConfig private static final RejectionPolicyFactory DEFAULT_REJECTION_POLICY_FACTORY = new ServerTimeRejectionPolicyFactory(); private static final int DEFAULT_MAX_PENDING_PERSISTS = 0; private static final ShardSpec DEFAULT_SHARD_SPEC = new NumberedShardSpec(0, 1); - private static final IndexSpec DEFAULT_INDEX_SPEC = new IndexSpec(); + private static final IndexSpec DEFAULT_INDEX_SPEC = IndexSpec.DEFAULT; private static final Boolean DEFAULT_REPORT_PARSE_EXCEPTIONS = Boolean.FALSE; private static final long DEFAULT_HANDOFF_CONDITION_TIMEOUT = 0; private static final long DEFAULT_ALERT_TIMEOUT = 0; diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIterator.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIterator.java index d746d91c346a..a0a656f38477 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIterator.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIterator.java @@ -374,7 +374,7 @@ private boolean needsCompaction(DataSourceCompactionConfig config, SegmentsToCom final IndexSpec segmentIndexSpec = objectMapper.convertValue(lastCompactionState.getIndexSpec(), IndexSpec.class); final IndexSpec configuredIndexSpec; if (tuningConfig.getIndexSpec() == null) { - configuredIndexSpec = new IndexSpec(); + configuredIndexSpec = IndexSpec.DEFAULT; } else { configuredIndexSpec = tuningConfig.getIndexSpec(); } diff --git a/server/src/test/java/org/apache/druid/indexing/ClientCompactQueryTuningConfigTest.java b/server/src/test/java/org/apache/druid/indexing/ClientCompactQueryTuningConfigTest.java index df2986cdceca..a8ae549df3fd 100644 --- a/server/src/test/java/org/apache/druid/indexing/ClientCompactQueryTuningConfigTest.java +++ b/server/src/test/java/org/apache/druid/indexing/ClientCompactQueryTuningConfigTest.java @@ -21,6 +21,8 @@ import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.data.CompressionStrategy; import org.junit.Test; public class ClientCompactQueryTuningConfigTest @@ -29,6 +31,12 @@ public class ClientCompactQueryTuningConfigTest public void testEqualsContract() { // If this test failed, make sure to validate that toString was also updated correctly! - EqualsVerifier.forClass(ClientCompactionTaskQueryTuningConfig.class).usingGetClass().verify(); + EqualsVerifier.forClass(ClientCompactionTaskQueryTuningConfig.class) + .withPrefabValues( + IndexSpec.class, + IndexSpec.DEFAULT, + IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build() + ).usingGetClass() + .verify(); } } diff --git a/server/src/test/java/org/apache/druid/segment/indexing/RealtimeTuningConfigTest.java b/server/src/test/java/org/apache/druid/segment/indexing/RealtimeTuningConfigTest.java index 23432c975862..535dd108fdf1 100644 --- a/server/src/test/java/org/apache/druid/segment/indexing/RealtimeTuningConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/indexing/RealtimeTuningConfigTest.java @@ -84,8 +84,8 @@ public void testSerdeWithDefaults() throws Exception Assert.assertEquals(new OnheapIncrementalIndex.Spec(), config.getAppendableIndexSpec()); Assert.assertEquals(0, config.getHandoffConditionTimeout()); Assert.assertEquals(0, config.getAlertTimeout()); - Assert.assertEquals(new IndexSpec(), config.getIndexSpec()); - Assert.assertEquals(new IndexSpec(), config.getIndexSpecForIntermediatePersists()); + Assert.assertEquals(IndexSpec.DEFAULT, config.getIndexSpec()); + Assert.assertEquals(IndexSpec.DEFAULT, config.getIndexSpecForIntermediatePersists()); Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod()); Assert.assertEquals(new NumberedShardSpec(0, 1), config.getShardSpec()); Assert.assertEquals(0, config.getMaxPendingPersists()); @@ -137,9 +137,12 @@ public void testSerdeWithNonDefaults() throws Exception Assert.assertEquals(100, config.getPersistThreadPriority()); Assert.assertEquals(new Period("PT1H"), config.getWindowPeriod()); Assert.assertEquals(true, config.isReportParseExceptions()); - Assert.assertEquals(new IndexSpec(null, null, CompressionStrategy.NONE, null), config.getIndexSpec()); Assert.assertEquals( - new IndexSpec(null, CompressionStrategy.UNCOMPRESSED, null, null), + IndexSpec.builder().withMetricCompression(CompressionStrategy.NONE).build(), + config.getIndexSpec() + ); + Assert.assertEquals( + IndexSpec.builder().withDimensionCompression(CompressionStrategy.UNCOMPRESSED).build(), config.getIndexSpecForIntermediatePersists() ); diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/ClosedSegmensSinksBatchAppenderatorTester.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/ClosedSegmensSinksBatchAppenderatorTester.java index 01895f077f8f..858b989acf42 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/ClosedSegmensSinksBatchAppenderatorTester.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/ClosedSegmensSinksBatchAppenderatorTester.java @@ -173,7 +173,7 @@ public ClosedSegmensSinksBatchAppenderatorTester( maxRowsInMemory, maxSizeInBytes == 0L ? getDefaultMaxBytesInMemory() : maxSizeInBytes, skipBytesInMemoryOverheadCheck, - new IndexSpec(), + IndexSpec.DEFAULT, 0, false, 0L, diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/OpenAndClosedSegmentsAppenderatorTester.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/OpenAndClosedSegmentsAppenderatorTester.java index e7e94d3a7e8e..e2357d95c9b5 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/OpenAndClosedSegmentsAppenderatorTester.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/OpenAndClosedSegmentsAppenderatorTester.java @@ -144,7 +144,7 @@ public OpenAndClosedSegmentsAppenderatorTester( maxRowsInMemory, maxSizeInBytes == 0L ? getDefaultMaxBytesInMemory() : maxSizeInBytes, skipBytesInMemoryOverheadCheck, - new IndexSpec(), + IndexSpec.DEFAULT, 0, false, 0L, diff --git a/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java b/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java index ae08ae368692..44f24f879a22 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java @@ -226,7 +226,7 @@ private void createTestIndex(File segmentDir) throws Exception for (String line : rows) { index.add(parser.parse(line)); } - indexMerger.persist(index, segmentDir, new IndexSpec(), null); + indexMerger.persist(index, segmentDir, IndexSpec.DEFAULT, null); } } } diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java index 110e2d2de087..3d54b279df10 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java @@ -348,13 +348,7 @@ private DataSegment createSegment(IncrementalIndex data, String interval, String data, Intervals.of(interval), segmentDir, - new IndexSpec( - null, - null, - null, - null, - factory - ), + IndexSpec.builder().withSegmentLoader(factory).build(), null ); final File factoryJson = new File(segmentDir, "factory.json"); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java index a0f4cf9a6101..e3f8c01c3dcb 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java @@ -33,7 +33,6 @@ import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.data.BitmapSerde.DefaultBitmapSerdeFactory; import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; @@ -223,18 +222,16 @@ public void testSerdeUserCompactionTuningConfig() throws IOException null, new SegmentsSplitHintSpec(new HumanReadableBytes(100000L), null), new DynamicPartitionsSpec(1000, 20000L), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.UNCOMPRESSED, - LongEncodingStrategy.AUTO - ), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZF) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.UNCOMPRESSED) + .withLongEncoding(LongEncodingStrategy.AUTO) + .build(), 2, 1000L, TmpFileSegmentWriteOutMediumFactory.instance(), @@ -264,18 +261,16 @@ public void testSerdeUserCompactionTuningConfigWithAppendableIndexSpec() throws null, new SegmentsSplitHintSpec(new HumanReadableBytes(100000L), null), new DynamicPartitionsSpec(1000, 20000L), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.UNCOMPRESSED, - LongEncodingStrategy.AUTO - ), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZ4) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.UNCOMPRESSED) + .withLongEncoding(LongEncodingStrategy.AUTO) + .build(), 2, 1000L, TmpFileSegmentWriteOutMediumFactory.instance(), diff --git a/server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java index 01c889ad2c3c..a92122e475d7 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java @@ -25,7 +25,6 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.data.BitmapSerde.DefaultBitmapSerdeFactory; import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; @@ -83,18 +82,16 @@ public void testSerde() throws IOException null, new SegmentsSplitHintSpec(new HumanReadableBytes(42L), null), new DynamicPartitionsSpec(1000, 20000L), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.LZF, - LongEncodingStrategy.LONGS - ), - new IndexSpec( - new DefaultBitmapSerdeFactory(), - CompressionStrategy.LZ4, - CompressionStrategy.UNCOMPRESSED, - LongEncodingStrategy.AUTO - ), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZ4) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), + IndexSpec.builder() + .withDimensionCompression(CompressionStrategy.LZ4) + .withMetricCompression(CompressionStrategy.LZ4) + .withLongEncoding(LongEncodingStrategy.LONGS) + .build(), 2, 1000L, TmpFileSegmentWriteOutMediumFactory.instance(), diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java index 73abcb8d5eac..c2e7a05e7be0 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java @@ -699,7 +699,7 @@ public void testIteratorReturnsAllMixedVersionSegmentsInConfiguredSegmentGranula public void testIteratorReturnsNothingAsSegmentsWasCompactedAndHaveSameSegmentGranularityAndSameTimezone() { // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -732,7 +732,7 @@ public void testIteratorReturnsNothingAsSegmentsWasCompactedAndHaveSameSegmentGr public void testIteratorReturnsNothingAsSegmentsWasCompactedAndHaveSameSegmentGranularityInLastCompactionState() { // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -765,7 +765,7 @@ public void testIteratorReturnsNothingAsSegmentsWasCompactedAndHaveSameSegmentGr public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentSegmentGranularity() { // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -808,7 +808,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentSeg public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentSegmentGranularityInLastCompactionState() { // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -851,7 +851,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentSeg public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentTimezone() { // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -903,7 +903,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentTim public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentOrigin() { // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -954,7 +954,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentOri public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentRollup() { // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -1014,7 +1014,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentRol public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentQueryGranularity() { // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -1074,7 +1074,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentQue public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentDimensions() { // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -1173,7 +1173,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentFil { NullHandling.initializeForTests(); // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -1297,7 +1297,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentMet .addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE) ); // Same indexSpec as what is set in the auto compaction config - Map indexSpec = mapper.convertValue(new IndexSpec(), new TypeReference>() {}); + Map indexSpec = IndexSpec.DEFAULT.asMap(mapper); // Same partitionsSpec as what is set in the auto compaction config PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -1446,7 +1446,7 @@ public void testIteratorReturnsSegmentsSmallerSegmentGranularityCoveringMultiple public void testIteratorReturnsSegmentsAsCompactionStateChangedWithCompactedStateHasSameSegmentGranularity() { // Different indexSpec as what is set in the auto compaction config - IndexSpec newIndexSpec = new IndexSpec(new ConciseBitmapSerdeFactory(), null, null, null); + IndexSpec newIndexSpec = IndexSpec.builder().withBitmapSerdeFactory(new ConciseBitmapSerdeFactory()).build(); Map newIndexSpecMap = mapper.convertValue(newIndexSpec, new TypeReference>() {}); PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null, null)); @@ -1508,7 +1508,7 @@ public void testIteratorDoesNotReturnSegmentWithChangingAppendableIndexSpec() null, null, null, - mapper.convertValue(new IndexSpec(), new TypeReference>() {}), + IndexSpec.DEFAULT.asMap(mapper), null ) ) @@ -1528,7 +1528,7 @@ public void testIteratorDoesNotReturnSegmentWithChangingAppendableIndexSpec() 1000L, null, partitionsSpec, - new IndexSpec(), + IndexSpec.DEFAULT, null, null, null, @@ -1563,7 +1563,7 @@ public void testIteratorDoesNotReturnSegmentWithChangingAppendableIndexSpec() 1000L, null, partitionsSpec, - new IndexSpec(), + IndexSpec.DEFAULT, null, null, null, diff --git a/services/src/test/java/org/apache/druid/cli/validate/DruidJsonValidatorTest.java b/services/src/test/java/org/apache/druid/cli/validate/DruidJsonValidatorTest.java index 3b920dfa656b..232c87eb8801 100644 --- a/services/src/test/java/org/apache/druid/cli/validate/DruidJsonValidatorTest.java +++ b/services/src/test/java/org/apache/druid/cli/validate/DruidJsonValidatorTest.java @@ -161,7 +161,7 @@ public void testTaskValidator() throws Exception null, null, new DynamicPartitionsSpec(10000, null), - new IndexSpec(), + IndexSpec.DEFAULT, null, 3, false,