From 240ca1b49c8ce42e42425a6870970a8c159b670d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sat, 27 May 2023 15:40:49 -0700 Subject: [PATCH 1/2] switch to front-coded v1 bucket size 4 by default --- docs/ingestion/ingestion-spec.md | 8 +-- .../column/StringEncodingStrategy.java | 5 +- .../druid/segment/data/FrontCodedIndexed.java | 2 +- .../ConciseBitmapIndexMergerV9Test.java | 5 +- .../druid/segment/IndexMergerTestBase.java | 24 +++++-- .../segment/NoBitmapIndexMergerV9Test.java | 5 +- .../RoaringBitmapIndexMergerV9Test.java | 5 +- .../column/StringEncodingStrategyTest.java | 3 +- .../ScalarStringColumnSupplierTest.java | 65 +++++++++++++++++-- .../virtual/DummyStringVirtualColumn.java | 12 ++-- 10 files changed, 106 insertions(+), 28 deletions(-) diff --git a/docs/ingestion/ingestion-spec.md b/docs/ingestion/ingestion-spec.md index e5a2ee062ddd..ba8519426c13 100644 --- a/docs/ingestion/ingestion-spec.md +++ b/docs/ingestion/ingestion-spec.md @@ -495,18 +495,18 @@ The `indexSpec` object can include the following properties: |-----|-----------|-------| |bitmap|Compression format for bitmap indexes. Should be a JSON object with `type` set to `roaring` or `concise`.|`{"type": "roaring"}`| |dimensionCompression|Compression format for dimension columns. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`| -|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns.

Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`
`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.
`formatVersion` can specify older versions for backwards compatibility during rolling upgrades, valid options are `0` and `1`. Defaults to `0` for backwards compatibility.

See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`| +|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns.

Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`
`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.
`formatVersion` can specify older versions for backwards compatibility during rolling upgrades, valid options are `0` and `1`, defaults to `1`.

See [Front coding](#front-coding) for more information.|`{"type":"frontCoded", "bucketSize": 4, "formatVersion": 1}`| |metricCompression|Compression format for primitive type metric columns. Options are `lz4`, `lzf`, `zstd`, `uncompressed`, or `none` (which is more efficient than `uncompressed`, but not supported by older versions of Druid).|`lz4`| |longEncoding|Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. Options are `auto` or `longs`. `auto` encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. `longs` stores the value as-is with 8 bytes each.|`longs`| |jsonCompression|Compression format to use for nested column raw data. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`| ##### Front coding -Front coding is an experimental feature starting in version 25.0. Front coding is an incremental encoding strategy that Druid can use to store STRING and [COMPLEX<json>](../querying/nested-columns.md) columns. It allows Druid to create smaller UTF-8 encoded segments with very little performance cost. +Front coding is an incremental encoding strategy that Druid uses by default to store STRING and [COMPLEX<json>](../querying/nested-columns.md) columns. It allows Druid to create smaller UTF-8 encoded segments with very little performance cost. -You can enable front coding with all types of ingestion. For information on defining an `indexSpec` in a query context, see [SQL-based ingestion reference](../multi-stage-query/reference.md#context-parameters). +For information on defining an `indexSpec` in a query context, see [SQL-based ingestion reference](../multi-stage-query/reference.md#context-parameters). -> Front coding was originally introduced in Druid 25.0, and an improved 'version 1' was introduced in Druid 26.0, with typically faster read speed and smaller storage size. The current recommendation is to enable it in a staging environment and fully test your use case before using in production. By default, segments created with front coding enabled in Druid 26.0 are backwards compatible with Druid 25.0, but those created with Druid 26.0 or 25.0 are not compatible with Druid versions older than 25.0. If using front coding in Druid 25.0 and upgrading to Druid 26.0, the `formatVersion` defaults to `0` to keep writing out the older format to enable seamless downgrades to Druid 25.0, and then later is recommended to be changed to `1` once determined that rollback is not necessary. +> Front coding was originally introduced in Druid 25.0, and an improved 'version 1' was introduced in Druid 26.0, with typically faster read speed and smaller storage size, before finally becoming the default in Druid 27.0. By default, segments created with Druid 27.0 are backwards compatible with Druid 26.0, but not compatible with Druid versions older than 26.0. If upgrading to Druid 27.0 from a version older than 26.0, the `stringDictionaryEncoding` should be set to `{"type": "utf8"}` to keep writing out the older format to enable seamless downgrades to Druid 25.0 and older, and then later is recommended to be changed to the new default once determined that rollback is not necessary. Beyond these properties, each ingestion method has its own specific tuning properties. See the documentation for each [ingestion method](./index.md#ingestion-methods) for details. diff --git a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java index a8246c9d75d1..8b2a3eda31d4 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java +++ b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java @@ -36,12 +36,13 @@ }) public interface StringEncodingStrategy { - Utf8 DEFAULT = new Utf8(); String UTF8 = "utf8"; String FRONT_CODED = "frontCoded"; - byte UTF8_ID = 0x00; byte FRONT_CODED_ID = 0x01; + int DEFAULT_BUCKET_SIZE = 4; + + StringEncodingStrategy DEFAULT = new FrontCoded(DEFAULT_BUCKET_SIZE, null); String getType(); diff --git a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java index ebbf13a91b09..bb135b0b9a24 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java +++ b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java @@ -79,7 +79,7 @@ public final class FrontCodedIndexed implements Indexed { public static final byte V0 = 0; public static final byte V1 = 1; - public static final byte DEFAULT_VERSION = V0; + public static final byte DEFAULT_VERSION = V1; public static final int DEFAULT_BUCKET_SIZE = 4; public static byte validateVersion(byte version) diff --git a/processing/src/test/java/org/apache/druid/segment/ConciseBitmapIndexMergerV9Test.java b/processing/src/test/java/org/apache/druid/segment/ConciseBitmapIndexMergerV9Test.java index 9bad930b200f..c0e657842599 100644 --- a/processing/src/test/java/org/apache/druid/segment/ConciseBitmapIndexMergerV9Test.java +++ b/processing/src/test/java/org/apache/druid/segment/ConciseBitmapIndexMergerV9Test.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import org.apache.druid.segment.column.StringEncodingStrategy; import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.ConciseBitmapSerdeFactory; @@ -33,6 +34,7 @@ public ConciseBitmapIndexMergerV9Test( CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, LongEncodingStrategy longEncodingStrategy, + StringEncodingStrategy stringEncodingStrategy, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { @@ -40,7 +42,8 @@ public ConciseBitmapIndexMergerV9Test( new ConciseBitmapSerdeFactory(), compressionStrategy, dimCompressionStrategy, - longEncodingStrategy + longEncodingStrategy, + stringEncodingStrategy ); indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); } diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java index 5a80e18a69e7..327ee75c4e84 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java @@ -48,13 +48,14 @@ import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.DictionaryEncodedColumn; -import org.apache.druid.segment.column.StringDictionaryEncodedColumn; +import org.apache.druid.segment.column.StringEncodingStrategy; import org.apache.druid.segment.column.StringValueSetIndex; import org.apache.druid.segment.data.BitmapSerdeFactory; import org.apache.druid.segment.data.BitmapValues; import org.apache.druid.segment.data.CompressionFactory; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.ConciseBitmapSerdeFactory; +import org.apache.druid.segment.data.FrontCodedIndexed; import org.apache.druid.segment.data.ImmutableBitmapValues; import org.apache.druid.segment.data.IncrementalIndexTest; import org.apache.druid.segment.incremental.IncrementalIndex; @@ -91,7 +92,7 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest protected IndexMerger indexMerger; - @Parameterized.Parameters(name = "{index}: metric compression={0}, dimension compression={1}, long encoding={2}, segment write-out medium={3}") + @Parameterized.Parameters(name = "{index}: metric compression={0}, dimension compression={1}, long encoding={2}, string encoding={3} segment write-out medium={4}") public static Collection data() { return Collections2.transform( @@ -100,6 +101,11 @@ public static Collection data() EnumSet.allOf(CompressionStrategy.class), ImmutableSet.copyOf(CompressionStrategy.noNoneValues()), EnumSet.allOf(CompressionFactory.LongEncodingStrategy.class), + ImmutableSet.of( + new StringEncodingStrategy.Utf8(), + new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V0), + new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1) + ), SegmentWriteOutMediumFactory.builtInFactories() ) ), new Function, Object[]>() @@ -148,7 +154,8 @@ protected IndexMergerTestBase( @Nullable BitmapSerdeFactory bitmapSerdeFactory, CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, - CompressionFactory.LongEncodingStrategy longEncodingStrategy + CompressionFactory.LongEncodingStrategy longEncodingStrategy, + StringEncodingStrategy stringEncodingStrategy ) { this.indexSpec = IndexSpec.builder() @@ -156,6 +163,7 @@ protected IndexMergerTestBase( .withDimensionCompression(dimCompressionStrategy) .withMetricCompression(compressionStrategy) .withLongEncoding(longEncodingStrategy) + .withStringDictionaryEncoding(stringEncodingStrategy) .build(); this.indexIO = TestHelper.getTestIndexIO(); this.useBitmapIndexes = bitmapSerdeFactory != null; @@ -510,6 +518,12 @@ public void testMergeSpecChange() throws Exception } else { builder.withLongEncoding(CompressionFactory.LongEncodingStrategy.LONGS); } + if (StringEncodingStrategy.UTF8_ID == indexSpec.getStringDictionaryEncoding().getId()) { + builder.withStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)); + } else { + builder.withStringDictionaryEncoding(new StringEncodingStrategy.Utf8()); + } + IndexSpec newSpec = builder.build(); AggregatorFactory[] mergedAggregators = new AggregatorFactory[]{new CountAggregatorFactory("count")}; @@ -548,12 +562,12 @@ private void assertDimCompression(QueryableIndex index, CompressionStrategy expe DictionaryEncodedColumn encodedColumn = (DictionaryEncodedColumn) index.getColumnHolder("dim2").getColumn(); Object obj; if (encodedColumn.hasMultipleValues()) { - Field field = StringDictionaryEncodedColumn.class.getDeclaredField("multiValueColumn"); + Field field = encodedColumn.getClass().getDeclaredField("multiValueColumn"); field.setAccessible(true); obj = field.get(encodedColumn); } else { - Field field = StringDictionaryEncodedColumn.class.getDeclaredField("column"); + Field field = encodedColumn.getClass().getDeclaredField("column"); field.setAccessible(true); obj = field.get(encodedColumn); diff --git a/processing/src/test/java/org/apache/druid/segment/NoBitmapIndexMergerV9Test.java b/processing/src/test/java/org/apache/druid/segment/NoBitmapIndexMergerV9Test.java index 8af1a701fd2b..33a89a630df3 100644 --- a/processing/src/test/java/org/apache/druid/segment/NoBitmapIndexMergerV9Test.java +++ b/processing/src/test/java/org/apache/druid/segment/NoBitmapIndexMergerV9Test.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import org.apache.druid.segment.column.StringEncodingStrategy; import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory; @@ -32,6 +33,7 @@ public NoBitmapIndexMergerV9Test( CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, LongEncodingStrategy longEncodingStrategy, + StringEncodingStrategy stringEncodingStrategy, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { @@ -39,7 +41,8 @@ public NoBitmapIndexMergerV9Test( null, compressionStrategy, dimCompressionStrategy, - longEncodingStrategy + longEncodingStrategy, + stringEncodingStrategy ); indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); } diff --git a/processing/src/test/java/org/apache/druid/segment/RoaringBitmapIndexMergerV9Test.java b/processing/src/test/java/org/apache/druid/segment/RoaringBitmapIndexMergerV9Test.java index 164cf6c4834a..3168f91c751d 100644 --- a/processing/src/test/java/org/apache/druid/segment/RoaringBitmapIndexMergerV9Test.java +++ b/processing/src/test/java/org/apache/druid/segment/RoaringBitmapIndexMergerV9Test.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import org.apache.druid.segment.column.StringEncodingStrategy; import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; @@ -33,6 +34,7 @@ public RoaringBitmapIndexMergerV9Test( CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, LongEncodingStrategy longEncodingStrategy, + StringEncodingStrategy stringEncodingStrategy, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { @@ -40,7 +42,8 @@ public RoaringBitmapIndexMergerV9Test( RoaringBitmapSerdeFactory.getInstance(), compressionStrategy, dimCompressionStrategy, - longEncodingStrategy + longEncodingStrategy, + stringEncodingStrategy ); indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); } diff --git a/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java index bd5dcfb39a46..e2b219c0525d 100644 --- a/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java +++ b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java @@ -54,8 +54,7 @@ public void testFrontCodedDefaultSerde() throws JsonProcessingException // this next assert seems silly, but its a sanity check to make us think hard before changing the default version, // to make us think of the backwards compatibility implications, as new versions of segment format stuff cannot be // downgraded to older versions of Druid and still read - // the default version should be changed to V1 after Druid 26.0 is released - Assert.assertEquals(FrontCodedIndexed.V0, FrontCodedIndexed.DEFAULT_VERSION); + Assert.assertEquals(FrontCodedIndexed.V1, FrontCodedIndexed.DEFAULT_VERSION); } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java index 430d786d38b2..d2762ac0fe6f 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java @@ -43,10 +43,14 @@ import org.apache.druid.segment.SimpleAscendingOffset; import org.apache.druid.segment.column.ColumnBuilder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.DictionaryEncodedColumn; import org.apache.druid.segment.column.DruidPredicateIndex; import org.apache.druid.segment.column.NullValueIndex; +import org.apache.druid.segment.column.StringEncodingStrategy; +import org.apache.druid.segment.column.StringFrontCodedDictionaryEncodedColumn; import org.apache.druid.segment.column.StringValueSetIndex; import org.apache.druid.segment.data.BitmapSerdeFactory; +import org.apache.druid.segment.data.FrontCodedIndexed; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory; import org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; @@ -97,8 +101,10 @@ public class ScalarStringColumnSupplierTest extends InitializedNullHandlingTest Closer closer = Closer.create(); SmooshedFileMapper fileMapper; + SmooshedFileMapper fileMapperUtf8; ByteBuffer baseBuffer; + ByteBuffer baseBufferUtf8; @BeforeClass public static void staticSetup() @@ -110,14 +116,31 @@ public static void staticSetup() public void setup() throws IOException { final String fileNameBase = "test"; - fileMapper = smooshify(fileNameBase, tempFolder.newFolder(), data); + fileMapper = smooshify( + fileNameBase, + tempFolder.newFolder(), + data, + IndexSpec.builder() + .withStringDictionaryEncoding( + new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.DEFAULT_VERSION) + ) + .build() + ); baseBuffer = fileMapper.mapFile(fileNameBase); + fileMapperUtf8 = smooshify( + fileNameBase, + tempFolder.newFolder(), + data, + IndexSpec.builder().withStringDictionaryEncoding(new StringEncodingStrategy.Utf8()).build() + ); + baseBufferUtf8 = fileMapperUtf8.mapFile(fileNameBase); } private SmooshedFileMapper smooshify( String fileNameBase, File tmpFile, - List data + List data, + IndexSpec indexSpec ) throws IOException { @@ -125,7 +148,7 @@ private SmooshedFileMapper smooshify( try (final FileSmoosher smoosher = new FileSmoosher(tmpFile)) { ScalarStringColumnSerializer serializer = new ScalarStringColumnSerializer( fileNameBase, - IndexSpec.DEFAULT, + indexSpec, writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), closer ); @@ -186,6 +209,23 @@ public void testBasicFunctionality() throws IOException bob, NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); + try (StringFrontCodedDictionaryEncodedColumn column = (StringFrontCodedDictionaryEncodedColumn) supplier.get()) { + smokeTest(supplier, column); + } + } + + @Test + public void testBasicFunctionalityUtf8() throws IOException + { + ColumnBuilder bob = new ColumnBuilder(); + bob.setFileMapper(fileMapperUtf8); + ScalarStringColumnAndIndexSupplier supplier = ScalarStringColumnAndIndexSupplier.read( + ByteOrder.nativeOrder(), + bitmapSerdeFactory, + baseBufferUtf8, + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES + ); try (ScalarStringDictionaryEncodedColumn column = (ScalarStringDictionaryEncodedColumn) supplier.get()) { smokeTest(supplier, column); } @@ -204,6 +244,15 @@ public void testConcurrency() throws ExecutionException, InterruptedException bob, NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); + ColumnBuilder bobUtf8 = new ColumnBuilder(); + bobUtf8.setFileMapper(fileMapperUtf8); + ScalarStringColumnAndIndexSupplier utf8Supplier = ScalarStringColumnAndIndexSupplier.read( + ByteOrder.nativeOrder(), + bitmapSerdeFactory, + baseBufferUtf8, + bobUtf8, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES + ); final String expectedReason = "none"; final AtomicReference failureReason = new AtomicReference<>(expectedReason); @@ -219,9 +268,12 @@ public void testConcurrency() throws ExecutionException, InterruptedException try { threadsStartLatch.await(); for (int iter = 0; iter < 5000; iter++) { - try (ScalarStringDictionaryEncodedColumn column = (ScalarStringDictionaryEncodedColumn) supplier.get()) { + try (StringFrontCodedDictionaryEncodedColumn column = (StringFrontCodedDictionaryEncodedColumn) supplier.get()) { smokeTest(supplier, column); } + try (ScalarStringDictionaryEncodedColumn column = (ScalarStringDictionaryEncodedColumn) utf8Supplier.get()) { + smokeTest(utf8Supplier, column); + } } } catch (Throwable ex) { @@ -235,7 +287,10 @@ public void testConcurrency() throws ExecutionException, InterruptedException Assert.assertEquals(expectedReason, failureReason.get()); } - private void smokeTest(ScalarStringColumnAndIndexSupplier supplier, ScalarStringDictionaryEncodedColumn column) + private & NestedCommonFormatColumn> void smokeTest( + ScalarStringColumnAndIndexSupplier supplier, + TColumn column + ) { SimpleAscendingOffset offset = new SimpleAscendingOffset(data.size()); ColumnValueSelector valueSelector = column.makeColumnValueSelector(offset); diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/DummyStringVirtualColumn.java b/processing/src/test/java/org/apache/druid/segment/virtual/DummyStringVirtualColumn.java index 8253b27bb596..86d150dd1ee1 100644 --- a/processing/src/test/java/org/apache/druid/segment/virtual/DummyStringVirtualColumn.java +++ b/processing/src/test/java/org/apache/druid/segment/virtual/DummyStringVirtualColumn.java @@ -38,7 +38,7 @@ import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.StringDictionaryEncodedColumn; +import org.apache.druid.segment.column.DictionaryEncodedColumn; import org.apache.druid.segment.data.IndexedInts; import org.apache.druid.segment.data.ReadableOffset; @@ -95,7 +95,7 @@ public DimensionSelector makeDimensionSelector( return DimensionSelector.constant(null); } - StringDictionaryEncodedColumn stringCol = toStringDictionaryEncodedColumn(holder.getColumn()); + DictionaryEncodedColumn stringCol = toStringDictionaryEncodedColumn(holder.getColumn()); DimensionSelector baseDimensionSelector = stringCol.makeDimensionSelector( offset, @@ -145,7 +145,7 @@ public ColumnValueSelector makeColumnValueSelector( return NilColumnValueSelector.instance(); } - StringDictionaryEncodedColumn stringCol = toStringDictionaryEncodedColumn(holder.getColumn()); + DictionaryEncodedColumn stringCol = toStringDictionaryEncodedColumn(holder.getColumn()); return stringCol.makeColumnValueSelector(offset); } else { return null; @@ -222,13 +222,13 @@ public byte[] getCacheKey() return new byte[0]; } - private StringDictionaryEncodedColumn toStringDictionaryEncodedColumn(BaseColumn column) + private DictionaryEncodedColumn toStringDictionaryEncodedColumn(BaseColumn column) { - if (!(column instanceof StringDictionaryEncodedColumn)) { + if (!(column instanceof DictionaryEncodedColumn)) { throw new IAE("I can only work with StringDictionaryEncodedColumn"); } - return (StringDictionaryEncodedColumn) column; + return (DictionaryEncodedColumn) column; } private DimensionSelector disableValueMatchers(DimensionSelector base) From 44afd0b9a046f628f6604696df05436b87c4cdb3 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 30 May 2023 13:55:53 -0700 Subject: [PATCH 2/2] fix expectations --- .../coordinator/duty/ITAutoCompactionTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java index fccaa83f3e3b..8d2541a37139 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java @@ -458,8 +458,8 @@ public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception fullDatasourceName, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, 0, - 13702, - 13701, + 13326, + 13325, 0, 2, 2, @@ -476,7 +476,7 @@ public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception fullDatasourceName, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, 0, - 21566, + 20906, 0, 0, 3, @@ -592,8 +592,8 @@ public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception getAndAssertCompactionStatus( fullDatasourceName, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, - 13702, - 13701, + 13326, + 13325, 0, 2, 2, @@ -601,7 +601,7 @@ public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception 1, 1, 0); - Assert.assertEquals(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize"), "13702"); + Assert.assertEquals(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize"), "13326"); // Run compaction again to compact the remaining day // Remaining day compacted (1 new segment). Now both days compacted (2 total) forceTriggerAutoCompaction(2); @@ -612,7 +612,7 @@ public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception fullDatasourceName, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, 0, - 21566, + 20906, 0, 0, 3,