diff --git a/docs/ingestion/ingestion-spec.md b/docs/ingestion/ingestion-spec.md index 18d54fd8e5ad..ee76d9c9f0e7 100644 --- a/docs/ingestion/ingestion-spec.md +++ b/docs/ingestion/ingestion-spec.md @@ -495,18 +495,18 @@ The `indexSpec` object can include the following properties: |-----|-----------|-------| |bitmap|Compression format for bitmap indexes. Should be a JSON object with `type` set to `roaring` or `concise`.|`{"type": "roaring"}`| |dimensionCompression|Compression format for dimension columns. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`| -|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns.

Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`
`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.
`formatVersion` can specify older versions for backwards compatibility during rolling upgrades, valid options are `0` and `1`. Defaults to `0` for backwards compatibility.

See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`| +|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns.

Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`
`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.
`formatVersion` can specify older versions for backwards compatibility during rolling upgrades, valid options are `0` and `1`, defaults to `1`.

See [Front coding](#front-coding) for more information.|`{"type":"frontCoded", "bucketSize": 4, "formatVersion": 1}`| |metricCompression|Compression format for primitive type metric columns. Options are `lz4`, `lzf`, `zstd`, `uncompressed`, or `none` (which is more efficient than `uncompressed`, but not supported by older versions of Druid).|`lz4`| |longEncoding|Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. Options are `auto` or `longs`. `auto` encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. `longs` stores the value as-is with 8 bytes each.|`longs`| |jsonCompression|Compression format to use for nested column raw data. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`| ##### Front coding -Front coding is an experimental feature starting in version 25.0. Front coding is an incremental encoding strategy that Druid can use to store STRING and [COMPLEX<json>](../querying/nested-columns.md) columns. It allows Druid to create smaller UTF-8 encoded segments with very little performance cost. +Front coding is an incremental encoding strategy that Druid uses by default to store STRING and [COMPLEX<json>](../querying/nested-columns.md) columns. It allows Druid to create smaller UTF-8 encoded segments with very little performance cost. -You can enable front coding with all types of ingestion. For information on defining an `indexSpec` in a query context, see [SQL-based ingestion reference](../multi-stage-query/reference.md#context-parameters). +For information on defining an `indexSpec` in a query context, see [SQL-based ingestion reference](../multi-stage-query/reference.md#context-parameters). -> Front coding was originally introduced in Druid 25.0, and an improved 'version 1' was introduced in Druid 26.0, with typically faster read speed and smaller storage size. The current recommendation is to enable it in a staging environment and fully test your use case before using in production. By default, segments created with front coding enabled in Druid 26.0 are backwards compatible with Druid 25.0, but those created with Druid 26.0 or 25.0 are not compatible with Druid versions older than 25.0. If using front coding in Druid 25.0 and upgrading to Druid 26.0, the `formatVersion` defaults to `0` to keep writing out the older format to enable seamless downgrades to Druid 25.0, and then later is recommended to be changed to `1` once determined that rollback is not necessary. +> Front coding was originally introduced in Druid 25.0, and an improved 'version 1' was introduced in Druid 26.0, with typically faster read speed and smaller storage size, before finally becoming the default in Druid 27.0. By default, segments created with Druid 27.0 are backwards compatible with Druid 26.0, but not compatible with Druid versions older than 26.0. If upgrading to Druid 27.0 from a version older than 26.0, the `stringDictionaryEncoding` should be set to `{"type": "utf8"}` to keep writing out the older format to enable seamless downgrades to Druid 25.0 and older, and then later is recommended to be changed to the new default once determined that rollback is not necessary. Beyond these properties, each ingestion method has its own specific tuning properties. See the documentation for each [ingestion method](./index.md#ingestion-methods) for details. diff --git a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java index 3c40affa7834..f672ed0d9dc6 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java @@ -466,8 +466,8 @@ public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception fullDatasourceName, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, 0, - 13702, - 13701, + 13326, + 13325, 0, 2, 2, @@ -484,7 +484,7 @@ public void testAutoCompactionDutySubmitAndVerifyCompaction() throws Exception fullDatasourceName, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, 0, - 21566, + 20906, 0, 0, 3, @@ -600,8 +600,8 @@ public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception getAndAssertCompactionStatus( fullDatasourceName, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, - 13702, - 13701, + 13326, + 13325, 0, 2, 2, @@ -609,7 +609,7 @@ public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception 1, 1, 0); - Assert.assertEquals(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize"), "13702"); + Assert.assertEquals(compactionResource.getCompactionProgress(fullDatasourceName).get("remainingSegmentSize"), "13326"); // Run compaction again to compact the remaining day // Remaining day compacted (1 new segment). Now both days compacted (2 total) forceTriggerAutoCompaction(2); @@ -620,7 +620,7 @@ public void testAutoCompactionDutyCanUpdateTaskSlots() throws Exception fullDatasourceName, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, 0, - 21566, + 20906, 0, 0, 3, diff --git a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java index a8246c9d75d1..8b2a3eda31d4 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java +++ b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java @@ -36,12 +36,13 @@ }) public interface StringEncodingStrategy { - Utf8 DEFAULT = new Utf8(); String UTF8 = "utf8"; String FRONT_CODED = "frontCoded"; - byte UTF8_ID = 0x00; byte FRONT_CODED_ID = 0x01; + int DEFAULT_BUCKET_SIZE = 4; + + StringEncodingStrategy DEFAULT = new FrontCoded(DEFAULT_BUCKET_SIZE, null); String getType(); diff --git a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java index ebbf13a91b09..bb135b0b9a24 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java +++ b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java @@ -79,7 +79,7 @@ public final class FrontCodedIndexed implements Indexed { public static final byte V0 = 0; public static final byte V1 = 1; - public static final byte DEFAULT_VERSION = V0; + public static final byte DEFAULT_VERSION = V1; public static final int DEFAULT_BUCKET_SIZE = 4; public static byte validateVersion(byte version) diff --git a/processing/src/test/java/org/apache/druid/segment/ConciseBitmapIndexMergerV9Test.java b/processing/src/test/java/org/apache/druid/segment/ConciseBitmapIndexMergerV9Test.java index 9bad930b200f..c0e657842599 100644 --- a/processing/src/test/java/org/apache/druid/segment/ConciseBitmapIndexMergerV9Test.java +++ b/processing/src/test/java/org/apache/druid/segment/ConciseBitmapIndexMergerV9Test.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import org.apache.druid.segment.column.StringEncodingStrategy; import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.ConciseBitmapSerdeFactory; @@ -33,6 +34,7 @@ public ConciseBitmapIndexMergerV9Test( CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, LongEncodingStrategy longEncodingStrategy, + StringEncodingStrategy stringEncodingStrategy, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { @@ -40,7 +42,8 @@ public ConciseBitmapIndexMergerV9Test( new ConciseBitmapSerdeFactory(), compressionStrategy, dimCompressionStrategy, - longEncodingStrategy + longEncodingStrategy, + stringEncodingStrategy ); indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); } diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java index 30c41bea70f5..a6466bc77f74 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java @@ -49,12 +49,14 @@ import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.DictionaryEncodedColumn; import org.apache.druid.segment.column.StringUtf8DictionaryEncodedColumn; +import org.apache.druid.segment.column.StringEncodingStrategy; import org.apache.druid.segment.column.StringValueSetIndex; import org.apache.druid.segment.data.BitmapSerdeFactory; import org.apache.druid.segment.data.BitmapValues; import org.apache.druid.segment.data.CompressionFactory; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.ConciseBitmapSerdeFactory; +import org.apache.druid.segment.data.FrontCodedIndexed; import org.apache.druid.segment.data.ImmutableBitmapValues; import org.apache.druid.segment.data.IncrementalIndexTest; import org.apache.druid.segment.incremental.IncrementalIndex; @@ -91,7 +93,7 @@ public class IndexMergerTestBase extends InitializedNullHandlingTest protected IndexMerger indexMerger; - @Parameterized.Parameters(name = "{index}: metric compression={0}, dimension compression={1}, long encoding={2}, segment write-out medium={3}") + @Parameterized.Parameters(name = "{index}: metric compression={0}, dimension compression={1}, long encoding={2}, string encoding={3} segment write-out medium={4}") public static Collection data() { return Collections2.transform( @@ -100,6 +102,11 @@ public static Collection data() EnumSet.allOf(CompressionStrategy.class), ImmutableSet.copyOf(CompressionStrategy.noNoneValues()), EnumSet.allOf(CompressionFactory.LongEncodingStrategy.class), + ImmutableSet.of( + new StringEncodingStrategy.Utf8(), + new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V0), + new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1) + ), SegmentWriteOutMediumFactory.builtInFactories() ) ), new Function, Object[]>() @@ -148,7 +155,8 @@ protected IndexMergerTestBase( @Nullable BitmapSerdeFactory bitmapSerdeFactory, CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, - CompressionFactory.LongEncodingStrategy longEncodingStrategy + CompressionFactory.LongEncodingStrategy longEncodingStrategy, + StringEncodingStrategy stringEncodingStrategy ) { this.indexSpec = IndexSpec.builder() @@ -156,6 +164,7 @@ protected IndexMergerTestBase( .withDimensionCompression(dimCompressionStrategy) .withMetricCompression(compressionStrategy) .withLongEncoding(longEncodingStrategy) + .withStringDictionaryEncoding(stringEncodingStrategy) .build(); this.indexIO = TestHelper.getTestIndexIO(); this.useBitmapIndexes = bitmapSerdeFactory != null; @@ -510,6 +519,12 @@ public void testMergeSpecChange() throws Exception } else { builder.withLongEncoding(CompressionFactory.LongEncodingStrategy.LONGS); } + if (StringEncodingStrategy.UTF8_ID == indexSpec.getStringDictionaryEncoding().getId()) { + builder.withStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)); + } else { + builder.withStringDictionaryEncoding(new StringEncodingStrategy.Utf8()); + } + IndexSpec newSpec = builder.build(); AggregatorFactory[] mergedAggregators = new AggregatorFactory[]{new CountAggregatorFactory("count")}; diff --git a/processing/src/test/java/org/apache/druid/segment/NoBitmapIndexMergerV9Test.java b/processing/src/test/java/org/apache/druid/segment/NoBitmapIndexMergerV9Test.java index 8af1a701fd2b..33a89a630df3 100644 --- a/processing/src/test/java/org/apache/druid/segment/NoBitmapIndexMergerV9Test.java +++ b/processing/src/test/java/org/apache/druid/segment/NoBitmapIndexMergerV9Test.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import org.apache.druid.segment.column.StringEncodingStrategy; import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory; @@ -32,6 +33,7 @@ public NoBitmapIndexMergerV9Test( CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, LongEncodingStrategy longEncodingStrategy, + StringEncodingStrategy stringEncodingStrategy, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { @@ -39,7 +41,8 @@ public NoBitmapIndexMergerV9Test( null, compressionStrategy, dimCompressionStrategy, - longEncodingStrategy + longEncodingStrategy, + stringEncodingStrategy ); indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); } diff --git a/processing/src/test/java/org/apache/druid/segment/RoaringBitmapIndexMergerV9Test.java b/processing/src/test/java/org/apache/druid/segment/RoaringBitmapIndexMergerV9Test.java index 164cf6c4834a..3168f91c751d 100644 --- a/processing/src/test/java/org/apache/druid/segment/RoaringBitmapIndexMergerV9Test.java +++ b/processing/src/test/java/org/apache/druid/segment/RoaringBitmapIndexMergerV9Test.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import org.apache.druid.segment.column.StringEncodingStrategy; import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; @@ -33,6 +34,7 @@ public RoaringBitmapIndexMergerV9Test( CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, LongEncodingStrategy longEncodingStrategy, + StringEncodingStrategy stringEncodingStrategy, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { @@ -40,7 +42,8 @@ public RoaringBitmapIndexMergerV9Test( RoaringBitmapSerdeFactory.getInstance(), compressionStrategy, dimCompressionStrategy, - longEncodingStrategy + longEncodingStrategy, + stringEncodingStrategy ); indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); } diff --git a/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java index bd5dcfb39a46..e2b219c0525d 100644 --- a/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java +++ b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java @@ -54,8 +54,7 @@ public void testFrontCodedDefaultSerde() throws JsonProcessingException // this next assert seems silly, but its a sanity check to make us think hard before changing the default version, // to make us think of the backwards compatibility implications, as new versions of segment format stuff cannot be // downgraded to older versions of Druid and still read - // the default version should be changed to V1 after Druid 26.0 is released - Assert.assertEquals(FrontCodedIndexed.V0, FrontCodedIndexed.DEFAULT_VERSION); + Assert.assertEquals(FrontCodedIndexed.V1, FrontCodedIndexed.DEFAULT_VERSION); } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java index 5b86747aa2ae..b99561f64d78 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java @@ -34,44 +34,22 @@ import org.apache.druid.java.util.common.io.smoosh.SmooshedWriter; import org.apache.druid.query.DefaultBitmapResultFactory; import org.apache.druid.query.filter.SelectorPredicateFactory; -import org.apache.druid.segment.AutoTypeColumnIndexer; -import org.apache.druid.segment.AutoTypeColumnMerger; -import org.apache.druid.segment.ColumnValueSelector; -import org.apache.druid.segment.DimensionSelector; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.IndexableAdapter; -import org.apache.druid.segment.SimpleAscendingOffset; -import org.apache.druid.segment.column.ColumnBuilder; -import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.DruidPredicateIndex; -import org.apache.druid.segment.column.NullValueIndex; -import org.apache.druid.segment.column.StringUtf8DictionaryEncodedColumn; -import org.apache.druid.segment.column.StringValueSetIndex; +import org.apache.druid.segment.*; +import org.apache.druid.segment.column.*; import org.apache.druid.segment.data.BitmapSerdeFactory; +import org.apache.druid.segment.data.FrontCodedIndexed; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory; import org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import org.apache.druid.testing.InitializedNullHandlingTest; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Objects; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.TreeSet; +import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; @@ -98,8 +76,10 @@ public class ScalarStringColumnSupplierTest extends InitializedNullHandlingTest Closer closer = Closer.create(); SmooshedFileMapper fileMapper; + SmooshedFileMapper fileMapperUtf8; ByteBuffer baseBuffer; + ByteBuffer baseBufferUtf8; @BeforeClass public static void staticSetup() @@ -111,14 +91,31 @@ public static void staticSetup() public void setup() throws IOException { final String fileNameBase = "test"; - fileMapper = smooshify(fileNameBase, tempFolder.newFolder(), data); + fileMapper = smooshify( + fileNameBase, + tempFolder.newFolder(), + data, + IndexSpec.builder() + .withStringDictionaryEncoding( + new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.DEFAULT_VERSION) + ) + .build() + ); baseBuffer = fileMapper.mapFile(fileNameBase); + fileMapperUtf8 = smooshify( + fileNameBase, + tempFolder.newFolder(), + data, + IndexSpec.builder().withStringDictionaryEncoding(new StringEncodingStrategy.Utf8()).build() + ); + baseBufferUtf8 = fileMapperUtf8.mapFile(fileNameBase); } private SmooshedFileMapper smooshify( String fileNameBase, File tmpFile, - List data + List data, + IndexSpec indexSpec ) throws IOException { @@ -126,7 +123,7 @@ private SmooshedFileMapper smooshify( try (final FileSmoosher smoosher = new FileSmoosher(tmpFile)) { ScalarStringColumnSerializer serializer = new ScalarStringColumnSerializer( fileNameBase, - IndexSpec.DEFAULT, + indexSpec, writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), closer ); @@ -198,6 +195,23 @@ public void testBasicFunctionality() throws IOException } } + @Test + public void testBasicFunctionalityUtf8() throws IOException + { + ColumnBuilder bob = new ColumnBuilder(); + bob.setFileMapper(fileMapperUtf8); + ScalarStringColumnAndIndexSupplier supplier = ScalarStringColumnAndIndexSupplier.read( + ByteOrder.nativeOrder(), + bitmapSerdeFactory, + baseBufferUtf8, + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES + ); + try (StringUtf8DictionaryEncodedColumn column = (StringUtf8DictionaryEncodedColumn) supplier.get()) { + smokeTest(supplier, column); + } + } + @Test public void testConcurrency() throws ExecutionException, InterruptedException { @@ -211,6 +225,15 @@ public void testConcurrency() throws ExecutionException, InterruptedException bob, NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); + ColumnBuilder bobUtf8 = new ColumnBuilder(); + bobUtf8.setFileMapper(fileMapperUtf8); + ScalarStringColumnAndIndexSupplier utf8Supplier = ScalarStringColumnAndIndexSupplier.read( + ByteOrder.nativeOrder(), + bitmapSerdeFactory, + baseBufferUtf8, + bobUtf8, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES + ); final String expectedReason = "none"; final AtomicReference failureReason = new AtomicReference<>(expectedReason); @@ -229,6 +252,9 @@ public void testConcurrency() throws ExecutionException, InterruptedException try (StringUtf8DictionaryEncodedColumn column = (StringUtf8DictionaryEncodedColumn) supplier.get()) { smokeTest(supplier, column); } + try (StringUtf8DictionaryEncodedColumn column = (StringUtf8DictionaryEncodedColumn) utf8Supplier.get()) { + smokeTest(utf8Supplier, column); + } } } catch (Throwable ex) {