diff --git a/api/src/main/java/io/druid/data/input/impl/DimensionSchema.java b/api/src/main/java/io/druid/data/input/impl/DimensionSchema.java index d721cbf0569b..d1a39e27d21b 100644 --- a/api/src/main/java/io/druid/data/input/impl/DimensionSchema.java +++ b/api/src/main/java/io/druid/data/input/impl/DimensionSchema.java @@ -120,11 +120,13 @@ public static MultiValueHandling ofDefault() private final String name; private final MultiValueHandling multiValueHandling; + private final boolean createBitmapIndex; - protected DimensionSchema(String name, MultiValueHandling multiValueHandling) + protected DimensionSchema(String name, MultiValueHandling multiValueHandling, boolean createBitmapIndex) { this.name = Preconditions.checkNotNull(name, "Dimension name cannot be null."); this.multiValueHandling = multiValueHandling == null ? MultiValueHandling.ofDefault() : multiValueHandling; + this.createBitmapIndex = createBitmapIndex; } @JsonProperty @@ -139,6 +141,12 @@ public MultiValueHandling getMultiValueHandling() return multiValueHandling; } + @JsonProperty("createBitmapIndex") + public boolean hasBitmapIndex() + { + return createBitmapIndex; + } + @JsonIgnore public abstract String getTypeName(); @@ -146,7 +154,7 @@ public MultiValueHandling getMultiValueHandling() public abstract ValueType getValueType(); @Override - public boolean equals(Object o) + public boolean equals(final Object o) { if (this == o) { return true; @@ -154,33 +162,29 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - - DimensionSchema that = (DimensionSchema) o; - - if (!name.equals(that.name)) { - return false; - } - - if (!getValueType().equals(that.getValueType())) { - return false; - } - - return Objects.equals(multiValueHandling, that.multiValueHandling); + final DimensionSchema that = (DimensionSchema) o; + return createBitmapIndex == that.createBitmapIndex && + Objects.equals(name, that.name) && + Objects.equals(getTypeName(), that.getTypeName()) && + Objects.equals(getValueType(), that.getValueType()) && + multiValueHandling == that.multiValueHandling; } @Override public int hashCode() { - return Objects.hash(name, getValueType(), multiValueHandling); + return Objects.hash(name, multiValueHandling, createBitmapIndex, getTypeName(), getValueType()); } @Override public String toString() { return "DimensionSchema{" + - "name='" + name + "'" + - ", valueType='" + getValueType() + "'" + - ", multiValueHandling='" + getMultiValueHandling() + "'" + - "}"; + "name='" + name + '\'' + + ", valueType=" + getValueType() + + ", typeName=" + getTypeName() + + ", multiValueHandling=" + multiValueHandling + + ", createBitmapIndex=" + createBitmapIndex + + '}'; } } diff --git a/api/src/main/java/io/druid/data/input/impl/DimensionsSpec.java b/api/src/main/java/io/druid/data/input/impl/DimensionsSpec.java index 70640c3aec1b..dd57c050154c 100644 --- a/api/src/main/java/io/druid/data/input/impl/DimensionsSpec.java +++ b/api/src/main/java/io/druid/data/input/impl/DimensionsSpec.java @@ -36,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @PublicApi public class DimensionsSpec @@ -56,17 +57,9 @@ public static List getDefaultSchemas( final DimensionSchema.MultiValueHandling multiValueHandling ) { - return Lists.transform( - dimNames, - new Function() - { - @Override - public DimensionSchema apply(String input) - { - return new StringDimensionSchema(input, multiValueHandling); - } - } - ); + return dimNames.stream() + .map(input -> new StringDimensionSchema(input, multiValueHandling, true)) + .collect(Collectors.toList()); } public static DimensionSchema convertSpatialSchema(SpatialDimensionSchema spatialSchema) diff --git a/api/src/main/java/io/druid/data/input/impl/DoubleDimensionSchema.java b/api/src/main/java/io/druid/data/input/impl/DoubleDimensionSchema.java index bcd642fe40e4..2f4f403fb86f 100644 --- a/api/src/main/java/io/druid/data/input/impl/DoubleDimensionSchema.java +++ b/api/src/main/java/io/druid/data/input/impl/DoubleDimensionSchema.java @@ -19,7 +19,6 @@ package io.druid.data.input.impl; - import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -28,7 +27,7 @@ public class DoubleDimensionSchema extends DimensionSchema @JsonCreator public DoubleDimensionSchema(@JsonProperty("name") String name) { - super(name, null); + super(name, null, false); } @Override diff --git a/api/src/main/java/io/druid/data/input/impl/FloatDimensionSchema.java b/api/src/main/java/io/druid/data/input/impl/FloatDimensionSchema.java index a457a226ee4b..d1752e7b06f8 100644 --- a/api/src/main/java/io/druid/data/input/impl/FloatDimensionSchema.java +++ b/api/src/main/java/io/druid/data/input/impl/FloatDimensionSchema.java @@ -30,7 +30,7 @@ public FloatDimensionSchema( @JsonProperty("name") String name ) { - super(name, null); + super(name, null, false); } @Override diff --git a/api/src/main/java/io/druid/data/input/impl/LongDimensionSchema.java b/api/src/main/java/io/druid/data/input/impl/LongDimensionSchema.java index 64af529360bc..7d4c417605af 100644 --- a/api/src/main/java/io/druid/data/input/impl/LongDimensionSchema.java +++ b/api/src/main/java/io/druid/data/input/impl/LongDimensionSchema.java @@ -30,7 +30,7 @@ public LongDimensionSchema( @JsonProperty("name") String name ) { - super(name, null); + super(name, null, false); } @Override diff --git a/api/src/main/java/io/druid/data/input/impl/NewSpatialDimensionSchema.java b/api/src/main/java/io/druid/data/input/impl/NewSpatialDimensionSchema.java index c5b823e39a9d..d22ab1319e89 100644 --- a/api/src/main/java/io/druid/data/input/impl/NewSpatialDimensionSchema.java +++ b/api/src/main/java/io/druid/data/input/impl/NewSpatialDimensionSchema.java @@ -43,7 +43,7 @@ public NewSpatialDimensionSchema( @JsonProperty("dims") List dims ) { - super(name, null); + super(name, null, true); this.dims = dims; } diff --git a/api/src/main/java/io/druid/data/input/impl/StringDimensionSchema.java b/api/src/main/java/io/druid/data/input/impl/StringDimensionSchema.java index dd6ffd401638..d00d6fb0c343 100644 --- a/api/src/main/java/io/druid/data/input/impl/StringDimensionSchema.java +++ b/api/src/main/java/io/druid/data/input/impl/StringDimensionSchema.java @@ -25,6 +25,8 @@ public class StringDimensionSchema extends DimensionSchema { + private static final boolean DEFAULT_CREATE_BITMAP_INDEX = true; + @JsonCreator public static StringDimensionSchema create(String name) { @@ -34,15 +36,16 @@ public static StringDimensionSchema create(String name) @JsonCreator public StringDimensionSchema( @JsonProperty("name") String name, - @JsonProperty("multiValueHandling") MultiValueHandling multiValueHandling + @JsonProperty("multiValueHandling") MultiValueHandling multiValueHandling, + @JsonProperty("createBitmapIndex") Boolean createBitmapIndex ) { - super(name, multiValueHandling); + super(name, multiValueHandling, createBitmapIndex == null ? DEFAULT_CREATE_BITMAP_INDEX : createBitmapIndex); } public StringDimensionSchema(String name) { - this(name, null); + this(name, null, DEFAULT_CREATE_BITMAP_INDEX); } @Override diff --git a/api/src/test/java/io/druid/data/input/impl/DimensionSchemaTest.java b/api/src/test/java/io/druid/data/input/impl/DimensionSchemaTest.java new file mode 100644 index 000000000000..8a807d763bad --- /dev/null +++ b/api/src/test/java/io/druid/data/input/impl/DimensionSchemaTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.data.input.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +public class DimensionSchemaTest +{ + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Test + public void testStringDimensionSchemaSerde() throws Exception + { + final StringDimensionSchema schema1 = new StringDimensionSchema("foo"); + Assert.assertEquals( + schema1, + OBJECT_MAPPER.readValue(OBJECT_MAPPER.writeValueAsString(schema1), DimensionSchema.class) + ); + + final StringDimensionSchema schema2 = new StringDimensionSchema( + "foo", + DimensionSchema.MultiValueHandling.ARRAY, + false + ); + Assert.assertEquals( + schema2, + OBJECT_MAPPER.readValue(OBJECT_MAPPER.writeValueAsString(schema2), DimensionSchema.class) + ); + } +} diff --git a/api/src/test/java/io/druid/data/input/impl/DimensionsSpecSerdeTest.java b/api/src/test/java/io/druid/data/input/impl/DimensionsSpecSerdeTest.java index 3bc8f86b776b..99bc3e64f6a3 100644 --- a/api/src/test/java/io/druid/data/input/impl/DimensionsSpecSerdeTest.java +++ b/api/src/test/java/io/druid/data/input/impl/DimensionsSpecSerdeTest.java @@ -19,6 +19,7 @@ package io.druid.data.input.impl; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import junit.framework.Assert; import org.junit.Test; @@ -30,7 +31,11 @@ */ public class DimensionsSpecSerdeTest { - private final ObjectMapper mapper = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + static { + OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } @Test public void testDimensionsSpecSerde() throws Exception @@ -59,9 +64,9 @@ public void testDimensionsSpecSerde() throws Exception + "\"spatialDimensions\": [{\"dimName\":\"IMPR\", \"dims\":[\"S\",\"P\",\"Q\",\"R\"]}]" + "}"; - DimensionsSpec actual = mapper.readValue( - mapper.writeValueAsString( - mapper.readValue(jsonStr, DimensionsSpec.class) + DimensionsSpec actual = OBJECT_MAPPER.readValue( + OBJECT_MAPPER.writeValueAsString( + OBJECT_MAPPER.readValue(jsonStr, DimensionsSpec.class) ), DimensionsSpec.class ); diff --git a/docs/content/ingestion/index.md b/docs/content/ingestion/index.md index 4d09374703ae..b533db622a09 100644 --- a/docs/content/ingestion/index.md +++ b/docs/content/ingestion/index.md @@ -198,7 +198,13 @@ handle all formatting decisions on their own, without using the ParseSpec. #### Dimension Schema A dimension schema specifies the type and name of a dimension to be ingested. -For example, the following `dimensionsSpec` section from a `dataSchema` ingests one column as Long (`countryNum`), two columns as Float (`userLatitude`, `userLongitude`), and the other columns as Strings: +For string columns, the dimension schema can also be used to enable or disable bitmap indexing by setting the +`createBitmapIndex` boolean. By default, bitmap indexes are enabled for all string columns. Only string columns can have +bitmap indexes; they are not supported for numeric columns. + +For example, the following `dimensionsSpec` section from a `dataSchema` ingests one column as Long (`countryNum`), two +columns as Float (`userLatitude`, `userLongitude`), and the other columns as Strings, with bitmap indexes disabled +for the `comment` column. ```json "dimensionsSpec" : { @@ -215,6 +221,11 @@ For example, the following `dimensionsSpec` section from a `dataSchema` ingests "country", "region", "city", + { + "type": "string", + "name": "comment", + "createBitmapIndex": false + }, { "type": "long", "name": "countryNum" @@ -233,7 +244,6 @@ For example, the following `dimensionsSpec` section from a `dataSchema` ingests } ``` - ## GranularitySpec The default granularity spec is `uniform`, and can be changed by setting the `type` field. diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java index 66202395f8cf..b87eb9f09c58 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java @@ -354,7 +354,8 @@ private static DimensionsSpec createDimensionsSpec(List queryabl createDimensionSchema( column.getCapabilities().getType(), dimension, - dimensionHandler.getMultivalueHandling() + dimensionHandler.getMultivalueHandling(), + column.getCapabilities().hasBitmapIndexes() ) ); } @@ -402,7 +403,8 @@ private static List loadSegments( private static DimensionSchema createDimensionSchema( ValueType type, String name, - MultiValueHandling multiValueHandling + MultiValueHandling multiValueHandling, + boolean hasBitmapIndexes ) { switch (type) { @@ -428,7 +430,7 @@ private static DimensionSchema createDimensionSchema( ); return new DoubleDimensionSchema(name); case STRING: - return new StringDimensionSchema(name, multiValueHandling); + return new StringDimensionSchema(name, multiValueHandling, hasBitmapIndexes); default: throw new ISE("Unsupported value type[%s] for dimension[%s]", type, name); } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java index cdab46e3f943..849aff34f0eb 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java @@ -69,8 +69,8 @@ import io.druid.segment.column.Column; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ValueType; -import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ListIndexed; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.incremental.IncrementalIndex; @@ -109,13 +109,13 @@ public class CompactionTaskTest private static final Interval COMPACTION_INTERVAL = Intervals.of("2017-01-01/2017-06-01"); private static final Map MIXED_TYPE_COLUMN_MAP = ImmutableMap.of( Intervals.of("2017-01-01/2017-02-01"), - new StringDimensionSchema(MIXED_TYPE_COLUMN, null), + new StringDimensionSchema(MIXED_TYPE_COLUMN), Intervals.of("2017-02-01/2017-03-01"), - new StringDimensionSchema(MIXED_TYPE_COLUMN, null), + new StringDimensionSchema(MIXED_TYPE_COLUMN), Intervals.of("2017-03-01/2017-04-01"), - new StringDimensionSchema(MIXED_TYPE_COLUMN, null), + new StringDimensionSchema(MIXED_TYPE_COLUMN), Intervals.of("2017-04-01/2017-05-01"), - new StringDimensionSchema(MIXED_TYPE_COLUMN, null), + new StringDimensionSchema(MIXED_TYPE_COLUMN), Intervals.of("2017-05-01/2017-06-01"), new DoubleDimensionSchema(MIXED_TYPE_COLUMN) ); @@ -138,6 +138,7 @@ public static void setup() for (int i = 0; i < 5; i++) { final StringDimensionSchema schema = new StringDimensionSchema( "string_dim_" + i, + null, null ); DIMENSIONS.put(schema.getName(), schema); diff --git a/integration-tests/src/test/resources/indexer/wikipedia_index_queries.json b/integration-tests/src/test/resources/indexer/wikipedia_index_queries.json index 873ed26c43a7..04565bd83866 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_index_queries.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_index_queries.json @@ -25,6 +25,11 @@ "dimensions":[ "page" ], + "filter":{ + "type":"selector", + "dimension":"language", + "value":"zh" + }, "aggregations":[ { "type":"count", diff --git a/integration-tests/src/test/resources/indexer/wikipedia_index_task.json b/integration-tests/src/test/resources/indexer/wikipedia_index_task.json index b0b6596c95ac..15e0de89560c 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_index_task.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_index_task.json @@ -37,8 +37,18 @@ }, "dimensionsSpec": { "dimensions": [ - "page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous", - "namespace", "continent", "country", "region", "city" + "page", + {"type": "string", "name": "language", "createBitmapIndex": false}, + "user", + "unpatrolled", + "newPage", + "robot", + "anonymous", + "namespace", + "continent", + "country", + "region", + "city" ] } } diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java index 5d7a56a5f046..8d7673356c58 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java @@ -62,16 +62,16 @@ public static DimensionHandler getHandlerFromCapabilities( ) { if (capabilities == null) { - return new StringDimensionHandler(dimensionName, multiValueHandling); + return new StringDimensionHandler(dimensionName, multiValueHandling, true); } multiValueHandling = multiValueHandling == null ? MultiValueHandling.ofDefault() : multiValueHandling; if (capabilities.getType() == ValueType.STRING) { - if (!capabilities.isDictionaryEncoded() || !capabilities.hasBitmapIndexes()) { - throw new IAE("String column must have dictionary encoding and bitmap index."); + if (!capabilities.isDictionaryEncoded()) { + throw new IAE("String column must have dictionary encoding."); } - return new StringDimensionHandler(dimensionName, multiValueHandling); + return new StringDimensionHandler(dimensionName, multiValueHandling, capabilities.hasBitmapIndexes()); } if (capabilities.getType() == ValueType.LONG) { @@ -87,7 +87,7 @@ public static DimensionHandler getHandlerFromCapabilities( } // Return a StringDimensionHandler by default (null columns will be treated as String typed) - return new StringDimensionHandler(dimensionName, multiValueHandling); + return new StringDimensionHandler(dimensionName, multiValueHandling, true); } public static List getValueTypesFromDimensionSpecs(List dimSpecs) diff --git a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java index cd4db8a3db46..2fb7ad9be92a 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java @@ -21,12 +21,13 @@ import com.google.common.primitives.Ints; import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; -import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.java.util.common.ISE; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedInts; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.Closeable; import java.lang.reflect.Array; @@ -36,11 +37,13 @@ public class StringDimensionHandler implements DimensionHandler makeIndexer() { - return new StringDimensionIndexer(multiValueHandling); + return new StringDimensionIndexer(multiValueHandling, hasBitmapIndexes); } @Override - public DimensionMergerV9 makeMerger( + public DimensionMergerV9 makeMerger( IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, ProgressIndicator progress ) { + // Sanity-check capabilities. + if (hasBitmapIndexes != capabilities.hasBitmapIndexes()) { + throw new ISE( + "capabilities.hasBitmapIndexes[%s] != this.hasBitmapIndexes[%s]", + capabilities.hasBitmapIndexes(), + hasBitmapIndexes + ); + } + return new StringDimensionMergerV9(dimensionName, indexSpec, segmentWriteOutMedium, capabilities, progress); } - } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index af994ecfd6e9..27d748e525b4 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -179,12 +179,14 @@ public String getValueFromSortedId(int index) private final DimensionDictionary dimLookup; private final MultiValueHandling multiValueHandling; + private final boolean hasBitmapIndexes; private SortedDimensionDictionary sortedLookup; - public StringDimensionIndexer(MultiValueHandling multiValueHandling) + public StringDimensionIndexer(MultiValueHandling multiValueHandling, boolean hasBitmapIndexes) { this.dimLookup = new DimensionDictionary(); this.multiValueHandling = multiValueHandling == null ? MultiValueHandling.ofDefault() : multiValueHandling; + this.hasBitmapIndexes = hasBitmapIndexes; } @Override @@ -621,6 +623,10 @@ public void fillBitmapsFromUnsortedEncodedKeyComponent( int[] key, int rowNum, MutableBitmap[] bitmapIndexes, BitmapFactory factory ) { + if (!hasBitmapIndexes) { + throw new UnsupportedOperationException("This column does not include bitmap indexes"); + } + for (int dimValIdx : key) { if (bitmapIndexes[dimValIdx] == null) { bitmapIndexes[dimValIdx] = factory.makeEmptyMutableBitmap(); diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index 8e97666bc184..e671a4cad240 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -38,14 +38,14 @@ import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.BitmapValues; import io.druid.segment.data.ByteBufferWriter; -import io.druid.segment.data.V3CompressedVSizeColumnarMultiIntsSerializer; +import io.druid.segment.data.ColumnarIntsSerializer; import io.druid.segment.data.CompressedVSizeColumnarIntsSerializer; import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.GenericIndexedWriter; import io.druid.segment.data.ImmutableRTreeObjectStrategy; import io.druid.segment.data.Indexed; -import io.druid.segment.data.ColumnarIntsSerializer; +import io.druid.segment.data.V3CompressedVSizeColumnarMultiIntsSerializer; import io.druid.segment.data.VSizeColumnarIntsSerializer; import io.druid.segment.data.VSizeColumnarMultiIntsSerializer; import io.druid.segment.serde.DictionaryEncodedColumnPartSerde; @@ -272,6 +272,10 @@ protected void processMergedRowHelper(int[] vals) throws IOException @Override public void writeIndexes(List segmentRowNumConversions) throws IOException { + if (!capabilities.hasBitmapIndexes()) { + return; + } + long dimStartTime = System.currentTimeMillis(); final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index d2f19696129c..2fa73ec6e727 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -284,6 +284,8 @@ protected IncrementalIndex( ValueType type = TYPE_MAP.get(dimSchema.getValueType()); String dimName = dimSchema.getName(); ColumnCapabilitiesImpl capabilities = makeCapabilitesFromValueType(type); + capabilities.setHasBitmapIndexes(dimSchema.hasBitmapIndex()); + if (dimSchema.getTypeName().equals(DimensionSchema.SPATIAL_TYPE_NAME)) { capabilities.setHasSpatialIndexes(true); } else { diff --git a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 6ebbfca8f0a7..b59b188b3456 100644 --- a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -58,11 +58,13 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde { private static final int NO_FLAGS = 0; + private static final int STARTING_FLAGS = Feature.NO_BITMAP_INDEX.getMask(); enum Feature { MULTI_VALUE, - MULTI_VALUE_V3; + MULTI_VALUE_V3, + NO_BITMAP_INDEX; public boolean isSet(int flags) { @@ -79,8 +81,8 @@ enum VERSION { UNCOMPRESSED_SINGLE_VALUE, // 0x0 UNCOMPRESSED_MULTI_VALUE, // 0x1 - COMPRESSED; // 0x2 - + COMPRESSED, // 0x2 + UNCOMPRESSED_WITH_FLAGS; // 0x3 public static VERSION fromByte(byte b) { @@ -143,7 +145,7 @@ public static SerializerBuilder serializerBuilder() public static class SerializerBuilder { private VERSION version = null; - private int flags = NO_FLAGS; + private int flags = STARTING_FLAGS; private GenericIndexedWriter dictionaryWriter = null; private ColumnarIntsSerializer valueWriter = null; private BitmapSerdeFactory bitmapSerdeFactory = null; @@ -163,8 +165,14 @@ public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFa return this; } - public SerializerBuilder withBitmapIndex(GenericIndexedWriter bitmapIndexWriter) + public SerializerBuilder withBitmapIndex(@Nullable GenericIndexedWriter bitmapIndexWriter) { + if (bitmapIndexWriter == null) { + flags |= Feature.NO_BITMAP_INDEX.getMask(); + } else { + flags &= ~Feature.NO_BITMAP_INDEX.getMask(); + } + this.bitmapIndexWriter = bitmapIndexWriter; return this; } @@ -204,6 +212,11 @@ public SerializerBuilder withValue(ColumnarIntsSerializer valueWriter, boolean h public DictionaryEncodedColumnPartSerde build() { + if (mustWriteFlags(flags) && version.compareTo(VERSION.COMPRESSED) < 0) { + // Must upgrade version so we can write out flags. + this.version = VERSION.UNCOMPRESSED_WITH_FLAGS; + } + return new DictionaryEncodedColumnPartSerde( byteOrder, bitmapSerdeFactory, @@ -309,20 +322,23 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo ); builder.setHasMultipleValues(hasMultipleValues).setDictionaryEncodedColumn(dictionaryEncodedColumnSupplier); - GenericIndexed rBitmaps = GenericIndexed.read( - buffer, bitmapSerdeFactory.getObjectStrategy(), builder.getFileMapper() - ); - builder.setBitmapIndex( - new BitmapIndexColumnPartSupplier( - bitmapSerdeFactory.getBitmapFactory(), - rBitmaps, - rDictionary - ) - ); + if (!Feature.NO_BITMAP_INDEX.isSet(rFlags)) { + GenericIndexed rBitmaps = GenericIndexed.read( + buffer, + bitmapSerdeFactory.getObjectStrategy(), + builder.getFileMapper() + ); + builder.setBitmapIndex( + new BitmapIndexColumnPartSupplier( + bitmapSerdeFactory.getBitmapFactory(), + rBitmaps, + rDictionary + ) + ); + } - ImmutableRTree rSpatialIndex = null; if (buffer.hasRemaining()) { - rSpatialIndex = + ImmutableRTree rSpatialIndex = new ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()).fromByteBufferWithSize(buffer); builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(rSpatialIndex)); } @@ -333,6 +349,7 @@ private WritableSupplier readSingleValuedColumn(VERSION version, B { switch (version) { case UNCOMPRESSED_SINGLE_VALUE: + case UNCOMPRESSED_WITH_FLAGS: return VSizeColumnarInts.readFromByteBuffer(buffer); case COMPRESSED: return CompressedVSizeColumnarIntsSupplier.fromByteBuffer(buffer, byteOrder); @@ -349,13 +366,20 @@ private WritableSupplier readMultiValuedColumn( case UNCOMPRESSED_MULTI_VALUE: { return VSizeColumnarMultiInts.readFromByteBuffer(buffer); } + case UNCOMPRESSED_WITH_FLAGS: { + if (Feature.MULTI_VALUE.isSet(flags)) { + return VSizeColumnarMultiInts.readFromByteBuffer(buffer); + } else { + throw new IAE("Unrecognized multi-value flag[%d] for version[%s]", flags, version); + } + } case COMPRESSED: { if (Feature.MULTI_VALUE.isSet(flags)) { return CompressedVSizeColumnarMultiIntsSupplier.fromByteBuffer(buffer, byteOrder); } else if (Feature.MULTI_VALUE_V3.isSet(flags)) { return V3CompressedVSizeColumnarMultiIntsSupplier.fromByteBuffer(buffer, byteOrder); } else { - throw new IAE("Unrecognized multi-value flag[%d]", flags); + throw new IAE("Unrecognized multi-value flag[%d] for version[%s]", flags, version); } } default: @@ -364,4 +388,10 @@ private WritableSupplier readMultiValuedColumn( } }; } + + private static boolean mustWriteFlags(final int flags) + { + // Flags that are not implied by version codes < COMPRESSED must be written. This includes MULTI_VALUE_V3. + return flags != NO_FLAGS && flags != Feature.MULTI_VALUE.getMask(); + } } diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index 63f79db5121d..93254f9fd80b 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -51,6 +51,7 @@ import io.druid.segment.data.BitmapValues; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.CompressionStrategy; +import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.segment.data.IncrementalIndexTest; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexAdapter; @@ -78,6 +79,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class IndexMergerTestBase { @@ -131,12 +133,13 @@ static IndexSpec makeIndexSpec( private final IndexSpec indexSpec; private final IndexIO indexIO; + private final boolean useBitmapIndexes; @Rule public final CloserRule closer = new CloserRule(false); protected IndexMergerTestBase( - BitmapSerdeFactory bitmapSerdeFactory, + @Nullable BitmapSerdeFactory bitmapSerdeFactory, CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, CompressionFactory.LongEncodingStrategy longEncodingStrategy, @@ -144,12 +147,13 @@ protected IndexMergerTestBase( ) { this.indexSpec = makeIndexSpec( - bitmapSerdeFactory, + bitmapSerdeFactory != null ? bitmapSerdeFactory : new ConciseBitmapSerdeFactory(), compressionStrategy, dimCompressionStrategy, longEncodingStrategy ); this.indexIO = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory); + this.useBitmapIndexes = bitmapSerdeFactory != null; } @Test @@ -923,7 +927,7 @@ public void testMergeWithDimensionsList() throws Exception { IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() .withDimensionsSpec(new DimensionsSpec( - DimensionsSpec.getDefaultSchemas(Arrays.asList("dimA", "dimB", "dimC")), + makeDimensionSchemas(Arrays.asList("dimA", "dimB", "dimC")), null, null )) @@ -1014,15 +1018,20 @@ public void testMergeWithDimensionsList() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}}, boatList.get(3).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList.get(3).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", "")); - checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimA", "1")); - checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "2")); + Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimA").hasBitmapIndexes()); + Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimC").hasBitmapIndexes()); + + if (useBitmapIndexes) { + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimA", "1")); + checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", "")); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimC", "1")); + checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimC", "2")); + } - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", "")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimC", "1")); - checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimC", "2")); + checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dimB", "")); } @@ -1139,14 +1148,20 @@ public void testDisjointDimMerge() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}}, boatList2.get(4).getDims()); Assert.assertArrayEquals(new Object[]{1L}, boatList2.get(4).getMetrics()); + // dimA always has bitmap indexes, since it has them in indexA (it comes in through discovery). + Assert.assertTrue(adapter2.getCapabilities("dimA").hasBitmapIndexes()); checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter2.getBitmapIndex("dimA", "")); checkBitmapIndex(Lists.newArrayList(3), adapter2.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(4), adapter2.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(3, 4), adapter2.getBitmapIndex("dimB", "")); - checkBitmapIndex(Lists.newArrayList(0), adapter2.getBitmapIndex("dimB", "1")); - checkBitmapIndex(Lists.newArrayList(1), adapter2.getBitmapIndex("dimB", "2")); - checkBitmapIndex(Lists.newArrayList(2), adapter2.getBitmapIndex("dimB", "3")); + // dimB may or may not have bitmap indexes, since it comes in through explicit definition in indexB2. + Assert.assertEquals(useBitmapIndexes, adapter2.getCapabilities("dimB").hasBitmapIndexes()); + if (useBitmapIndexes) { + checkBitmapIndex(Lists.newArrayList(3, 4), adapter2.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(0), adapter2.getBitmapIndex("dimB", "1")); + checkBitmapIndex(Lists.newArrayList(1), adapter2.getBitmapIndex("dimB", "2")); + checkBitmapIndex(Lists.newArrayList(2), adapter2.getBitmapIndex("dimB", "3")); + } } @Test @@ -1565,7 +1580,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception private void checkBitmapIndex(ArrayList expected, BitmapValues real) { - Assert.assertEquals(expected.size(), real.size()); + Assert.assertEquals("bitmap size", expected.size(), real.size()); int i = 0; for (IntIterator iterator = real.iterator(); iterator.hasNext(); ) { int index = iterator.nextInt(); @@ -2122,7 +2137,7 @@ private IncrementalIndex getIndexWithNumericDims() throws Exception Arrays.asList( new LongDimensionSchema("dimA"), new FloatDimensionSchema("dimB"), - new StringDimensionSchema("dimC") + new StringDimensionSchema("dimC", MultiValueHandling.SORTED_ARRAY, useBitmapIndexes) ) ); @@ -2281,7 +2296,7 @@ private void addDimValuesToIndex(IncrementalIndex index, String dimName, List dims) { IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() - .withDimensionsSpec(new DimensionsSpec(DimensionsSpec.getDefaultSchemas(dims), null, null)) + .withDimensionsSpec(new DimensionsSpec(makeDimensionSchemas(dims), null, null)) .withMetrics(new CountAggregatorFactory("count")) .build(); @@ -2384,7 +2399,7 @@ public void testMultiValueHandling() throws Exception List boatList; // xaab-axbx + abx-xab --> aabx-abxx + abx-abx --> abx-abx + aabx-abxx - schema = DimensionsSpec.getDefaultSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.SORTED_ARRAY); + schema = makeDimensionSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.SORTED_ARRAY); index = persistAndLoad(schema, rows); adapter = new QueryableIndexIndexableAdapter(index); boatList = ImmutableList.copyOf(adapter.getRows()); @@ -2397,17 +2412,22 @@ public void testMultiValueHandling() throws Exception Assert.assertArrayEquals(new int[][]{{0, 1, 2}, {0, 1, 2}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{0, 0, 1, 2}, {0, 1, 2, 2}}, boatList.get(1).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); + Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim1").hasBitmapIndexes()); + Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes()); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "a")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "b")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "x")); + if (useBitmapIndexes) { + checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); + + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "a")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "b")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "x")); + } // xaab-axbx + abx-xab --> abx-abx + abx-abx --> abx-abx - schema = DimensionsSpec.getDefaultSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.SORTED_SET); + schema = makeDimensionSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.SORTED_SET); index = persistAndLoad(schema, rows); Assert.assertEquals(1, index.getColumn(Column.TIME_COLUMN_NAME).getLength()); @@ -2420,17 +2440,22 @@ public void testMultiValueHandling() throws Exception Assert.assertEquals(1, boatList.size()); Assert.assertArrayEquals(new int[][]{{0, 1, 2}, {0, 1, 2}}, boatList.get(0).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "a")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "b")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "x")); + Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim1").hasBitmapIndexes()); + Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes()); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "a")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "b")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "x")); + if (useBitmapIndexes) { + checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "a")); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "b")); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "x")); + + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "a")); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "b")); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "x")); + } // xaab-axbx + abx-xab --> abx-xab + xaab-axbx - schema = DimensionsSpec.getDefaultSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.ARRAY); + schema = makeDimensionSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.ARRAY); index = persistAndLoad(schema, rows); Assert.assertEquals(2, index.getColumn(Column.TIME_COLUMN_NAME).getLength()); @@ -2444,14 +2469,19 @@ public void testMultiValueHandling() throws Exception Assert.assertArrayEquals(new int[][]{{0, 1, 2}, {2, 0, 1}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{2, 0, 0, 1}, {0, 2, 1, 2}}, boatList.get(1).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); + Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim1").hasBitmapIndexes()); + Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes()); + + if (useBitmapIndexes) { + checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "a")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "b")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "x")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "a")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "b")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim2", "x")); + } } private QueryableIndex persistAndLoad(List schema, InputRow... rows) throws IOException @@ -2464,4 +2494,25 @@ private QueryableIndex persistAndLoad(List schema, InputRow... final File tempDir = temporaryFolder.newFolder(); return closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))); } + + private List makeDimensionSchemas(final List dimensions) + { + return makeDimensionSchemas(dimensions, MultiValueHandling.SORTED_ARRAY); + } + + private List makeDimensionSchemas( + final List dimensions, + final MultiValueHandling multiValueHandling + ) + { + return dimensions.stream() + .map( + dimension -> new StringDimensionSchema( + dimension, + multiValueHandling, + useBitmapIndexes + ) + ) + .collect(Collectors.toList()); + } } diff --git a/processing/src/test/java/io/druid/segment/NoBitmapIndexMergerV9Test.java b/processing/src/test/java/io/druid/segment/NoBitmapIndexMergerV9Test.java new file mode 100644 index 000000000000..3fe422f889e8 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/NoBitmapIndexMergerV9Test.java @@ -0,0 +1,47 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import io.druid.segment.data.CompressionStrategy; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class NoBitmapIndexMergerV9Test extends IndexMergerTestBase +{ + public NoBitmapIndexMergerV9Test( + CompressionStrategy compressionStrategy, + CompressionStrategy dimCompressionStrategy, + LongEncodingStrategy longEncodingStrategy, + SegmentWriteOutMediumFactory segmentWriteOutMediumFactory + ) + { + super( + null, + compressionStrategy, + dimCompressionStrategy, + longEncodingStrategy, + segmentWriteOutMediumFactory + ); + indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); + } +} diff --git a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java index d53c6b778ddc..c33307cdafa9 100644 --- a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java @@ -104,15 +104,8 @@ public abstract class BaseFilterTest // For filter tests, the test setup creates a segment. // Creating a new segment for every test method call is pretty slow, so cache the StorageAdapters. // Each thread gets its own map. - protected static ThreadLocal>>> adapterCache = - new ThreadLocal>>>() - { - @Override - protected Map>> initialValue() - { - return new HashMap<>(); - } - }; + private static ThreadLocal>>> adapterCache = + ThreadLocal.withInitial(HashMap::new); public BaseFilterTest( String testName, diff --git a/processing/src/test/java/io/druid/segment/filter/RowboatTest.java b/processing/src/test/java/io/druid/segment/filter/RowboatTest.java index 3789e72ff524..4c80b109bee3 100644 --- a/processing/src/test/java/io/druid/segment/filter/RowboatTest.java +++ b/processing/src/test/java/io/druid/segment/filter/RowboatTest.java @@ -34,7 +34,7 @@ private static DimensionHandler[] getDefaultHandlers(int size) { DimensionHandler[] handlers = new DimensionHandler[size]; for (int i = 0; i < size; i++) { - handlers[i] = new StringDimensionHandler(String.valueOf(i), null); + handlers[i] = new StringDimensionHandler(String.valueOf(i), null, true); } return handlers; } diff --git a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java index c3981d4f7e26..4feac1adff95 100644 --- a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java @@ -39,6 +39,7 @@ import io.druid.query.lookup.LookupExtractor; import io.druid.segment.IndexBuilder; import io.druid.segment.StorageAdapter; +import io.druid.segment.incremental.IncrementalIndexSchema; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Test; @@ -83,7 +84,17 @@ public SelectorFilterTest( boolean optimize ) { - super(testName, ROWS, indexBuilder, finisher, cnf, optimize); + super( + testName, + ROWS, + indexBuilder.schema( + new IncrementalIndexSchema.Builder() + .withDimensionsSpec(PARSER.getParseSpec().getDimensionsSpec()).build() + ), + finisher, + cnf, + optimize + ); } @AfterClass diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java index a5fcff133316..491ba18427a4 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java @@ -45,9 +45,9 @@ public void test() throws IndexSizeExceededException { DimensionsSpec dimensionsSpec = new DimensionsSpec( Arrays.asList( - new StringDimensionSchema("string1", DimensionSchema.MultiValueHandling.ARRAY), - new StringDimensionSchema("string2", DimensionSchema.MultiValueHandling.SORTED_ARRAY), - new StringDimensionSchema("string3", DimensionSchema.MultiValueHandling.SORTED_SET) + new StringDimensionSchema("string1", DimensionSchema.MultiValueHandling.ARRAY, true), + new StringDimensionSchema("string2", DimensionSchema.MultiValueHandling.SORTED_ARRAY, true), + new StringDimensionSchema("string3", DimensionSchema.MultiValueHandling.SORTED_SET, true) ), null, null );