From ddcbe16bf4fc74de8385e5518821241ecd86d045 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 20 Feb 2019 17:06:58 -0800 Subject: [PATCH 1/6] segment metadata fallback analysis if no bitmaps --- .../druid/query/metadata/SegmentAnalyzer.java | 52 ++++++++++------ .../metadata/SegmentMetadataQueryTest.java | 56 +++++++++++------- .../org/apache/druid/segment/TestIndex.java | 59 +++++++++++++++++-- 3 files changed, 125 insertions(+), 42 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java index 8db528560e40..6f6bbe8eb0d7 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java @@ -42,6 +42,7 @@ import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ComplexColumn; +import org.apache.druid.segment.column.DictionaryEncodedColumn; import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.data.IndexedInts; import org.apache.druid.segment.serde.ComplexMetricSerde; @@ -198,26 +199,43 @@ private ColumnAnalysis analyzeStringColumn( Comparable min = null; Comparable max = null; - - if (!capabilities.hasBitmapIndexes()) { - return ColumnAnalysis.error("string_no_bitmap"); - } - - final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); - final int cardinality = bitmapIndex.getCardinality(); - - if (analyzingSize()) { - for (int i = 0; i < cardinality; ++i) { - String value = bitmapIndex.getValue(i); - if (value != null) { - size += StringUtils.estimatedBinaryLengthAsUTF8(value) * bitmapIndex.getBitmap(bitmapIndex.getIndex(value)).size(); + int cardinality = 0; + if (capabilities.hasBitmapIndexes()) { + final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); + cardinality = bitmapIndex.getCardinality(); + + if (analyzingSize()) { + for (int i = 0; i < cardinality; ++i) { + String value = bitmapIndex.getValue(i); + if (value != null) { + size += StringUtils.estimatedBinaryLengthAsUTF8(value) * bitmapIndex.getBitmap(bitmapIndex.getIndex(value)) + .size(); + } } } - } - if (analyzingMinMax() && cardinality > 0) { - min = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(0)); - max = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(cardinality - 1)); + if (analyzingMinMax() && cardinality > 0) { + min = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(0)); + max = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(cardinality - 1)); + } + } else if (capabilities.isDictionaryEncoded()) { + // fallback if no bitmap index + DictionaryEncodedColumn theColumn = (DictionaryEncodedColumn) columnHolder.getColumn(); + cardinality = theColumn.getCardinality(); + theColumn.lookupName(0); + + if (analyzingSize()) { + for (int i = 0; i < cardinality; ++i) { + String value = theColumn.lookupName(i); + if (value != null && !value.isEmpty()) { + size += StringUtils.estimatedBinaryLengthAsUTF8(value); + } + } + } + if (analyzingMinMax() && cardinality > 0) { + min = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(0)); + max = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(cardinality - 1)); + } } return new ColumnAnalysis( diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java index 31e70d376644..dd2a76b79df8 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java @@ -83,10 +83,15 @@ public class SegmentMetadataQueryTest public static QueryRunner makeMMappedQueryRunner( SegmentId segmentId, boolean rollup, + boolean bitmaps, QueryRunnerFactory factory ) { - QueryableIndex index = rollup ? TestIndex.getMMappedTestIndex() : TestIndex.getNoRollupMMappedTestIndex(); + QueryableIndex index = bitmaps + ? rollup + ? TestIndex.getMMappedTestIndex() + : TestIndex.getNoRollupMMappedTestIndex() + : TestIndex.getNoBitmapMMappedTestIndex(); return QueryRunnerTestHelper.makeQueryRunner( factory, segmentId, @@ -99,10 +104,15 @@ public static QueryRunner makeMMappedQueryRunner( public static QueryRunner makeIncrementalIndexQueryRunner( SegmentId segmentId, boolean rollup, + boolean bitmaps, QueryRunnerFactory factory ) { - IncrementalIndex index = rollup ? TestIndex.getIncrementalTestIndex() : TestIndex.getNoRollupIncrementalTestIndex(); + IncrementalIndex index = bitmaps + ? rollup + ? TestIndex.getIncrementalTestIndex() + : TestIndex.getNoRollupIncrementalTestIndex() + : TestIndex.getNoBitmapIncrementalTestIndex(); return QueryRunnerTestHelper.makeQueryRunner( factory, segmentId, @@ -121,17 +131,19 @@ public static QueryRunner makeIncrementalIndexQueryRunner( private final SegmentMetadataQuery testQuery; private final SegmentAnalysis expectedSegmentAnalysis1; private final SegmentAnalysis expectedSegmentAnalysis2; + private final boolean bitmaps; - @Parameterized.Parameters(name = "mmap1 = {0}, mmap2 = {1}, rollup1 = {2}, rollup2 = {3}, differentIds = {4}") + @Parameterized.Parameters(name = "mmap1 = {0}, mmap2 = {1}, rollup1 = {2}, rollup2 = {3}, differentIds = {4}, bitmaps={5}") public static Collection constructorFeeder() { return ImmutableList.of( - new Object[]{true, true, true, true, false}, - new Object[]{true, false, true, false, false}, - new Object[]{false, true, true, false, false}, - new Object[]{false, false, false, false, false}, - new Object[]{false, false, true, true, false}, - new Object[]{false, false, false, true, true} + new Object[]{true, true, true, true, false, true}, + new Object[]{true, false, true, false, false, true}, + new Object[]{false, true, true, false, false, true}, + new Object[]{false, false, false, false, false, true}, + new Object[]{false, false, true, true, false, true}, + new Object[]{false, false, false, true, true, true}, + new Object[]{true, true, false, false, false, false} ); } @@ -140,22 +152,24 @@ public SegmentMetadataQueryTest( boolean mmap2, boolean rollup1, boolean rollup2, - boolean differentIds + boolean differentIds, + boolean bitmaps ) { final SegmentId id1 = SegmentId.dummy(differentIds ? "testSegment1" : "testSegment"); final SegmentId id2 = SegmentId.dummy(differentIds ? "testSegment2" : "testSegment"); this.runner1 = mmap1 - ? makeMMappedQueryRunner(id1, rollup1, FACTORY) - : makeIncrementalIndexQueryRunner(id1, rollup1, FACTORY); + ? makeMMappedQueryRunner(id1, rollup1, bitmaps, FACTORY) + : makeIncrementalIndexQueryRunner(id1, rollup1, bitmaps, FACTORY); this.runner2 = mmap2 - ? makeMMappedQueryRunner(id2, rollup2, FACTORY) - : makeIncrementalIndexQueryRunner(id2, rollup2, FACTORY); + ? makeMMappedQueryRunner(id2, rollup2, bitmaps, FACTORY) + : makeIncrementalIndexQueryRunner(id2, rollup2, bitmaps, FACTORY); this.mmap1 = mmap1; this.mmap2 = mmap2; this.rollup1 = rollup1; this.rollup2 = rollup2; this.differentIds = differentIds; + this.bitmaps = bitmaps; testQuery = Druids.newSegmentMetadataQueryBuilder() .dataSource("testing") .intervals("2013/2014") @@ -187,7 +201,7 @@ public SegmentMetadataQueryTest( new ColumnAnalysis( ValueType.STRING.toString(), false, - mmap1 ? 10881 : 10764, + mmap1 ? (bitmaps ? 10881 : 9) : 10764, 1, "preferred", "preferred", @@ -203,7 +217,7 @@ public SegmentMetadataQueryTest( null, null ) - ), mmap1 ? 167493 : 168188, + ), mmap1 ? (bitmaps ? 167493 : 119872) : 168188, 1209, null, null, @@ -228,7 +242,7 @@ public SegmentMetadataQueryTest( new ColumnAnalysis( ValueType.STRING.toString(), false, - mmap2 ? 10881 : 0, + mmap2 ? (bitmaps ? 10881 : 9) : 0, 1, null, null, @@ -245,7 +259,7 @@ public SegmentMetadataQueryTest( null ) // null_column will be included only for incremental index, which makes a little bigger result than expected - ), mmap2 ? 167493 : 168188, + ), mmap2 ? (bitmaps ? 167493 : 119872) : 168188, 1209, null, null, @@ -473,7 +487,7 @@ public void testSegmentMetadataQueryWithDefaultAnalysisMerge() ColumnAnalysis analysis = new ColumnAnalysis( ValueType.STRING.toString(), false, - (mmap1 ? 10881 : 10764) + (mmap2 ? 10881 : 10764), + (mmap1 ? bitmaps ? 10881 : 9 : 10764) + (mmap2 ? bitmaps ? 10881 : 9 : 10764), 1, "preferred", "preferred", @@ -488,7 +502,7 @@ public void testSegmentMetadataQueryWithDefaultAnalysisMerge2() ColumnAnalysis analysis = new ColumnAnalysis( ValueType.STRING.toString(), false, - (mmap1 ? 6882 : 6808) + (mmap2 ? 6882 : 6808), + (mmap1 ? bitmaps ? 6882 : 23 : 6808) + (mmap2 ? bitmaps ? 6882 : 23 : 6808), 3, "spot", "upfront", @@ -503,7 +517,7 @@ public void testSegmentMetadataQueryWithDefaultAnalysisMerge3() ColumnAnalysis analysis = new ColumnAnalysis( ValueType.STRING.toString(), false, - (mmap1 ? 9765 : 9660) + (mmap2 ? 9765 : 9660), + (mmap1 ? bitmaps ? 9765 : 73 : 9660) + (mmap2 ? bitmaps ? 9765 : 73 : 9660), 9, "automotive", "travel", diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index 72b52873386f..f44254f3677b 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -111,12 +111,31 @@ public class TestIndex new StringDimensionSchema("null_column") ); + public static final List DIMENSION_SCHEMAS_NO_BITMAP = Arrays.asList( + new StringDimensionSchema("market", null, false), + new StringDimensionSchema("quality", null, false), + new LongDimensionSchema("qualityLong"), + new FloatDimensionSchema("qualityFloat"), + new DoubleDimensionSchema("qualityDouble"), + new StringDimensionSchema("qualityNumericString", null, false), + new StringDimensionSchema("placement", null, false), + new StringDimensionSchema("placementish", null, false), + new StringDimensionSchema("partial_null_column", null, false), + new StringDimensionSchema("null_column", null, false) + ); + public static final DimensionsSpec DIMENSIONS_SPEC = new DimensionsSpec( DIMENSION_SCHEMAS, null, null ); + public static final DimensionsSpec DIMENSIONS_SPEC_NO_BITMAPS = new DimensionsSpec( + DIMENSION_SCHEMAS_NO_BITMAP, + null, + null + ); + public static final String[] DOUBLE_METRICS = new String[]{"index", "indexMin", "indexMaxPlusTen"}; public static final String[] FLOAT_METRICS = new String[]{"indexFloat", "indexMinFloat", "indexMaxFloat"}; private static final Logger log = new Logger(TestIndex.class); @@ -149,9 +168,11 @@ public class TestIndex private static IncrementalIndex realtimeIndex = null; private static IncrementalIndex noRollupRealtimeIndex = null; + private static IncrementalIndex noBitmapRealtimeIndex = null; private static QueryableIndex mmappedIndex = null; private static QueryableIndex noRollupMmappedIndex = null; private static QueryableIndex mergedRealtime = null; + private static QueryableIndex noBitmapMmappedIndex = null; public static IncrementalIndex getIncrementalTestIndex() { @@ -175,6 +196,17 @@ public static IncrementalIndex getNoRollupIncrementalTestIndex() return noRollupRealtimeIndex = makeRealtimeIndex("druid.sample.numeric.tsv", false); } + public static IncrementalIndex getNoBitmapIncrementalTestIndex() + { + synchronized (log) { + if (noBitmapRealtimeIndex != null) { + return noBitmapRealtimeIndex; + } + } + + return noBitmapRealtimeIndex = makeRealtimeIndex("druid.sample.numeric.tsv", false, false); + } + public static QueryableIndex getMMappedTestIndex() { synchronized (log) { @@ -203,6 +235,20 @@ public static QueryableIndex getNoRollupMMappedTestIndex() return noRollupMmappedIndex; } + public static QueryableIndex getNoBitmapMMappedTestIndex() + { + synchronized (log) { + if (noBitmapMmappedIndex != null) { + return noBitmapMmappedIndex; + } + } + + IncrementalIndex incrementalIndex = getNoBitmapIncrementalTestIndex(); + noBitmapMmappedIndex = persistRealtimeAndLoadMMapped(incrementalIndex); + + return noBitmapMmappedIndex; + } + public static QueryableIndex mergedRealtimeIndex() { synchronized (log) { @@ -256,6 +302,11 @@ public static IncrementalIndex makeRealtimeIndex(final String resourceFilename) } public static IncrementalIndex makeRealtimeIndex(final String resourceFilename, boolean rollup) + { + return makeRealtimeIndex(resourceFilename, rollup, true); + } + + public static IncrementalIndex makeRealtimeIndex(final String resourceFilename, boolean rollup, boolean bitmap) { final URL resource = TestIndex.class.getClassLoader().getResource(resourceFilename); if (resource == null) { @@ -263,20 +314,20 @@ public static IncrementalIndex makeRealtimeIndex(final String resourceFilename, } log.info("Realtime loading index file[%s]", resource); CharSource stream = Resources.asByteSource(resource).asCharSource(StandardCharsets.UTF_8); - return makeRealtimeIndex(stream, rollup); + return makeRealtimeIndex(stream, rollup, bitmap); } public static IncrementalIndex makeRealtimeIndex(final CharSource source) { - return makeRealtimeIndex(source, true); + return makeRealtimeIndex(source, true, true); } - public static IncrementalIndex makeRealtimeIndex(final CharSource source, boolean rollup) + public static IncrementalIndex makeRealtimeIndex(final CharSource source, boolean rollup, boolean bitmap) { final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() .withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis()) .withTimestampSpec(new TimestampSpec("ds", "auto", null)) - .withDimensionsSpec(DIMENSIONS_SPEC) + .withDimensionsSpec(bitmap ? DIMENSIONS_SPEC : DIMENSIONS_SPEC_NO_BITMAPS) .withVirtualColumns(VIRTUAL_COLUMNS) .withMetrics(METRIC_AGGS) .withRollup(rollup) From 098d2eb1c090aa3ecaddc435db640eebec8e49c6 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 20 Feb 2019 17:16:45 -0800 Subject: [PATCH 2/6] remove accidental line --- .../java/org/apache/druid/query/metadata/SegmentAnalyzer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java index 6f6bbe8eb0d7..538de4444012 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java @@ -222,7 +222,6 @@ private ColumnAnalysis analyzeStringColumn( // fallback if no bitmap index DictionaryEncodedColumn theColumn = (DictionaryEncodedColumn) columnHolder.getColumn(); cardinality = theColumn.getCardinality(); - theColumn.lookupName(0); if (analyzingSize()) { for (int i = 0; i < cardinality; ++i) { From d29d7a2d10be4f749895c18bfad965c9124976fa Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 20 Feb 2019 17:54:13 -0800 Subject: [PATCH 3/6] remove nonsense size estimation --- .../druid/query/metadata/SegmentAnalyzer.java | 9 ---- .../metadata/SegmentMetadataQueryTest.java | 42 +++++++++++++++---- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java index 538de4444012..376ddaaa12a4 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java @@ -222,15 +222,6 @@ private ColumnAnalysis analyzeStringColumn( // fallback if no bitmap index DictionaryEncodedColumn theColumn = (DictionaryEncodedColumn) columnHolder.getColumn(); cardinality = theColumn.getCardinality(); - - if (analyzingSize()) { - for (int i = 0; i < cardinality; ++i) { - String value = theColumn.lookupName(i); - if (value != null && !value.isEmpty()) { - size += StringUtils.estimatedBinaryLengthAsUTF8(value); - } - } - } if (analyzingMinMax() && cardinality > 0) { min = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(0)); max = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(cardinality - 1)); diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java index dd2a76b79df8..40c477b08f7a 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java @@ -183,6 +183,16 @@ public SegmentMetadataQueryTest( .merge(true) .build(); + int preferedSize1 = 0; + int placementSize2 = 0; + int overallSize1 = 119691; + int overallSize2 = 119691; + if (bitmaps) { + preferedSize1 = mmap1 ? 10881 : 10764; + placementSize2 = mmap2 ? 10881 : 0; + overallSize1 = mmap1 ? 167493 : 168188; + overallSize2 = mmap2 ? 167493 : 168188; + } expectedSegmentAnalysis1 = new SegmentAnalysis( id1.toString(), ImmutableList.of(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")), @@ -201,7 +211,7 @@ public SegmentMetadataQueryTest( new ColumnAnalysis( ValueType.STRING.toString(), false, - mmap1 ? (bitmaps ? 10881 : 9) : 10764, + preferedSize1, 1, "preferred", "preferred", @@ -217,7 +227,7 @@ public SegmentMetadataQueryTest( null, null ) - ), mmap1 ? (bitmaps ? 167493 : 119872) : 168188, + ), overallSize1, 1209, null, null, @@ -242,7 +252,7 @@ public SegmentMetadataQueryTest( new ColumnAnalysis( ValueType.STRING.toString(), false, - mmap2 ? (bitmaps ? 10881 : 9) : 0, + placementSize2, 1, null, null, @@ -259,7 +269,7 @@ public SegmentMetadataQueryTest( null ) // null_column will be included only for incremental index, which makes a little bigger result than expected - ), mmap2 ? (bitmaps ? 167493 : 119872) : 168188, + ), overallSize2, 1209, null, null, @@ -484,10 +494,16 @@ public void testSegmentMetadataQueryWithComplexColumnMerge() @Test public void testSegmentMetadataQueryWithDefaultAnalysisMerge() { + int size1 = 0; + int size2 = 0; + if (bitmaps) { + size1 = mmap1 ? 10881 : 10764; + size2 = mmap2 ? 10881 : 10764; + } ColumnAnalysis analysis = new ColumnAnalysis( ValueType.STRING.toString(), false, - (mmap1 ? bitmaps ? 10881 : 9 : 10764) + (mmap2 ? bitmaps ? 10881 : 9 : 10764), + size1 + size2, 1, "preferred", "preferred", @@ -499,10 +515,16 @@ public void testSegmentMetadataQueryWithDefaultAnalysisMerge() @Test public void testSegmentMetadataQueryWithDefaultAnalysisMerge2() { + int size1 = 0; + int size2 = 0; + if (bitmaps) { + size1 = mmap1 ? 6882 : 6808; + size2 = mmap2 ? 6882 : 6808; + } ColumnAnalysis analysis = new ColumnAnalysis( ValueType.STRING.toString(), false, - (mmap1 ? bitmaps ? 6882 : 23 : 6808) + (mmap2 ? bitmaps ? 6882 : 23 : 6808), + size1 + size2, 3, "spot", "upfront", @@ -514,10 +536,16 @@ public void testSegmentMetadataQueryWithDefaultAnalysisMerge2() @Test public void testSegmentMetadataQueryWithDefaultAnalysisMerge3() { + int size1 = 0; + int size2 = 0; + if (bitmaps) { + size1 = mmap1 ? 9765 : 9660; + size2 = mmap2 ? 9765 : 9660; + } ColumnAnalysis analysis = new ColumnAnalysis( ValueType.STRING.toString(), false, - (mmap1 ? bitmaps ? 9765 : 73 : 9660) + (mmap2 ? bitmaps ? 9765 : 73 : 9660), + size1 + size2, 9, "automotive", "travel", From 02fdf68eb854c6abbf007ac5d267055917ee126e Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 20 Feb 2019 17:56:14 -0800 Subject: [PATCH 4/6] less ternary --- .../metadata/SegmentMetadataQueryTest.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java index 40c477b08f7a..5c613acf658a 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java @@ -87,11 +87,12 @@ public static QueryRunner makeMMappedQueryRunner( QueryRunnerFactory factory ) { - QueryableIndex index = bitmaps - ? rollup - ? TestIndex.getMMappedTestIndex() - : TestIndex.getNoRollupMMappedTestIndex() - : TestIndex.getNoBitmapMMappedTestIndex(); + QueryableIndex index; + if (bitmaps) { + index = rollup ? TestIndex.getMMappedTestIndex() : TestIndex.getNoRollupMMappedTestIndex(); + } else { + index = TestIndex.getNoBitmapMMappedTestIndex(); + } return QueryRunnerTestHelper.makeQueryRunner( factory, segmentId, @@ -108,11 +109,12 @@ public static QueryRunner makeIncrementalIndexQueryRunner( QueryRunnerFactory factory ) { - IncrementalIndex index = bitmaps - ? rollup - ? TestIndex.getIncrementalTestIndex() - : TestIndex.getNoRollupIncrementalTestIndex() - : TestIndex.getNoBitmapIncrementalTestIndex(); + IncrementalIndex index; + if (bitmaps) { + index = rollup ? TestIndex.getIncrementalTestIndex() : TestIndex.getNoRollupIncrementalTestIndex(); + } else { + index = TestIndex.getNoBitmapIncrementalTestIndex(); + } return QueryRunnerTestHelper.makeQueryRunner( factory, segmentId, From 8e1c37a5cb54869546b0ab957e939f60544fcef4 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 22 Feb 2019 19:41:27 -0800 Subject: [PATCH 5/6] fix it --- .../org/apache/druid/segment/TestIndex.java | 164 +++++++----------- 1 file changed, 63 insertions(+), 101 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index f44254f3677b..87d8abf11586 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -20,6 +20,7 @@ package org.apache.druid.segment; import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.base.Throwables; import com.google.common.io.CharSource; import com.google.common.io.LineProcessor; @@ -166,134 +167,95 @@ public class TestIndex } } - private static IncrementalIndex realtimeIndex = null; - private static IncrementalIndex noRollupRealtimeIndex = null; - private static IncrementalIndex noBitmapRealtimeIndex = null; - private static QueryableIndex mmappedIndex = null; - private static QueryableIndex noRollupMmappedIndex = null; - private static QueryableIndex mergedRealtime = null; - private static QueryableIndex noBitmapMmappedIndex = null; + private static Supplier realtimeIndex = Suppliers.memoize( + () -> makeRealtimeIndex("druid.sample.numeric.tsv") + ); + private static Supplier noRollupRealtimeIndex = Suppliers.memoize( + () -> makeRealtimeIndex("druid.sample.numeric.tsv", false) + ); + private static Supplier noBitmapRealtimeIndex = Suppliers.memoize( + () -> makeRealtimeIndex("druid.sample.numeric.tsv", false, false) + ); + private static Supplier mmappedIndex = Suppliers.memoize( + () -> persistRealtimeAndLoadMMapped(realtimeIndex.get()) + ); + private static Supplier noRollupMmappedIndex = Suppliers.memoize( + () -> persistRealtimeAndLoadMMapped(noRollupRealtimeIndex.get()) + ); + private static Supplier noBitmapMmappedIndex = Suppliers.memoize( + () -> persistRealtimeAndLoadMMapped(noBitmapRealtimeIndex.get()) + ); + private static Supplier mergedRealtime = Suppliers.memoize(() -> { + try { + IncrementalIndex top = makeRealtimeIndex("druid.sample.numeric.tsv.top"); + IncrementalIndex bottom = makeRealtimeIndex("druid.sample.numeric.tsv.bottom"); + + File tmpFile = File.createTempFile("yay", "who"); + tmpFile.delete(); + + File topFile = new File(tmpFile, "top"); + File bottomFile = new File(tmpFile, "bottom"); + File mergedFile = new File(tmpFile, "merged"); + + topFile.mkdirs(); + topFile.deleteOnExit(); + bottomFile.mkdirs(); + bottomFile.deleteOnExit(); + mergedFile.mkdirs(); + mergedFile.deleteOnExit(); + + INDEX_MERGER.persist(top, DATA_INTERVAL, topFile, indexSpec, null); + INDEX_MERGER.persist(bottom, DATA_INTERVAL, bottomFile, indexSpec, null); + + return INDEX_IO.loadIndex( + INDEX_MERGER.mergeQueryableIndex( + Arrays.asList(INDEX_IO.loadIndex(topFile), INDEX_IO.loadIndex(bottomFile)), + true, + METRIC_AGGS, + mergedFile, + indexSpec, + null + ) + ); + } + catch (IOException e) { + throw Throwables.propagate(e); + } + }); public static IncrementalIndex getIncrementalTestIndex() { - synchronized (log) { - if (realtimeIndex != null) { - return realtimeIndex; - } - } - - return realtimeIndex = makeRealtimeIndex("druid.sample.numeric.tsv"); + return realtimeIndex.get(); } public static IncrementalIndex getNoRollupIncrementalTestIndex() { - synchronized (log) { - if (noRollupRealtimeIndex != null) { - return noRollupRealtimeIndex; - } - } - - return noRollupRealtimeIndex = makeRealtimeIndex("druid.sample.numeric.tsv", false); + return noRollupRealtimeIndex.get(); } public static IncrementalIndex getNoBitmapIncrementalTestIndex() { - synchronized (log) { - if (noBitmapRealtimeIndex != null) { - return noBitmapRealtimeIndex; - } - } - - return noBitmapRealtimeIndex = makeRealtimeIndex("druid.sample.numeric.tsv", false, false); + return noBitmapRealtimeIndex.get(); } public static QueryableIndex getMMappedTestIndex() { - synchronized (log) { - if (mmappedIndex != null) { - return mmappedIndex; - } - } - - IncrementalIndex incrementalIndex = getIncrementalTestIndex(); - mmappedIndex = persistRealtimeAndLoadMMapped(incrementalIndex); - - return mmappedIndex; + return mmappedIndex.get(); } public static QueryableIndex getNoRollupMMappedTestIndex() { - synchronized (log) { - if (noRollupMmappedIndex != null) { - return noRollupMmappedIndex; - } - } - - IncrementalIndex incrementalIndex = getNoRollupIncrementalTestIndex(); - noRollupMmappedIndex = persistRealtimeAndLoadMMapped(incrementalIndex); - - return noRollupMmappedIndex; + return noRollupMmappedIndex.get(); } public static QueryableIndex getNoBitmapMMappedTestIndex() { - synchronized (log) { - if (noBitmapMmappedIndex != null) { - return noBitmapMmappedIndex; - } - } - - IncrementalIndex incrementalIndex = getNoBitmapIncrementalTestIndex(); - noBitmapMmappedIndex = persistRealtimeAndLoadMMapped(incrementalIndex); - - return noBitmapMmappedIndex; + return noBitmapMmappedIndex.get(); } public static QueryableIndex mergedRealtimeIndex() { - synchronized (log) { - if (mergedRealtime != null) { - return mergedRealtime; - } - - try { - IncrementalIndex top = makeRealtimeIndex("druid.sample.numeric.tsv.top"); - IncrementalIndex bottom = makeRealtimeIndex("druid.sample.numeric.tsv.bottom"); - - File tmpFile = File.createTempFile("yay", "who"); - tmpFile.delete(); - - File topFile = new File(tmpFile, "top"); - File bottomFile = new File(tmpFile, "bottom"); - File mergedFile = new File(tmpFile, "merged"); - - topFile.mkdirs(); - topFile.deleteOnExit(); - bottomFile.mkdirs(); - bottomFile.deleteOnExit(); - mergedFile.mkdirs(); - mergedFile.deleteOnExit(); - - INDEX_MERGER.persist(top, DATA_INTERVAL, topFile, indexSpec, null); - INDEX_MERGER.persist(bottom, DATA_INTERVAL, bottomFile, indexSpec, null); - - mergedRealtime = INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( - Arrays.asList(INDEX_IO.loadIndex(topFile), INDEX_IO.loadIndex(bottomFile)), - true, - METRIC_AGGS, - mergedFile, - indexSpec, - null - ) - ); - - return mergedRealtime; - } - catch (IOException e) { - throw Throwables.propagate(e); - } - } + return mergedRealtime.get(); } public static IncrementalIndex makeRealtimeIndex(final String resourceFilename) From 59b10dbd987f2dc240d5270dc889940afc386d39 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 25 Feb 2019 14:59:18 -0800 Subject: [PATCH 6/6] do the thing --- .../org/apache/druid/query/metadata/SegmentAnalyzer.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java index 376ddaaa12a4..15acc4ce7abc 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java @@ -195,11 +195,10 @@ private ColumnAnalysis analyzeStringColumn( final ColumnHolder columnHolder ) { - long size = 0; - Comparable min = null; Comparable max = null; - int cardinality = 0; + long size = 0; + final int cardinality; if (capabilities.hasBitmapIndexes()) { final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); cardinality = bitmapIndex.getCardinality(); @@ -226,6 +225,8 @@ private ColumnAnalysis analyzeStringColumn( min = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(0)); max = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(cardinality - 1)); } + } else { + cardinality = 0; } return new ColumnAnalysis(