From f8d5c5ee8cb9d536017f6aaa538dbdc21f2ff9d3 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 21 Dec 2016 03:57:30 +0900 Subject: [PATCH 1/9] Add an option to SearchQuery to choose a search query execution strategy. Supported strategies are 1) Index-only query execution 2) Cursor-based scan 3) Auto: choose an efficient strategy for a given query --- .../benchmark/query/SearchBenchmark.java | 58 ++++-- .../src/main/java/io/druid/query/Druids.java | 17 +- .../druid/query/search/SearchQueryRunner.java | 190 +++++++++++++----- .../query/search/search/SearchQuery.java | 54 +++-- .../search/search/SearchQueryConfig.java | 4 + .../search/SearchQueryQueryToolChestTest.java | 1 + 6 files changed, 236 insertions(+), 88 deletions(-) diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java index 0627bb9712d3..71794ff77d02 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java @@ -24,7 +24,6 @@ import com.google.common.collect.Maps; import com.google.common.hash.Hashing; import com.google.common.io.Files; - import io.druid.benchmark.datagen.BenchmarkDataGenerator; import io.druid.benchmark.datagen.BenchmarkSchemaInfo; import io.druid.benchmark.datagen.BenchmarkSchemas; @@ -44,13 +43,10 @@ import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; import io.druid.query.Result; -import io.druid.query.aggregation.AggregatorFactory; -import io.druid.query.aggregation.DoubleMinAggregatorFactory; -import io.druid.query.aggregation.DoubleSumAggregatorFactory; -import io.druid.query.aggregation.LongMaxAggregatorFactory; -import io.druid.query.aggregation.LongSumAggregatorFactory; -import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesSerde; +import io.druid.query.filter.AndDimFilter; +import io.druid.query.filter.DimFilter; +import io.druid.query.filter.InDimFilter; import io.druid.query.search.SearchQueryQueryToolChest; import io.druid.query.search.SearchQueryRunnerFactory; import io.druid.query.search.SearchResultValue; @@ -112,7 +108,6 @@ public class SearchBenchmark private int limit; private static final Logger log = new Logger(SearchBenchmark.class); - private static final int RNG_SEED = 9999; private static final IndexMergerV9 INDEX_MERGER_V9; private static final IndexIO INDEX_IO; public static final ObjectMapper JSON_MAPPER; @@ -154,13 +149,6 @@ private void setupQueries() { // basic.A QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(Arrays.asList(basicSchema.getDataInterval())); - List queryAggs = new ArrayList<>(); - queryAggs.add(new LongSumAggregatorFactory("sumLongSequential", "sumLongSequential")); - queryAggs.add(new LongMaxAggregatorFactory("maxLongUniform", "maxLongUniform")); - queryAggs.add(new DoubleSumAggregatorFactory("sumFloatNormal", "sumFloatNormal")); - queryAggs.add(new DoubleMinAggregatorFactory("minFloatZipf", "minFloatZipf")); - queryAggs.add(new HyperUniquesAggregatorFactory("hyperUniquesMet", "hyper")); - Druids.SearchQueryBuilder queryBuilderA = Druids.newSearchQueryBuilder() .dataSource("blah") @@ -171,6 +159,39 @@ private void setupQueries() basicQueries.put("A", queryBuilderA); } + { // basic.B + QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(Arrays.asList(basicSchema.getDataInterval())); + + List dimUniformFilterVals = Lists.newArrayList(); + int resultNum = (int) (100000 * 0.5); + int step = 100000 / resultNum; + for (int i = 1; i < 100001 && dimUniformFilterVals.size() < resultNum; i += step) { + dimUniformFilterVals.add(String.valueOf(i)); + } + + List dimHyperUniqueFilterVals = Lists.newArrayList(); + resultNum = (int) (100000 * 1.0); + step = 100000 / resultNum; + for (int i = 0; i < 100001 && dimHyperUniqueFilterVals.size() < resultNum; i += step) { + dimHyperUniqueFilterVals.add(String.valueOf(i)); + } + + final List dimFilters = Lists.newArrayList(); + dimFilters.add(new InDimFilter("dimUniform", dimUniformFilterVals, null)); + dimFilters.add(new InDimFilter("dimHyperUnique", dimHyperUniqueFilterVals, null)); + + Druids.SearchQueryBuilder queryBuilderB = + Druids.newSearchQueryBuilder() + .dataSource("blah") + .granularity(QueryGranularities.ALL) + .intervals(intervalSpec) + .query("") + .dimensions(Lists.newArrayList("dimUniform", "dimHyperUnique")) + .filters(new AndDimFilter(dimFilters)); + + basicQueries.put("B", queryBuilderB); + } + SCHEMA_QUERY_MAP.put("basic", basicQueries); } @@ -200,7 +221,7 @@ public void setup() throws IOException log.info("Generating rows for segment " + i); BenchmarkDataGenerator gen = new BenchmarkDataGenerator( schemaInfo.getColumnSchemas(), - RNG_SEED + i, + System.currentTimeMillis(), schemaInfo.getDataInterval(), rowsPerSegment ); @@ -233,9 +254,12 @@ public void setup() throws IOException qIndexes.add(qIndex); } + final SearchQueryConfig config = new SearchQueryConfig(); + config.setMaxSearchLimit(limit); + factory = new SearchQueryRunnerFactory( new SearchQueryQueryToolChest( - new SearchQueryConfig(), + config, QueryBenchmarkUtil.NoopIntervalChunkingQueryRunnerDecorator() ), QueryBenchmarkUtil.NOOP_QUERYWATCHER diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index dfa7d238fb50..e873498a7c88 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -45,6 +45,7 @@ import io.druid.query.search.search.FragmentSearchQuerySpec; import io.druid.query.search.search.InsensitiveContainsSearchQuerySpec; import io.druid.query.search.search.SearchQuery; +import io.druid.query.search.search.SearchQuery.Strategy; import io.druid.query.search.search.SearchQuerySpec; import io.druid.query.search.search.SearchSortSpec; import io.druid.query.select.PagingSpec; @@ -549,6 +550,7 @@ public static class SearchQueryBuilder private SearchQuerySpec querySpec; private SearchSortSpec sortSpec; private Map context; + private Strategy strategy; public SearchQueryBuilder() { @@ -560,6 +562,7 @@ public SearchQueryBuilder() dimensions = null; querySpec = null; context = null; + strategy = null; } public SearchQuery build() @@ -573,7 +576,8 @@ public SearchQuery build() dimensions, querySpec, sortSpec, - context + context, + strategy ); } @@ -587,7 +591,8 @@ public SearchQueryBuilder copy(SearchQuery query) .limit(query.getLimit()) .dimensions(query.getDimensions()) .query(query.getQuery()) - .context(query.getContext()); + .context(query.getContext()) + .strategy(query.getStrategy()); } public SearchQueryBuilder copy(SearchQueryBuilder builder) @@ -600,7 +605,8 @@ public SearchQueryBuilder copy(SearchQueryBuilder builder) .limit(builder.limit) .dimensions(builder.dimensions) .query(builder.querySpec) - .context(builder.context); + .context(builder.context) + .strategy(builder.strategy); } public SearchQueryBuilder dataSource(String d) @@ -750,6 +756,11 @@ public SearchQueryBuilder context(Map c) context = c; return this; } + + public SearchQueryBuilder strategy(Strategy strategy) { + this.strategy = strategy; + return this; + } } public static SearchQueryBuilder newSearchQueryBuilder() diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 9e6b7e85920d..17dbd5d66297 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -25,10 +25,11 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.metamx.emitter.EmittingLogger; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.MutableBitmap; -import com.metamx.emitter.EmittingLogger; +import io.druid.collections.bitmap.RoaringBitmapFactory; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Accumulator; @@ -45,6 +46,7 @@ import io.druid.query.filter.Filter; import io.druid.query.search.search.SearchHit; import io.druid.query.search.search.SearchQuery; +import io.druid.query.search.search.SearchQuery.Strategy; import io.druid.query.search.search.SearchQuerySpec; import io.druid.segment.ColumnSelectorBitmapIndexSelector; import io.druid.segment.Cursor; @@ -71,6 +73,12 @@ public class SearchQueryRunner implements QueryRunner> { private static final EmittingLogger log = new EmittingLogger(SearchQueryRunner.class); + + private static final double HIGH_FILTER_SELECTIVITY_THRESHOLD_FOR_CONCISE = 0.99; + private static final double HIGH_FILTER_SELECTIVITY_THRESHOLD_FOR_ROARING = 0.65; + private static final int LOW_CARDINALITY_THRESHOLD_FOR_CONCISE = 5000; + private static final int LOW_CARDINALITY_THRESHOLD_FOR_ROARING = 1000; + private final Segment segment; public SearchQueryRunner(Segment segment) @@ -94,6 +102,7 @@ public Sequence> run( final SearchQuerySpec searchQuerySpec = query.getQuery(); final int limit = query.getLimit(); final boolean descending = query.isDescending(); + final Strategy strategy = query.getStrategy(); final List intervals = query.getQuerySegmentSpec().getIntervals(); if (intervals.size() != 1) { throw new IAE("Should only have one interval, got[%s]", intervals); @@ -103,22 +112,32 @@ public Sequence> run( // Closing this will cause segfaults in unit tests. final QueryableIndex index = segment.asQueryableIndex(); - if (index != null) { - final TreeMap retVal = Maps.newTreeMap(query.getSort().getComparator()); + final StorageAdapter adapter = segment.asStorageAdapter(); - Iterable dimsToSearch; - if (dimensions == null || dimensions.isEmpty()) { - dimsToSearch = Iterables.transform(index.getAvailableDimensions(), Druids.DIMENSION_IDENTITY); - } else { - dimsToSearch = dimensions; - } + if (adapter == null) { + log.makeAlert("WTF!? Unable to process search query on segment.") + .addData("segment", segment.getIdentifier()) + .addData("query", query).emit(); + throw new ISE( + "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped." + ); + } + + final Iterable dimsToSearch; + if (dimensions == null || dimensions.isEmpty()) { + dimsToSearch = Iterables.transform(adapter.getAvailableDimensions(), Druids.DIMENSION_IDENTITY); + } else { + dimsToSearch = dimensions; + } + + if (index != null) { final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); final ImmutableBitmap baseFilter = filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); - ImmutableBitmap timeFilteredBitmap; + final ImmutableBitmap timeFilteredBitmap; if (!interval.contains(segment.getDataInterval())) { MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); final Column timeColumn = index.getColumn(Column.TIME_COLUMN_NAME); @@ -142,62 +161,125 @@ public Sequence> run( timeFilteredBitmap = baseFilter; } - for (DimensionSpec dimension : dimsToSearch) { - final Column column = index.getColumn(dimension.getDimension()); - if (column == null) { - continue; - } - - final BitmapIndex bitmapIndex = column.getBitmapIndex(); - ExtractionFn extractionFn = dimension.getExtractionFn(); - if (extractionFn == null) { - extractionFn = IdentityExtractionFn.getInstance(); - } - if (bitmapIndex != null) { - for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { - String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); - if (!searchQuerySpec.accept(dimVal)) { - continue; - } - ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); - if (timeFilteredBitmap != null) { - bitmap = bitmapFactory.intersection(Arrays.asList(timeFilteredBitmap, bitmap)); - } - if (bitmap.size() > 0) { - MutableInt counter = new MutableInt(bitmap.size()); - MutableInt prev = retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter); - if (prev != null) { - counter.add(prev.intValue()); - } - if (retVal.size() >= limit) { - return makeReturnResult(limit, retVal); + switch (strategy) { + case AUTO: + long totalCard = 0; + for (DimensionSpec dimension : dimsToSearch) { + final Column column = index.getColumn(dimension.getDimension()); + if (column != null) { + final BitmapIndex bitmapIndex = column.getBitmapIndex(); + if (bitmapIndex != null) { + totalCard += bitmapIndex.getCardinality(); } } } - } + + // Choose a search query execution strategy depending on the query. + // Index-only strategy is selected when + // 1) there is no filter, + // 2) the total cardinality is very low, or + // 3) the filter has a very high selectivity. + // The below thresholds are got from some experiments. + if (filter == null || + isLowCardinality(bitmapFactory, totalCard) || + isHighSelectivityFilter(index, bitmapFactory, timeFilteredBitmap)) { + return indexOnlyExecute(query, searchQuerySpec, timeFilteredBitmap, dimsToSearch, index, limit, bitmapFactory); + } else { + return cursorBasedExecute(query, searchQuerySpec, filter, dimsToSearch, adapter, limit, descending, interval); + } + + case INDEX_ONLY: + return indexOnlyExecute(query, searchQuerySpec, timeFilteredBitmap, dimsToSearch, index, limit, bitmapFactory); + + case CURSOR_BASED: + return cursorBasedExecute(query, searchQuerySpec, filter, dimsToSearch, adapter, limit, descending, interval); + + default: + throw new IAE("Unknown strategy: " + strategy); } - return makeReturnResult(limit, retVal); + } else { + return cursorBasedExecute(query, searchQuerySpec, filter, dimsToSearch, adapter, limit, descending, interval); } + } - final StorageAdapter adapter = segment.asStorageAdapter(); - - if (adapter == null) { - log.makeAlert("WTF!? Unable to process search query on segment.") - .addData("segment", segment.getIdentifier()) - .addData("query", query).emit(); - throw new ISE( - "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped." - ); + private static boolean isLowCardinality(final BitmapFactory bitmapFactory, final long totalCard) { + if (bitmapFactory.getClass().equals(RoaringBitmapFactory.class)) { + return totalCard < LOW_CARDINALITY_THRESHOLD_FOR_ROARING; + } else { + return totalCard < LOW_CARDINALITY_THRESHOLD_FOR_CONCISE; } + } - final Iterable dimsToSearch; - if (dimensions == null || dimensions.isEmpty()) { - dimsToSearch = Iterables.transform(adapter.getAvailableDimensions(), Druids.DIMENSION_IDENTITY); + private static boolean isHighSelectivityFilter(final QueryableIndex index, + final BitmapFactory bitmapFactory, + final ImmutableBitmap bitmap) { + if (bitmapFactory.getClass().equals(RoaringBitmapFactory.class)) { + return index.getNumRows() * HIGH_FILTER_SELECTIVITY_THRESHOLD_FOR_ROARING < bitmap.size(); } else { - dimsToSearch = dimensions; + return index.getNumRows() * HIGH_FILTER_SELECTIVITY_THRESHOLD_FOR_CONCISE < bitmap.size(); + } + } + + private Sequence> indexOnlyExecute(final SearchQuery query, + final SearchQuerySpec searchQuerySpec, + final ImmutableBitmap timeFilteredBitmap, + final Iterable dimsToSearch, + final QueryableIndex index, + final int limit, + final BitmapFactory bitmapFactory) + { + log.info("Index only query execution strategy is selected."); + final TreeMap retVal = Maps.newTreeMap(query.getSort().getComparator()); + for (DimensionSpec dimension : dimsToSearch) { + final Column column = index.getColumn(dimension.getDimension()); + if (column == null) { + continue; + } + + final BitmapIndex bitmapIndex = column.getBitmapIndex(); + ExtractionFn extractionFn = dimension.getExtractionFn(); + if (extractionFn == null) { + extractionFn = IdentityExtractionFn.getInstance(); + } + if (bitmapIndex != null) { + for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { + String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); + if (!searchQuerySpec.accept(dimVal)) { + continue; + } + ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); + if (timeFilteredBitmap != null) { + bitmap = bitmapFactory.intersection(Arrays.asList(timeFilteredBitmap, bitmap)); + } + if (bitmap.size() > 0) { + MutableInt counter = new MutableInt(bitmap.size()); + MutableInt prev = retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter); + if (prev != null) { + counter.add(prev.intValue()); + } + if (retVal.size() >= limit) { + return makeReturnResult(limit, retVal); + } + } + } + } } + return makeReturnResult(limit, retVal); + } + + private Sequence> cursorBasedExecute(final SearchQuery query, + final SearchQuerySpec searchQuerySpec, + final Filter filter, + final Iterable dimsToSearch, + final StorageAdapter adapter, + final int limit, + final boolean descending, + final Interval interval) + { + log.info("Cursor-based query execution strategy is selected."); + final Sequence cursors = adapter.makeCursors( filter, interval, diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQuery.java b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java index 2792800d7ca8..23fddc4bcb99 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQuery.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java @@ -49,6 +49,14 @@ public class SearchQuery extends BaseQuery> private final List dimensions; private final SearchQuerySpec querySpec; private final int limit; + private final Strategy strategy; + + public enum Strategy + { + AUTO, + INDEX_ONLY, + CURSOR_BASED + } @JsonCreator public SearchQuery( @@ -60,7 +68,8 @@ public SearchQuery( @JsonProperty("searchDimensions") List dimensions, @JsonProperty("query") SearchQuerySpec querySpec, @JsonProperty("sort") SearchSortSpec sortSpec, - @JsonProperty("context") Map context + @JsonProperty("context") Map context, + @JsonProperty("strategy") Strategy strategy ) { super(dataSource, querySegmentSpec, false, context); @@ -70,6 +79,7 @@ public SearchQuery( this.limit = (limit == 0) ? 1000 : limit; this.dimensions = dimensions; this.querySpec = querySpec == null ? new AllSearchQuerySpec() : querySpec; + this.strategy = strategy == null ? Strategy.AUTO : strategy; Preconditions.checkNotNull(querySegmentSpec, "Must specify an interval"); } @@ -104,7 +114,8 @@ public SearchQuery withQuerySegmentSpec(QuerySegmentSpec spec) dimensions, querySpec, sortSpec, - getContext() + getContext(), + strategy ); } @@ -120,7 +131,8 @@ public Query> withDataSource(DataSource dataSource) dimensions, querySpec, sortSpec, - getContext() + getContext(), + strategy ); } @@ -136,7 +148,8 @@ public SearchQuery withOverriddenContext(Map contextOverrides) dimensions, querySpec, sortSpec, - computeOverridenContext(contextOverrides) + computeOverridenContext(contextOverrides), + strategy ); } @@ -151,7 +164,8 @@ public SearchQuery withDimFilter(DimFilter dimFilter) dimensions, querySpec, sortSpec, - getContext() + getContext(), + strategy ); } @@ -191,6 +205,11 @@ public SearchSortSpec getSort() return sortSpec; } + @JsonProperty("strategy") + public Strategy getStrategy() { + return strategy; + } + public SearchQuery withLimit(int newLimit) { return new SearchQuery( @@ -202,7 +221,8 @@ public SearchQuery withLimit(int newLimit) dimensions, querySpec, sortSpec, - getContext() + getContext(), + strategy ); } @@ -210,14 +230,15 @@ public SearchQuery withLimit(int newLimit) public String toString() { return "SearchQuery{" + - "dataSource='" + getDataSource() + '\'' + - ", dimFilter=" + dimFilter + - ", granularity='" + granularity + '\'' + - ", dimensions=" + dimensions + - ", querySpec=" + querySpec + - ", querySegmentSpec=" + getQuerySegmentSpec() + - ", limit=" + limit + - '}'; + "dataSource='" + getDataSource() + '\'' + + ", dimFilter=" + dimFilter + + ", granularity='" + granularity + '\'' + + ", dimensions=" + dimensions + + ", querySpec=" + querySpec + + ", querySegmentSpec=" + getQuerySegmentSpec() + + ", limit=" + limit + + ", strategy=" + strategy + + '}'; } @Override @@ -254,6 +275,10 @@ public boolean equals(Object o) return false; } + if (!strategy.equals(that.strategy)) { + return false; + } + return true; } @@ -267,6 +292,7 @@ public int hashCode() result = 31 * result + (dimensions != null ? dimensions.hashCode() : 0); result = 31 * result + (querySpec != null ? querySpec.hashCode() : 0); result = 31 * result + limit; + result = 31 * result + strategy.hashCode(); return result; } } diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java index 9a6ae69afc59..7d2ed451bb4e 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java @@ -31,6 +31,10 @@ public class SearchQueryConfig @Min(1) private int maxSearchLimit = 1000; + public void setMaxSearchLimit(int maxSearchLimit) { + this.maxSearchLimit = maxSearchLimit; + } + public int getMaxSearchLimit() { return maxSearchLimit; diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryQueryToolChestTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryQueryToolChestTest.java index a3e9b61b9417..9787e4f6770c 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryQueryToolChestTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryQueryToolChestTest.java @@ -59,6 +59,7 @@ public void testCacheStrategy() throws Exception ImmutableList.of(Druids.DIMENSION_IDENTITY.apply("dim1")), new FragmentSearchQuerySpec(ImmutableList.of("a", "b")), null, + null, null ) ); From 3e5029bd40dde0025531573ed7e08eb3bc003ab0 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 22 Dec 2016 02:22:52 +0900 Subject: [PATCH 2/9] Add SearchStrategy and SearchQueryExecutor --- .../src/main/java/io/druid/query/Druids.java | 6 +- .../druid/query/search/SearchQueryRunner.java | 345 +----------------- .../query/search/search/AutoStrategy.java | 60 +++ .../search/ConciseBitmapDecisionHelper.java | 17 + .../search/search/CursorBasedStrategy.java | 101 +++++ .../search/search/IndexOnlyStrategy.java | 167 +++++++++ .../search/RoaringBitmapDecisionHelper.java | 17 + .../query/search/search/SearchQuery.java | 15 +- .../search/SearchQueryDecisionHelper.java | 50 +++ .../search/search/SearchQueryExecutor.java | 89 +++++ .../query/search/search/SearchStrategy.java | 54 +++ .../main/java/io/druid/segment/IndexIO.java | 13 +- .../java/io/druid/segment/QueryableIndex.java | 17 +- .../druid/segment/SimpleQueryableIndex.java | 10 + .../segment/data/BitmapSerdeFactory.java | 7 +- .../data/ConciseBitmapSerdeFactory.java | 9 + .../data/RoaringBitmapSerdeFactory.java | 9 + 17 files changed, 619 insertions(+), 367 deletions(-) create mode 100644 processing/src/main/java/io/druid/query/search/search/AutoStrategy.java create mode 100644 processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java create mode 100644 processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java create mode 100644 processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java create mode 100644 processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java create mode 100644 processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java create mode 100644 processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java create mode 100644 processing/src/main/java/io/druid/query/search/search/SearchStrategy.java diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index e873498a7c88..d5b1ed0470aa 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -45,9 +45,9 @@ import io.druid.query.search.search.FragmentSearchQuerySpec; import io.druid.query.search.search.InsensitiveContainsSearchQuerySpec; import io.druid.query.search.search.SearchQuery; -import io.druid.query.search.search.SearchQuery.Strategy; import io.druid.query.search.search.SearchQuerySpec; import io.druid.query.search.search.SearchSortSpec; +import io.druid.query.search.search.SearchStrategy; import io.druid.query.select.PagingSpec; import io.druid.query.select.SelectQuery; import io.druid.query.spec.LegacySegmentSpec; @@ -550,7 +550,7 @@ public static class SearchQueryBuilder private SearchQuerySpec querySpec; private SearchSortSpec sortSpec; private Map context; - private Strategy strategy; + private SearchStrategy strategy; public SearchQueryBuilder() { @@ -757,7 +757,7 @@ public SearchQueryBuilder context(Map c) return this; } - public SearchQueryBuilder strategy(Strategy strategy) { + public SearchQueryBuilder strategy(SearchStrategy strategy) { this.strategy = strategy; return this; } diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 17dbd5d66297..7dd385d75b7a 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -19,66 +19,21 @@ package io.druid.query.search; -import com.google.common.base.Function; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.metamx.emitter.EmittingLogger; -import io.druid.collections.bitmap.BitmapFactory; -import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.collections.bitmap.MutableBitmap; -import io.druid.collections.bitmap.RoaringBitmapFactory; -import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; -import io.druid.java.util.common.guava.Accumulator; -import io.druid.java.util.common.guava.FunctionalIterable; import io.druid.java.util.common.guava.Sequence; -import io.druid.java.util.common.guava.Sequences; -import io.druid.query.Druids; import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.Result; -import io.druid.query.dimension.DimensionSpec; -import io.druid.query.extraction.ExtractionFn; -import io.druid.query.extraction.IdentityExtractionFn; -import io.druid.query.filter.Filter; -import io.druid.query.search.search.SearchHit; import io.druid.query.search.search.SearchQuery; -import io.druid.query.search.search.SearchQuery.Strategy; -import io.druid.query.search.search.SearchQuerySpec; -import io.druid.segment.ColumnSelectorBitmapIndexSelector; -import io.druid.segment.Cursor; -import io.druid.segment.DimensionSelector; -import io.druid.segment.QueryableIndex; +import io.druid.query.search.search.SearchStrategy; import io.druid.segment.Segment; -import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; -import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; -import io.druid.segment.column.GenericColumn; -import io.druid.segment.data.IndexedInts; -import io.druid.segment.filter.Filters; -import org.apache.commons.lang.mutable.MutableInt; -import org.joda.time.Interval; -import java.util.Arrays; -import java.util.List; import java.util.Map; -import java.util.TreeMap; /** */ public class SearchQueryRunner implements QueryRunner> { - private static final EmittingLogger log = new EmittingLogger(SearchQueryRunner.class); - - private static final double HIGH_FILTER_SELECTIVITY_THRESHOLD_FOR_CONCISE = 0.99; - private static final double HIGH_FILTER_SELECTIVITY_THRESHOLD_FOR_ROARING = 0.65; - private static final int LOW_CARDINALITY_THRESHOLD_FOR_CONCISE = 5000; - private static final int LOW_CARDINALITY_THRESHOLD_FOR_ROARING = 1000; - private final Segment segment; public SearchQueryRunner(Segment segment) @@ -97,302 +52,8 @@ public Sequence> run( } final SearchQuery query = (SearchQuery) input; - final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter())); - final List dimensions = query.getDimensions(); - final SearchQuerySpec searchQuerySpec = query.getQuery(); - final int limit = query.getLimit(); - final boolean descending = query.isDescending(); - final Strategy strategy = query.getStrategy(); - final List intervals = query.getQuerySegmentSpec().getIntervals(); - if (intervals.size() != 1) { - throw new IAE("Should only have one interval, got[%s]", intervals); - } - final Interval interval = intervals.get(0); - - // Closing this will cause segfaults in unit tests. - final QueryableIndex index = segment.asQueryableIndex(); - - final StorageAdapter adapter = segment.asStorageAdapter(); - - if (adapter == null) { - log.makeAlert("WTF!? Unable to process search query on segment.") - .addData("segment", segment.getIdentifier()) - .addData("query", query).emit(); - throw new ISE( - "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped." - ); - } - - final Iterable dimsToSearch; - if (dimensions == null || dimensions.isEmpty()) { - dimsToSearch = Iterables.transform(adapter.getAvailableDimensions(), Druids.DIMENSION_IDENTITY); - } else { - dimsToSearch = dimensions; - } - - if (index != null) { - - final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); - - final ImmutableBitmap baseFilter = - filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); - - final ImmutableBitmap timeFilteredBitmap; - if (!interval.contains(segment.getDataInterval())) { - MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); - final Column timeColumn = index.getColumn(Column.TIME_COLUMN_NAME); - try (final GenericColumn timeValues = timeColumn.getGenericColumn()) { - - int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true)); - int endIndex = Math.min( - timeValues.length() - 1, - getStartIndexOfTime(timeValues, interval.getEndMillis(), false) - ); - - for (int i = startIndex; i <= endIndex; i++) { - timeBitmap.add(i); - } - - final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap); - timeFilteredBitmap = - (baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter); - } - } else { - timeFilteredBitmap = baseFilter; - } - - switch (strategy) { - case AUTO: - long totalCard = 0; - for (DimensionSpec dimension : dimsToSearch) { - final Column column = index.getColumn(dimension.getDimension()); - if (column != null) { - final BitmapIndex bitmapIndex = column.getBitmapIndex(); - if (bitmapIndex != null) { - totalCard += bitmapIndex.getCardinality(); - } - } - } - - // Choose a search query execution strategy depending on the query. - // Index-only strategy is selected when - // 1) there is no filter, - // 2) the total cardinality is very low, or - // 3) the filter has a very high selectivity. - // The below thresholds are got from some experiments. - if (filter == null || - isLowCardinality(bitmapFactory, totalCard) || - isHighSelectivityFilter(index, bitmapFactory, timeFilteredBitmap)) { - return indexOnlyExecute(query, searchQuerySpec, timeFilteredBitmap, dimsToSearch, index, limit, bitmapFactory); - } else { - return cursorBasedExecute(query, searchQuerySpec, filter, dimsToSearch, adapter, limit, descending, interval); - } - - case INDEX_ONLY: - return indexOnlyExecute(query, searchQuerySpec, timeFilteredBitmap, dimsToSearch, index, limit, bitmapFactory); - - case CURSOR_BASED: - return cursorBasedExecute(query, searchQuerySpec, filter, dimsToSearch, adapter, limit, descending, interval); - - default: - throw new IAE("Unknown strategy: " + strategy); - } - - } else { - return cursorBasedExecute(query, searchQuerySpec, filter, dimsToSearch, adapter, limit, descending, interval); - } - } - - private static boolean isLowCardinality(final BitmapFactory bitmapFactory, final long totalCard) { - if (bitmapFactory.getClass().equals(RoaringBitmapFactory.class)) { - return totalCard < LOW_CARDINALITY_THRESHOLD_FOR_ROARING; - } else { - return totalCard < LOW_CARDINALITY_THRESHOLD_FOR_CONCISE; - } - } + final SearchStrategy strategy = query.getStrategy(); - private static boolean isHighSelectivityFilter(final QueryableIndex index, - final BitmapFactory bitmapFactory, - final ImmutableBitmap bitmap) { - if (bitmapFactory.getClass().equals(RoaringBitmapFactory.class)) { - return index.getNumRows() * HIGH_FILTER_SELECTIVITY_THRESHOLD_FOR_ROARING < bitmap.size(); - } else { - return index.getNumRows() * HIGH_FILTER_SELECTIVITY_THRESHOLD_FOR_CONCISE < bitmap.size(); - } - } - - private Sequence> indexOnlyExecute(final SearchQuery query, - final SearchQuerySpec searchQuerySpec, - final ImmutableBitmap timeFilteredBitmap, - final Iterable dimsToSearch, - final QueryableIndex index, - final int limit, - final BitmapFactory bitmapFactory) - { - log.info("Index only query execution strategy is selected."); - final TreeMap retVal = Maps.newTreeMap(query.getSort().getComparator()); - for (DimensionSpec dimension : dimsToSearch) { - final Column column = index.getColumn(dimension.getDimension()); - if (column == null) { - continue; - } - - final BitmapIndex bitmapIndex = column.getBitmapIndex(); - ExtractionFn extractionFn = dimension.getExtractionFn(); - if (extractionFn == null) { - extractionFn = IdentityExtractionFn.getInstance(); - } - if (bitmapIndex != null) { - for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { - String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); - if (!searchQuerySpec.accept(dimVal)) { - continue; - } - ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); - if (timeFilteredBitmap != null) { - bitmap = bitmapFactory.intersection(Arrays.asList(timeFilteredBitmap, bitmap)); - } - if (bitmap.size() > 0) { - MutableInt counter = new MutableInt(bitmap.size()); - MutableInt prev = retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter); - if (prev != null) { - counter.add(prev.intValue()); - } - if (retVal.size() >= limit) { - return makeReturnResult(limit, retVal); - } - } - } - } - } - - return makeReturnResult(limit, retVal); - } - - private Sequence> cursorBasedExecute(final SearchQuery query, - final SearchQuerySpec searchQuerySpec, - final Filter filter, - final Iterable dimsToSearch, - final StorageAdapter adapter, - final int limit, - final boolean descending, - final Interval interval) - { - log.info("Cursor-based query execution strategy is selected."); - - final Sequence cursors = adapter.makeCursors( - filter, - interval, - VirtualColumns.EMPTY, - query.getGranularity(), - descending - ); - - final TreeMap retVal = cursors.accumulate( - Maps.newTreeMap(query.getSort().getComparator()), - new Accumulator, Cursor>() - { - @Override - public TreeMap accumulate(TreeMap set, Cursor cursor) - { - if (set.size() >= limit) { - return set; - } - - Map dimSelectors = Maps.newHashMap(); - for (DimensionSpec dim : dimsToSearch) { - dimSelectors.put( - dim.getOutputName(), - cursor.makeDimensionSelector(dim) - ); - } - - while (!cursor.isDone()) { - for (Map.Entry entry : dimSelectors.entrySet()) { - final DimensionSelector selector = entry.getValue(); - - if (selector != null) { - final IndexedInts vals = selector.getRow(); - for (int i = 0; i < vals.size(); ++i) { - final String dimVal = selector.lookupName(vals.get(i)); - if (searchQuerySpec.accept(dimVal)) { - MutableInt counter = new MutableInt(1); - MutableInt prev = set.put(new SearchHit(entry.getKey(), dimVal), counter); - if (prev != null) { - counter.add(prev.intValue()); - } - if (set.size() >= limit) { - return set; - } - } - } - } - } - - cursor.advance(); - } - - return set; - } - } - ); - - return makeReturnResult(limit, retVal); - } - - protected int getStartIndexOfTime(GenericColumn timeValues, long time, boolean inclusive) - { - int low = 0; - int high = timeValues.length() - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - long midVal = timeValues.getLongSingleValueRow(mid); - - if (midVal < time) { - low = mid + 1; - } else if (midVal > time) { - high = mid - 1; - } else { // key found - int i; - // rewind the index of the same time values - for (i = mid - 1; i >= 0; i--) { - long prev = timeValues.getLongSingleValueRow(i); - if (time != prev) { - break; - } - } - return inclusive ? i + 1 : i; - } - } - // key not found. - // return insert index - return inclusive ? low : low - 1; - } - - private Sequence> makeReturnResult( - int limit, TreeMap retVal) - { - Iterable source = Iterables.transform( - retVal.entrySet(), new Function, SearchHit>() - { - @Override - public SearchHit apply(Map.Entry input) - { - SearchHit hit = input.getKey(); - return new SearchHit(hit.getDimension(), hit.getValue(), input.getValue().intValue()); - } - } - ); - return Sequences.simple( - ImmutableList.of( - new Result( - segment.getDataInterval().getStart(), - new SearchResultValue( - Lists.newArrayList(new FunctionalIterable(source).limit(limit)) - ) - ) - ) - ); + return strategy.getExecutionPlan(query, segment).execute(); } } diff --git a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java new file mode 100644 index 000000000000..f4a6d8328b87 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java @@ -0,0 +1,60 @@ +package io.druid.query.search.search; + +import com.metamx.emitter.EmittingLogger; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.java.util.common.IAE; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.filter.Filter; +import io.druid.query.search.search.CursorBasedStrategy.CursorBasedExecutor; +import io.druid.query.search.search.IndexOnlyStrategy.IndexOnlyExecutor; +import io.druid.segment.QueryableIndex; +import io.druid.segment.Segment; +import io.druid.segment.filter.Filters; +import org.joda.time.Interval; + +import java.util.List; + +public class AutoStrategy extends SearchStrategy +{ + private static final EmittingLogger log = new EmittingLogger(AutoStrategy.class); + + @Override + public SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment) + { + final QueryableIndex index = segment.asQueryableIndex(); + + if (index != null) { + final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter())); + final List intervals = query.getQuerySegmentSpec().getIntervals(); + if (intervals.size() != 1) { + throw new IAE("Should only have one interval, got[%s]", intervals); + } + final Interval interval = intervals.get(0); + final ImmutableBitmap timeFilteredBitmap = IndexOnlyExecutor.makeTimeFilteredBitmap(index, + segment, + filter, + interval); + final Iterable dimsToSearch = SearchQueryExecutor.getDimsToSearch(index.getAvailableDimensions(), + query.getDimensions()); + + // Choose a search query execution strategy depending on the query. + // Index-only strategy is selected when + // 1) there is no filter, + // 2) the total cardinality is very low, or + // 3) the filter has a very high selectivity. + if (filter == null || + index.getDecisionHelper().hasLowCardinality(index, dimsToSearch) || + index.getDecisionHelper().hasHighSelectivity(index, timeFilteredBitmap)) { + log.info("Index-only execution strategy is selected"); + return new IndexOnlyExecutor(query, segment); + } else { + log.info("Cursor-based execution strategy is selected"); + return new CursorBasedExecutor(query, segment); + } + + } else { + log.info("Index doesn't exist. Fall back to cursor-based execution strategy"); + return new CursorBasedExecutor(query, segment); + } + } +} diff --git a/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java new file mode 100644 index 000000000000..be71480353a0 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java @@ -0,0 +1,17 @@ +package io.druid.query.search.search; + +public class ConciseBitmapDecisionHelper extends SearchQueryDecisionHelper +{ + private static final double HIGH_FILTER_SELECTIVITY_THRESHOLD = 0.99; + private static final int LOW_CARDINALITY_THRESHOLD = 5000; + private static final ConciseBitmapDecisionHelper instance = new ConciseBitmapDecisionHelper(); + + protected ConciseBitmapDecisionHelper() + { + super(HIGH_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); + } + + public static ConciseBitmapDecisionHelper getInstance() { + return instance; + } +} diff --git a/processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java b/processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java new file mode 100644 index 000000000000..e310c5afdea4 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java @@ -0,0 +1,101 @@ +package io.druid.query.search.search; + +import com.google.common.collect.Maps; +import io.druid.java.util.common.guava.Accumulator; +import io.druid.java.util.common.guava.Sequence; +import io.druid.query.Result; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.search.SearchResultValue; +import io.druid.segment.Cursor; +import io.druid.segment.DimensionSelector; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; +import io.druid.segment.VirtualColumns; +import io.druid.segment.data.IndexedInts; +import org.apache.commons.lang.mutable.MutableInt; + +import java.util.Map; +import java.util.TreeMap; + +public class CursorBasedStrategy extends SearchStrategy +{ + + @Override + public SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment) + { + return new CursorBasedExecutor(query, segment); + } + + public static class CursorBasedExecutor extends SearchQueryExecutor { + + public CursorBasedExecutor(SearchQuery query, Segment segment) + { + super(query, segment); + } + + @Override + public Sequence> execute() + { + final StorageAdapter adapter = segment.asStorageAdapter(); + + final Sequence cursors = adapter.makeCursors( + filter, + interval, + VirtualColumns.EMPTY, + query.getGranularity(), + query.isDescending() + ); + + final TreeMap retVal = cursors.accumulate( + Maps.newTreeMap(query.getSort().getComparator()), + new Accumulator, Cursor>() + { + @Override + public TreeMap accumulate(TreeMap set, Cursor cursor) + { + if (set.size() >= limit) { + return set; + } + + Map dimSelectors = Maps.newHashMap(); + for (DimensionSpec dim : dimsToSearch) { + dimSelectors.put( + dim.getOutputName(), + cursor.makeDimensionSelector(dim) + ); + } + + while (!cursor.isDone()) { + for (Map.Entry entry : dimSelectors.entrySet()) { + final DimensionSelector selector = entry.getValue(); + + if (selector != null) { + final IndexedInts vals = selector.getRow(); + for (int i = 0; i < vals.size(); ++i) { + final String dimVal = selector.lookupName(vals.get(i)); + if (searchQuerySpec.accept(dimVal)) { + MutableInt counter = new MutableInt(1); + MutableInt prev = set.put(new SearchHit(entry.getKey(), dimVal), counter); + if (prev != null) { + counter.add(prev.intValue()); + } + if (set.size() >= limit) { + return set; + } + } + } + } + } + + cursor.advance(); + } + + return set; + } + } + ); + + return makeReturnResult(segment, limit, retVal); + } + } +} diff --git a/processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java b/processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java new file mode 100644 index 000000000000..7fe4fcc745a1 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java @@ -0,0 +1,167 @@ +package io.druid.query.search.search; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Maps; +import com.metamx.emitter.EmittingLogger; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.java.util.common.guava.Sequence; +import io.druid.query.Result; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.extraction.ExtractionFn; +import io.druid.query.extraction.IdentityExtractionFn; +import io.druid.query.filter.Filter; +import io.druid.query.search.SearchResultValue; +import io.druid.query.search.search.CursorBasedStrategy.CursorBasedExecutor; +import io.druid.segment.ColumnSelectorBitmapIndexSelector; +import io.druid.segment.QueryableIndex; +import io.druid.segment.Segment; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.column.Column; +import io.druid.segment.column.GenericColumn; +import org.apache.commons.lang.mutable.MutableInt; +import org.joda.time.Interval; + +import java.util.Arrays; +import java.util.TreeMap; + +public class IndexOnlyStrategy extends SearchStrategy +{ + private static final EmittingLogger log = new EmittingLogger(IndexOnlyStrategy.class); + + @Override + public SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment) + { + final QueryableIndex index = segment.asQueryableIndex(); + + if (index == null) { + log.info("Index doesn't exist. Fall back to cursor-based execution strategy"); + return new CursorBasedExecutor(query, segment); + } else { + return new IndexOnlyExecutor(query, segment); + } + } + + public static class IndexOnlyExecutor extends SearchQueryExecutor { + + public IndexOnlyExecutor(SearchQuery query, Segment segment) + { + super(query, segment); + } + + @Override + public Sequence> execute() + { + final QueryableIndex index = segment.asQueryableIndex(); + Preconditions.checkArgument(index != null, "Index should not be null"); + + final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); + final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(index, segment, filter, interval); + + final TreeMap retVal = Maps.newTreeMap(query.getSort().getComparator()); + for (DimensionSpec dimension : dimsToSearch) { + final Column column = index.getColumn(dimension.getDimension()); + if (column == null) { + continue; + } + + final BitmapIndex bitmapIndex = column.getBitmapIndex(); + ExtractionFn extractionFn = dimension.getExtractionFn(); + if (extractionFn == null) { + extractionFn = IdentityExtractionFn.getInstance(); + } + if (bitmapIndex != null) { + for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { + String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); + if (!searchQuerySpec.accept(dimVal)) { + continue; + } + ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); + if (timeFilteredBitmap != null) { + bitmap = bitmapFactory.intersection(Arrays.asList(timeFilteredBitmap, bitmap)); + } + if (bitmap.size() > 0) { + MutableInt counter = new MutableInt(bitmap.size()); + MutableInt prev = retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter); + if (prev != null) { + counter.add(prev.intValue()); + } + if (retVal.size() >= limit) { + return makeReturnResult(segment, limit, retVal); + } + } + } + } + } + + return makeReturnResult(segment, limit, retVal); + } + + static ImmutableBitmap makeTimeFilteredBitmap(final QueryableIndex index, + final Segment segment, + final Filter filter, + final Interval interval) { + final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); + final ImmutableBitmap baseFilter = + filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); + + final ImmutableBitmap timeFilteredBitmap; + if (!interval.contains(segment.getDataInterval())) { + MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); + final Column timeColumn = index.getColumn(Column.TIME_COLUMN_NAME); + try (final GenericColumn timeValues = timeColumn.getGenericColumn()) { + + int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true)); + int endIndex = Math.min( + timeValues.length() - 1, + getStartIndexOfTime(timeValues, interval.getEndMillis(), false) + ); + + for (int i = startIndex; i <= endIndex; i++) { + timeBitmap.add(i); + } + + final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap); + timeFilteredBitmap = + (baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter); + } + } else { + timeFilteredBitmap = baseFilter; + } + + return timeFilteredBitmap; + } + + private static int getStartIndexOfTime(GenericColumn timeValues, long time, boolean inclusive) + { + int low = 0; + int high = timeValues.length() - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + long midVal = timeValues.getLongSingleValueRow(mid); + + if (midVal < time) { + low = mid + 1; + } else if (midVal > time) { + high = mid - 1; + } else { // key found + int i; + // rewind the index of the same time values + for (i = mid - 1; i >= 0; i--) { + long prev = timeValues.getLongSingleValueRow(i); + if (time != prev) { + break; + } + } + return inclusive ? i + 1 : i; + } + } + // key not found. + // return insert index + return inclusive ? low : low - 1; + } + } +} diff --git a/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java new file mode 100644 index 000000000000..8bc4e9e4e1e7 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java @@ -0,0 +1,17 @@ +package io.druid.query.search.search; + +public class RoaringBitmapDecisionHelper extends SearchQueryDecisionHelper +{ + private static final double HIGH_FILTER_SELECTIVITY_THRESHOLD = 0.65; + private static final int LOW_CARDINALITY_THRESHOLD = 1000; + private static final RoaringBitmapDecisionHelper instance = new RoaringBitmapDecisionHelper(); + + public static RoaringBitmapDecisionHelper getInstance() { + return instance; + } + + protected RoaringBitmapDecisionHelper() + { + super(HIGH_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); + } +} diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQuery.java b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java index 23fddc4bcb99..b22abbf66d89 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQuery.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java @@ -49,14 +49,7 @@ public class SearchQuery extends BaseQuery> private final List dimensions; private final SearchQuerySpec querySpec; private final int limit; - private final Strategy strategy; - - public enum Strategy - { - AUTO, - INDEX_ONLY, - CURSOR_BASED - } + private final SearchStrategy strategy; @JsonCreator public SearchQuery( @@ -69,7 +62,7 @@ public SearchQuery( @JsonProperty("query") SearchQuerySpec querySpec, @JsonProperty("sort") SearchSortSpec sortSpec, @JsonProperty("context") Map context, - @JsonProperty("strategy") Strategy strategy + @JsonProperty("strategy") SearchStrategy strategy ) { super(dataSource, querySegmentSpec, false, context); @@ -79,7 +72,7 @@ public SearchQuery( this.limit = (limit == 0) ? 1000 : limit; this.dimensions = dimensions; this.querySpec = querySpec == null ? new AllSearchQuerySpec() : querySpec; - this.strategy = strategy == null ? Strategy.AUTO : strategy; + this.strategy = strategy == null ? new AutoStrategy() : strategy; Preconditions.checkNotNull(querySegmentSpec, "Must specify an interval"); } @@ -206,7 +199,7 @@ public SearchSortSpec getSort() } @JsonProperty("strategy") - public Strategy getStrategy() { + public SearchStrategy getStrategy() { return strategy; } diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java new file mode 100644 index 000000000000..24c7ee1a25f2 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java @@ -0,0 +1,50 @@ +package io.druid.query.search.search; + +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.QueryableIndex; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.column.Column; + +public abstract class SearchQueryDecisionHelper +{ + protected final double highFilterSelectivityThreshold; + protected final int lowCardinalityThreshold; + + protected SearchQueryDecisionHelper(final double highFilterSelectivityThreshold, final int lowCardinalityThreshold) + { + this.highFilterSelectivityThreshold = highFilterSelectivityThreshold; + this.lowCardinalityThreshold = lowCardinalityThreshold; + } + + public double getHighFilterSelectivityThreshold() + { + return highFilterSelectivityThreshold; + } + + public int getLowCardinalityThreshold() + { + return lowCardinalityThreshold; + } + + public boolean hasLowCardinality(final QueryableIndex index, final Iterable dimensionSpecs) + { + long totalCard = 0; + for (DimensionSpec dimension : dimensionSpecs) { + final Column column = index.getColumn(dimension.getDimension()); + if (column != null) { + final BitmapIndex bitmapIndex = column.getBitmapIndex(); + if (bitmapIndex != null) { + totalCard += bitmapIndex.getCardinality(); + } + } + } + + return totalCard < lowCardinalityThreshold; + } + + public boolean hasHighSelectivity(final QueryableIndex index, final ImmutableBitmap bitmap) + { + return index.getNumRows() * highFilterSelectivityThreshold < bitmap.size(); + } +} diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java new file mode 100644 index 000000000000..ad9bd895dbf9 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java @@ -0,0 +1,89 @@ +package io.druid.query.search.search; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.guava.FunctionalIterable; +import io.druid.java.util.common.guava.Sequence; +import io.druid.java.util.common.guava.Sequences; +import io.druid.query.Druids; +import io.druid.query.Result; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.filter.Filter; +import io.druid.query.search.SearchResultValue; +import io.druid.segment.Segment; +import io.druid.segment.data.Indexed; +import io.druid.segment.filter.Filters; +import org.apache.commons.lang.mutable.MutableInt; +import org.joda.time.Interval; + +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +public abstract class SearchQueryExecutor +{ + protected final SearchQuery query; + protected final SearchQuerySpec searchQuerySpec; + protected final Segment segment; + protected final Filter filter; + protected final Interval interval; + protected final Iterable dimsToSearch; + protected final int limit; + + public SearchQueryExecutor(SearchQuery query, Segment segment) { + this.query = query; + this.segment = segment; + + this.filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter())); + final List intervals = query.getQuerySegmentSpec().getIntervals(); + if (intervals.size() != 1) { + throw new IAE("Should only have one interval, got[%s]", intervals); + } + this.interval = intervals.get(0); + + this.searchQuerySpec = query.getQuery(); + this.dimsToSearch = getDimsToSearch(segment.asStorageAdapter().getAvailableDimensions(), query.getDimensions()); + this.limit = query.getLimit(); + } + + public abstract Sequence> execute(); + + static Iterable getDimsToSearch(Indexed availableDimensions, List dimensions) + { + if (dimensions == null || dimensions.isEmpty()) { + return Iterables.transform(availableDimensions, Druids.DIMENSION_IDENTITY); + } else { + return dimensions; + } + } + + static Sequence> makeReturnResult( + Segment segment, int limit, TreeMap retVal) + { + Iterable source = Iterables.transform( + retVal.entrySet(), new Function, SearchHit>() + { + @Override + public SearchHit apply(Map.Entry input) + { + SearchHit hit = input.getKey(); + return new SearchHit(hit.getDimension(), hit.getValue(), input.getValue().intValue()); + } + } + ); + return Sequences.simple( + ImmutableList.of( + new Result<>( + segment.getDataInterval().getStart(), + new SearchResultValue( + Lists.newArrayList(new FunctionalIterable<>(source).limit(limit)) + ) + ) + ) + ); + } +} diff --git a/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java b/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java new file mode 100644 index 000000000000..960e2d47d2d0 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java @@ -0,0 +1,54 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.search.search; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.druid.segment.Segment; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "auto", value = AutoStrategy.class), + @JsonSubTypes.Type(name = "indexOnly", value = IndexOnlyStrategy.class), + @JsonSubTypes.Type(name = "cursorBased", value = CursorBasedStrategy.class) +}) +public abstract class SearchStrategy +{ + public abstract SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment); + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 80c9b69dd930..0656bebef1f2 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -36,12 +36,12 @@ import com.google.common.io.Files; import com.google.common.primitives.Ints; import com.google.inject.Inject; +import com.metamx.emitter.EmittingLogger; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.MutableBitmap; import io.druid.collections.spatial.ImmutableRTree; -import com.metamx.emitter.EmittingLogger; import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; @@ -50,6 +50,7 @@ import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; import io.druid.java.util.common.logger.Logger; +import io.druid.query.search.search.ConciseBitmapDecisionHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnCapabilities; @@ -949,6 +950,7 @@ Column.TIME_COLUMN_NAME, new ColumnBuilder() new ArrayIndexed<>(cols, String.class), index.getAvailableDimensions(), new ConciseBitmapFactory(), + ConciseBitmapDecisionHelper.getInstance(), columns, index.getFileMapper(), null @@ -1027,7 +1029,14 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException columns.put(Column.TIME_COLUMN_NAME, deserializeColumn(mapper, smooshedFiles.mapFile("__time"))); final QueryableIndex index = new SimpleQueryableIndex( - dataInterval, cols, dims, segmentBitmapSerdeFactory.getBitmapFactory(), columns, smooshedFiles, metadata + dataInterval, + cols, + dims, + segmentBitmapSerdeFactory.getBitmapFactory(), + segmentBitmapSerdeFactory.getDecisionHelper(), + columns, + smooshedFiles, + metadata ); log.debug("Mapped v9 index[%s] in %,d millis", inDir, System.currentTimeMillis() - startTime); diff --git a/processing/src/main/java/io/druid/segment/QueryableIndex.java b/processing/src/main/java/io/druid/segment/QueryableIndex.java index 31492b18cc43..4ab3a78020e8 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndex.java @@ -20,6 +20,7 @@ package io.druid.segment; import io.druid.collections.bitmap.BitmapFactory; +import io.druid.query.search.search.SearchQueryDecisionHelper; import io.druid.segment.data.Indexed; import org.joda.time.Interval; @@ -31,17 +32,19 @@ */ public interface QueryableIndex extends ColumnSelector, Closeable { - public Interval getDataInterval(); - public int getNumRows(); - public Indexed getAvailableDimensions(); - public BitmapFactory getBitmapFactoryForDimensions(); - public Metadata getMetadata(); - public Map getDimensionHandlers(); + Interval getDataInterval(); + int getNumRows(); + Indexed getAvailableDimensions(); + BitmapFactory getBitmapFactoryForDimensions(); + Metadata getMetadata(); + Map getDimensionHandlers(); + + SearchQueryDecisionHelper getDecisionHelper(); /** * The close method shouldn't actually be here as this is nasty. We will adjust it in the future. * @throws java.io.IOException if an exception was thrown closing the index */ //@Deprecated // This is still required for SimpleQueryableIndex. It should not go away unitl SimpleQueryableIndex is fixed - public void close() throws IOException; + void close() throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java index f91fa7aa0291..ab9af66942a0 100644 --- a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java @@ -23,6 +23,7 @@ import com.google.common.collect.Maps; import io.druid.collections.bitmap.BitmapFactory; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.query.search.search.SearchQueryDecisionHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.data.Indexed; @@ -43,12 +44,14 @@ public class SimpleQueryableIndex implements QueryableIndex private final SmooshedFileMapper fileMapper; private final Metadata metadata; private final Map dimensionHandlers; + private final SearchQueryDecisionHelper decisionHelper; public SimpleQueryableIndex( Interval dataInterval, Indexed columnNames, Indexed dimNames, BitmapFactory bitmapFactory, + SearchQueryDecisionHelper decisionHelper, Map columns, SmooshedFileMapper fileMapper, Metadata metadata @@ -59,6 +62,7 @@ public SimpleQueryableIndex( this.columnNames = columnNames; this.availableDimensions = dimNames; this.bitmapFactory = bitmapFactory; + this.decisionHelper = decisionHelper; this.columns = columns; this.fileMapper = fileMapper; this.metadata = metadata; @@ -120,6 +124,12 @@ public Map getDimensionHandlers() return dimensionHandlers; } + @Override + public SearchQueryDecisionHelper getDecisionHelper() + { + return decisionHelper; + } + private void initDimensionHandlers() { for (String dim : availableDimensions) { diff --git a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java index a7c6fe6bcfb2..22c32b5a2c5f 100644 --- a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.query.search.search.SearchQueryDecisionHelper; /** */ @@ -33,7 +34,9 @@ }) public interface BitmapSerdeFactory { - public ObjectStrategy getObjectStrategy(); + ObjectStrategy getObjectStrategy(); - public BitmapFactory getBitmapFactory(); + BitmapFactory getBitmapFactory(); + + SearchQueryDecisionHelper getDecisionHelper(); } diff --git a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java index c5ef97f594d2..5fe7c086aa1a 100644 --- a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java @@ -27,6 +27,8 @@ import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.WrappedImmutableConciseBitmap; +import io.druid.query.search.search.ConciseBitmapDecisionHelper; +import io.druid.query.search.search.SearchQueryDecisionHelper; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; /** @@ -35,6 +37,7 @@ public class ConciseBitmapSerdeFactory implements BitmapSerdeFactory { private static final ObjectStrategy objectStrategy = new ImmutableConciseSetObjectStrategy(); private static final BitmapFactory bitmapFactory = new ConciseBitmapFactory(); + private static final SearchQueryDecisionHelper decisionHelper = ConciseBitmapDecisionHelper.getInstance(); @Override public ObjectStrategy getObjectStrategy() @@ -48,6 +51,12 @@ public BitmapFactory getBitmapFactory() return bitmapFactory; } + @Override + public SearchQueryDecisionHelper getDecisionHelper() + { + return decisionHelper; + } + private static Ordering conciseComparator = new Ordering() { @Override diff --git a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java index 9c2ae1d21a85..184c6bd5c0a5 100644 --- a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java @@ -26,6 +26,8 @@ import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.RoaringBitmapFactory; import io.druid.collections.bitmap.WrappedImmutableRoaringBitmap; +import io.druid.query.search.search.SearchQueryDecisionHelper; +import io.druid.query.search.search.RoaringBitmapDecisionHelper; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; import java.nio.ByteBuffer; @@ -36,6 +38,7 @@ public class RoaringBitmapSerdeFactory implements BitmapSerdeFactory { private static final boolean DEFAULT_COMPRESS_RUN_ON_SERIALIZATION = true; private static final ObjectStrategy objectStrategy = new ImmutableRoaringBitmapObjectStrategy(); + private static final SearchQueryDecisionHelper decisionHelper = RoaringBitmapDecisionHelper.getInstance(); private final boolean compressRunOnSerialization; private final BitmapFactory bitmapFactory; @@ -69,6 +72,12 @@ public BitmapFactory getBitmapFactory() return bitmapFactory; } + @Override + public SearchQueryDecisionHelper getDecisionHelper() + { + return decisionHelper; + } + private static Ordering roaringComparator = new Ordering() { @Override From 6a63f2b3ff8964144c9398c45e6f4a070f0218fe Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 27 Dec 2016 19:26:50 +0900 Subject: [PATCH 3/9] Address comments --- .../benchmark/query/SearchBenchmark.java | 18 ++- .../src/main/java/io/druid/query/Druids.java | 17 +- .../druid/query/search/SearchQueryRunner.java | 9 +- .../search/SearchQueryRunnerFactory.java | 5 +- .../query/search/SearchStrategySelector.java | 38 +++++ .../query/search/search/AutoStrategy.java | 40 +++-- .../search/ConciseBitmapDecisionHelper.java | 11 +- .../search/search/CursorBasedStrategy.java | 22 ++- .../search/search/IndexOnlyStrategy.java | 152 ++++++++++-------- .../search/RoaringBitmapDecisionHelper.java | 11 +- .../query/search/search/SearchQuery.java | 35 +--- .../search/search/SearchQueryConfig.java | 22 ++- .../search/SearchQueryDecisionHelper.java | 14 +- .../search/search/SearchQueryExecutor.java | 16 +- .../query/search/search/SearchStrategy.java | 52 +++--- .../main/java/io/druid/segment/IndexIO.java | 3 - .../java/io/druid/segment/QueryableIndex.java | 3 - .../druid/segment/SimpleQueryableIndex.java | 10 -- .../segment/data/BitmapSerdeFactory.java | 3 - .../data/ConciseBitmapSerdeFactory.java | 14 +- .../data/RoaringBitmapSerdeFactory.java | 9 -- .../java/io/druid/query/TestQueryRunners.java | 15 +- .../search/SearchQueryQueryToolChestTest.java | 1 - .../query/search/SearchQueryRunnerTest.java | 13 +- .../search/SearchQueryRunnerWithCaseTest.java | 70 ++++++-- 25 files changed, 339 insertions(+), 264 deletions(-) create mode 100644 processing/src/main/java/io/druid/query/search/SearchStrategySelector.java diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java index 71794ff77d02..1866cad41e15 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java @@ -20,6 +20,7 @@ package io.druid.benchmark.query; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.hash.Hashing; @@ -50,6 +51,7 @@ import io.druid.query.search.SearchQueryQueryToolChest; import io.druid.query.search.SearchQueryRunnerFactory; import io.druid.query.search.SearchResultValue; +import io.druid.query.search.SearchStrategySelector; import io.druid.query.search.search.SearchHit; import io.druid.query.search.search.SearchQuery; import io.druid.query.search.search.SearchQueryConfig; @@ -163,14 +165,14 @@ private void setupQueries() QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(Arrays.asList(basicSchema.getDataInterval())); List dimUniformFilterVals = Lists.newArrayList(); - int resultNum = (int) (100000 * 0.5); + int resultNum = (int) (100000 * 0.1); int step = 100000 / resultNum; for (int i = 1; i < 100001 && dimUniformFilterVals.size() < resultNum; i += step) { dimUniformFilterVals.add(String.valueOf(i)); } List dimHyperUniqueFilterVals = Lists.newArrayList(); - resultNum = (int) (100000 * 1.0); + resultNum = (int) (100000 * 0.1); step = 100000 / resultNum; for (int i = 0; i < 100001 && dimHyperUniqueFilterVals.size() < resultNum; i += step) { dimHyperUniqueFilterVals.add(String.valueOf(i)); @@ -254,10 +256,16 @@ public void setup() throws IOException qIndexes.add(qIndex); } - final SearchQueryConfig config = new SearchQueryConfig(); - config.setMaxSearchLimit(limit); - + final SearchQueryConfig config = new SearchQueryConfig().withOverrides(query); factory = new SearchQueryRunnerFactory( + new SearchStrategySelector(new Supplier() + { + @Override + public SearchQueryConfig get() + { + return config; + } + }), new SearchQueryQueryToolChest( config, QueryBenchmarkUtil.NoopIntervalChunkingQueryRunnerDecorator() diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index d5b1ed0470aa..dfa7d238fb50 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -47,7 +47,6 @@ import io.druid.query.search.search.SearchQuery; import io.druid.query.search.search.SearchQuerySpec; import io.druid.query.search.search.SearchSortSpec; -import io.druid.query.search.search.SearchStrategy; import io.druid.query.select.PagingSpec; import io.druid.query.select.SelectQuery; import io.druid.query.spec.LegacySegmentSpec; @@ -550,7 +549,6 @@ public static class SearchQueryBuilder private SearchQuerySpec querySpec; private SearchSortSpec sortSpec; private Map context; - private SearchStrategy strategy; public SearchQueryBuilder() { @@ -562,7 +560,6 @@ public SearchQueryBuilder() dimensions = null; querySpec = null; context = null; - strategy = null; } public SearchQuery build() @@ -576,8 +573,7 @@ public SearchQuery build() dimensions, querySpec, sortSpec, - context, - strategy + context ); } @@ -591,8 +587,7 @@ public SearchQueryBuilder copy(SearchQuery query) .limit(query.getLimit()) .dimensions(query.getDimensions()) .query(query.getQuery()) - .context(query.getContext()) - .strategy(query.getStrategy()); + .context(query.getContext()); } public SearchQueryBuilder copy(SearchQueryBuilder builder) @@ -605,8 +600,7 @@ public SearchQueryBuilder copy(SearchQueryBuilder builder) .limit(builder.limit) .dimensions(builder.dimensions) .query(builder.querySpec) - .context(builder.context) - .strategy(builder.strategy); + .context(builder.context); } public SearchQueryBuilder dataSource(String d) @@ -756,11 +750,6 @@ public SearchQueryBuilder context(Map c) context = c; return this; } - - public SearchQueryBuilder strategy(SearchStrategy strategy) { - this.strategy = strategy; - return this; - } } public static SearchQueryBuilder newSearchQueryBuilder() diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 7dd385d75b7a..4cead1e85b0e 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -25,7 +25,6 @@ import io.druid.query.QueryRunner; import io.druid.query.Result; import io.druid.query.search.search.SearchQuery; -import io.druid.query.search.search.SearchStrategy; import io.druid.segment.Segment; import java.util.Map; @@ -35,10 +34,12 @@ public class SearchQueryRunner implements QueryRunner> { private final Segment segment; + private final SearchStrategySelector strategySelector; - public SearchQueryRunner(Segment segment) + public SearchQueryRunner(Segment segment, SearchStrategySelector strategySelector) { this.segment = segment; + this.strategySelector = strategySelector; } @Override @@ -52,8 +53,6 @@ public Sequence> run( } final SearchQuery query = (SearchQuery) input; - final SearchStrategy strategy = query.getStrategy(); - - return strategy.getExecutionPlan(query, segment).execute(); + return strategySelector.strategize(query).getExecutionPlan(query, segment).execute(); } } diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java index 1fde10cc7b85..46093cf0739c 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java @@ -35,15 +35,18 @@ */ public class SearchQueryRunnerFactory implements QueryRunnerFactory, SearchQuery> { + private final SearchStrategySelector strategySelector; private final SearchQueryQueryToolChest toolChest; private final QueryWatcher queryWatcher; @Inject public SearchQueryRunnerFactory( + SearchStrategySelector strategySelector, SearchQueryQueryToolChest toolChest, QueryWatcher queryWatcher ) { + this.strategySelector = strategySelector; this.toolChest = toolChest; this.queryWatcher = queryWatcher; } @@ -51,7 +54,7 @@ public SearchQueryRunnerFactory( @Override public QueryRunner> createRunner(final Segment segment) { - return new SearchQueryRunner(segment); + return new SearchQueryRunner(segment, strategySelector); } @Override diff --git a/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java b/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java new file mode 100644 index 000000000000..e451b74b7384 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java @@ -0,0 +1,38 @@ +package io.druid.query.search; + +import com.google.common.base.Supplier; +import com.google.inject.Inject; +import io.druid.java.util.common.ISE; +import io.druid.query.search.search.AutoStrategy; +import io.druid.query.search.search.CursorBasedStrategy; +import io.druid.query.search.search.IndexOnlyStrategy; +import io.druid.query.search.search.SearchQuery; +import io.druid.query.search.search.SearchQueryConfig; +import io.druid.query.search.search.SearchStrategy; + +public class SearchStrategySelector +{ + private final SearchQueryConfig config; + + @Inject + public SearchStrategySelector(Supplier configSupplier) + { + this.config = configSupplier.get(); + } + + public SearchStrategy strategize(SearchQuery query) + { + final String strategyString = config.withOverrides(query).getSearchStrategy(); + + switch (strategyString) { + case AutoStrategy.NAME: + return new AutoStrategy(query); + case IndexOnlyStrategy.NAME: + return new IndexOnlyStrategy(query); + case CursorBasedStrategy.NAME: + return new CursorBasedStrategy(query); + default: + throw new ISE("Unknown strategy[%s]", strategyString); + } + } +} diff --git a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java index f4a6d8328b87..4938d2c06030 100644 --- a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java @@ -2,35 +2,30 @@ import com.metamx.emitter.EmittingLogger; import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.java.util.common.IAE; import io.druid.query.dimension.DimensionSpec; -import io.druid.query.filter.Filter; import io.druid.query.search.search.CursorBasedStrategy.CursorBasedExecutor; import io.druid.query.search.search.IndexOnlyStrategy.IndexOnlyExecutor; import io.druid.segment.QueryableIndex; import io.druid.segment.Segment; -import io.druid.segment.filter.Filters; -import org.joda.time.Interval; - -import java.util.List; public class AutoStrategy extends SearchStrategy { + public static final String NAME = "auto"; + private static final EmittingLogger log = new EmittingLogger(AutoStrategy.class); + public AutoStrategy(SearchQuery query) + { + super(query); + } + @Override public SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment) { final QueryableIndex index = segment.asQueryableIndex(); if (index != null) { - final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter())); - final List intervals = query.getQuerySegmentSpec().getIntervals(); - if (intervals.size() != 1) { - throw new IAE("Should only have one interval, got[%s]", intervals); - } - final Interval interval = intervals.get(0); - final ImmutableBitmap timeFilteredBitmap = IndexOnlyExecutor.makeTimeFilteredBitmap(index, + final ImmutableBitmap timeFilteredBitmap = IndexOnlyStrategy.makeTimeFilteredBitmap(index, segment, filter, interval); @@ -41,20 +36,21 @@ public SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment) // Index-only strategy is selected when // 1) there is no filter, // 2) the total cardinality is very low, or - // 3) the filter has a very high selectivity. + // 3) the filter has a very low selectivity. + final SearchQueryDecisionHelper helper = getDecisionHelper(index); if (filter == null || - index.getDecisionHelper().hasLowCardinality(index, dimsToSearch) || - index.getDecisionHelper().hasHighSelectivity(index, timeFilteredBitmap)) { - log.info("Index-only execution strategy is selected"); - return new IndexOnlyExecutor(query, segment); + helper.hasLowCardinality(index, dimsToSearch) || + helper.hasLowSelectivity(index, timeFilteredBitmap)) { + log.debug("Index-only execution strategy is selected"); + return new IndexOnlyExecutor(query, segment, timeFilteredBitmap); } else { - log.info("Cursor-based execution strategy is selected"); - return new CursorBasedExecutor(query, segment); + log.debug("Cursor-based execution strategy is selected"); + return new CursorBasedExecutor(query, segment, filter, interval); } } else { - log.info("Index doesn't exist. Fall back to cursor-based execution strategy"); - return new CursorBasedExecutor(query, segment); + log.debug("Index doesn't exist. Fall back to cursor-based execution strategy"); + return new CursorBasedExecutor(query, segment, filter, interval); } } } diff --git a/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java index be71480353a0..292747ed460e 100644 --- a/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java +++ b/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java @@ -2,16 +2,11 @@ public class ConciseBitmapDecisionHelper extends SearchQueryDecisionHelper { - private static final double HIGH_FILTER_SELECTIVITY_THRESHOLD = 0.99; + private static final double LOW_FILTER_SELECTIVITY_THRESHOLD = 0.99; private static final int LOW_CARDINALITY_THRESHOLD = 5000; - private static final ConciseBitmapDecisionHelper instance = new ConciseBitmapDecisionHelper(); - protected ConciseBitmapDecisionHelper() + public ConciseBitmapDecisionHelper() { - super(HIGH_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); - } - - public static ConciseBitmapDecisionHelper getInstance() { - return instance; + super(LOW_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); } } diff --git a/processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java b/processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java index e310c5afdea4..209df3bcf08a 100644 --- a/processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java @@ -1,10 +1,12 @@ package io.druid.query.search.search; import com.google.common.collect.Maps; +import com.metamx.emitter.EmittingLogger; import io.druid.java.util.common.guava.Accumulator; import io.druid.java.util.common.guava.Sequence; import io.druid.query.Result; import io.druid.query.dimension.DimensionSpec; +import io.druid.query.filter.Filter; import io.druid.query.search.SearchResultValue; import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; @@ -13,24 +15,40 @@ import io.druid.segment.VirtualColumns; import io.druid.segment.data.IndexedInts; import org.apache.commons.lang.mutable.MutableInt; +import org.joda.time.Interval; import java.util.Map; import java.util.TreeMap; public class CursorBasedStrategy extends SearchStrategy { + public static final String NAME = "cursorBased"; + + private static final EmittingLogger log = new EmittingLogger(CursorBasedStrategy.class); + + public CursorBasedStrategy(SearchQuery query) + { + super(query); + } @Override public SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment) { - return new CursorBasedExecutor(query, segment); + log.debug("Cursor-based execution strategy is selected"); + return new CursorBasedExecutor(query, segment, filter, interval); } public static class CursorBasedExecutor extends SearchQueryExecutor { - public CursorBasedExecutor(SearchQuery query, Segment segment) + protected Filter filter; + protected Interval interval; + + public CursorBasedExecutor(SearchQuery query, Segment segment, Filter filter, Interval interval) { super(query, segment); + + this.filter = filter; + this.interval = interval; } @Override diff --git a/processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java b/processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java index 7fe4fcc745a1..ab14bcf0a3d2 100644 --- a/processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java @@ -29,26 +29,104 @@ public class IndexOnlyStrategy extends SearchStrategy { + public static final String NAME = "indexOnly"; + private static final EmittingLogger log = new EmittingLogger(IndexOnlyStrategy.class); + public IndexOnlyStrategy(SearchQuery query) + { + super(query); + } + @Override public SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment) { final QueryableIndex index = segment.asQueryableIndex(); if (index == null) { - log.info("Index doesn't exist. Fall back to cursor-based execution strategy"); - return new CursorBasedExecutor(query, segment); + log.debug("Index doesn't exist. Fall back to cursor-based execution strategy"); + return new CursorBasedExecutor(query, segment, filter, interval); } else { - return new IndexOnlyExecutor(query, segment); + log.debug("Index-only execution strategy is selected"); + final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(index, segment, filter, interval); + return new IndexOnlyExecutor(query, segment, timeFilteredBitmap); + } + } + + static ImmutableBitmap makeTimeFilteredBitmap(final QueryableIndex index, + final Segment segment, + final Filter filter, + final Interval interval) + { + final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); + final ImmutableBitmap baseFilter = + filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); + + final ImmutableBitmap timeFilteredBitmap; + if (!interval.contains(segment.getDataInterval())) { + MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); + final Column timeColumn = index.getColumn(Column.TIME_COLUMN_NAME); + try (final GenericColumn timeValues = timeColumn.getGenericColumn()) { + + int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true)); + int endIndex = Math.min( + timeValues.length() - 1, + getStartIndexOfTime(timeValues, interval.getEndMillis(), false) + ); + + for (int i = startIndex; i <= endIndex; i++) { + timeBitmap.add(i); + } + + final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap); + timeFilteredBitmap = + (baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter); + } + } else { + timeFilteredBitmap = baseFilter; + } + + return timeFilteredBitmap; + } + + private static int getStartIndexOfTime(GenericColumn timeValues, long time, boolean inclusive) + { + int low = 0; + int high = timeValues.length() - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + long midVal = timeValues.getLongSingleValueRow(mid); + + if (midVal < time) { + low = mid + 1; + } else if (midVal > time) { + high = mid - 1; + } else { // key found + int i; + // rewind the index of the same time values + for (i = mid - 1; i >= 0; i--) { + long prev = timeValues.getLongSingleValueRow(i); + if (time != prev) { + break; + } + } + return inclusive ? i + 1 : i; + } } + // key not found. + // return insert index + return inclusive ? low : low - 1; } public static class IndexOnlyExecutor extends SearchQueryExecutor { - public IndexOnlyExecutor(SearchQuery query, Segment segment) + private final ImmutableBitmap timeFilteredBitmap; + + public IndexOnlyExecutor(SearchQuery query, Segment segment, ImmutableBitmap timeFilteredBitmap) { super(query, segment); + this.timeFilteredBitmap = timeFilteredBitmap; } @Override @@ -58,7 +136,6 @@ public Sequence> execute() Preconditions.checkArgument(index != null, "Index should not be null"); final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); - final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(index, segment, filter, interval); final TreeMap retVal = Maps.newTreeMap(query.getSort().getComparator()); for (DimensionSpec dimension : dimsToSearch) { @@ -98,70 +175,5 @@ public Sequence> execute() return makeReturnResult(segment, limit, retVal); } - - static ImmutableBitmap makeTimeFilteredBitmap(final QueryableIndex index, - final Segment segment, - final Filter filter, - final Interval interval) { - final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); - final ImmutableBitmap baseFilter = - filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); - - final ImmutableBitmap timeFilteredBitmap; - if (!interval.contains(segment.getDataInterval())) { - MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); - final Column timeColumn = index.getColumn(Column.TIME_COLUMN_NAME); - try (final GenericColumn timeValues = timeColumn.getGenericColumn()) { - - int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true)); - int endIndex = Math.min( - timeValues.length() - 1, - getStartIndexOfTime(timeValues, interval.getEndMillis(), false) - ); - - for (int i = startIndex; i <= endIndex; i++) { - timeBitmap.add(i); - } - - final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap); - timeFilteredBitmap = - (baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter); - } - } else { - timeFilteredBitmap = baseFilter; - } - - return timeFilteredBitmap; - } - - private static int getStartIndexOfTime(GenericColumn timeValues, long time, boolean inclusive) - { - int low = 0; - int high = timeValues.length() - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - long midVal = timeValues.getLongSingleValueRow(mid); - - if (midVal < time) { - low = mid + 1; - } else if (midVal > time) { - high = mid - 1; - } else { // key found - int i; - // rewind the index of the same time values - for (i = mid - 1; i >= 0; i--) { - long prev = timeValues.getLongSingleValueRow(i); - if (time != prev) { - break; - } - } - return inclusive ? i + 1 : i; - } - } - // key not found. - // return insert index - return inclusive ? low : low - 1; - } } } diff --git a/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java index 8bc4e9e4e1e7..4196d2edf853 100644 --- a/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java +++ b/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java @@ -2,16 +2,11 @@ public class RoaringBitmapDecisionHelper extends SearchQueryDecisionHelper { - private static final double HIGH_FILTER_SELECTIVITY_THRESHOLD = 0.65; + private static final double LOW_FILTER_SELECTIVITY_THRESHOLD = 0.65; private static final int LOW_CARDINALITY_THRESHOLD = 1000; - private static final RoaringBitmapDecisionHelper instance = new RoaringBitmapDecisionHelper(); - public static RoaringBitmapDecisionHelper getInstance() { - return instance; - } - - protected RoaringBitmapDecisionHelper() + public RoaringBitmapDecisionHelper() { - super(HIGH_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); + super(LOW_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); } } diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQuery.java b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java index b22abbf66d89..0e0757f96791 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQuery.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java @@ -49,7 +49,6 @@ public class SearchQuery extends BaseQuery> private final List dimensions; private final SearchQuerySpec querySpec; private final int limit; - private final SearchStrategy strategy; @JsonCreator public SearchQuery( @@ -61,20 +60,18 @@ public SearchQuery( @JsonProperty("searchDimensions") List dimensions, @JsonProperty("query") SearchQuerySpec querySpec, @JsonProperty("sort") SearchSortSpec sortSpec, - @JsonProperty("context") Map context, - @JsonProperty("strategy") SearchStrategy strategy + @JsonProperty("context") Map context ) { super(dataSource, querySegmentSpec, false, context); + Preconditions.checkNotNull(querySegmentSpec, "Must specify an interval"); + this.dimFilter = dimFilter; this.sortSpec = sortSpec == null ? DEFAULT_SORT_SPEC : sortSpec; this.granularity = granularity == null ? QueryGranularities.ALL : granularity; this.limit = (limit == 0) ? 1000 : limit; this.dimensions = dimensions; this.querySpec = querySpec == null ? new AllSearchQuerySpec() : querySpec; - this.strategy = strategy == null ? new AutoStrategy() : strategy; - - Preconditions.checkNotNull(querySegmentSpec, "Must specify an interval"); } @Override @@ -107,8 +104,7 @@ public SearchQuery withQuerySegmentSpec(QuerySegmentSpec spec) dimensions, querySpec, sortSpec, - getContext(), - strategy + getContext() ); } @@ -124,8 +120,7 @@ public Query> withDataSource(DataSource dataSource) dimensions, querySpec, sortSpec, - getContext(), - strategy + getContext() ); } @@ -141,8 +136,7 @@ public SearchQuery withOverriddenContext(Map contextOverrides) dimensions, querySpec, sortSpec, - computeOverridenContext(contextOverrides), - strategy + computeOverridenContext(contextOverrides) ); } @@ -157,8 +151,7 @@ public SearchQuery withDimFilter(DimFilter dimFilter) dimensions, querySpec, sortSpec, - getContext(), - strategy + getContext() ); } @@ -198,11 +191,6 @@ public SearchSortSpec getSort() return sortSpec; } - @JsonProperty("strategy") - public SearchStrategy getStrategy() { - return strategy; - } - public SearchQuery withLimit(int newLimit) { return new SearchQuery( @@ -214,8 +202,7 @@ public SearchQuery withLimit(int newLimit) dimensions, querySpec, sortSpec, - getContext(), - strategy + getContext() ); } @@ -230,7 +217,6 @@ public String toString() ", querySpec=" + querySpec + ", querySegmentSpec=" + getQuerySegmentSpec() + ", limit=" + limit + - ", strategy=" + strategy + '}'; } @@ -268,10 +254,6 @@ public boolean equals(Object o) return false; } - if (!strategy.equals(that.strategy)) { - return false; - } - return true; } @@ -285,7 +267,6 @@ public int hashCode() result = 31 * result + (dimensions != null ? dimensions.hashCode() : 0); result = 31 * result + (querySpec != null ? querySpec.hashCode() : 0); result = 31 * result + limit; - result = 31 * result + strategy.hashCode(); return result; } } diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java index 7d2ed451bb4e..88cd52c765a9 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java @@ -27,16 +27,32 @@ */ public class SearchQueryConfig { + public static final String CTX_KEY_STRATEGY = "searchStrategy"; + @JsonProperty @Min(1) private int maxSearchLimit = 1000; - public void setMaxSearchLimit(int maxSearchLimit) { - this.maxSearchLimit = maxSearchLimit; - } + @JsonProperty + private String searchStrategy = AutoStrategy.NAME; public int getMaxSearchLimit() { return maxSearchLimit; } + + public String getSearchStrategy() { + return searchStrategy; + } + + public void setSearchStrategy(final String strategy) { + this.searchStrategy = strategy; + } + + public SearchQueryConfig withOverrides(final SearchQuery query) { + final SearchQueryConfig newConfig = new SearchQueryConfig(); + newConfig.maxSearchLimit = query.getLimit(); + newConfig.searchStrategy = query.getContextValue(CTX_KEY_STRATEGY, searchStrategy); + return newConfig; + } } diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java index 24c7ee1a25f2..4ec8efd0983d 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java @@ -8,18 +8,18 @@ public abstract class SearchQueryDecisionHelper { - protected final double highFilterSelectivityThreshold; + protected final double lowFilterSelectivityThreshold; protected final int lowCardinalityThreshold; - protected SearchQueryDecisionHelper(final double highFilterSelectivityThreshold, final int lowCardinalityThreshold) + protected SearchQueryDecisionHelper(final double lowFilterSelectivityThreshold, final int lowCardinalityThreshold) { - this.highFilterSelectivityThreshold = highFilterSelectivityThreshold; + this.lowFilterSelectivityThreshold = lowFilterSelectivityThreshold; this.lowCardinalityThreshold = lowCardinalityThreshold; } - public double getHighFilterSelectivityThreshold() + public double getLowFilterSelectivityThreshold() { - return highFilterSelectivityThreshold; + return lowFilterSelectivityThreshold; } public int getLowCardinalityThreshold() @@ -43,8 +43,8 @@ public boolean hasLowCardinality(final QueryableIndex index, final Iterable dimsToSearch; protected final int limit; public SearchQueryExecutor(SearchQuery query, Segment segment) { this.query = query; this.segment = segment; - - this.filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter())); - final List intervals = query.getQuerySegmentSpec().getIntervals(); - if (intervals.size() != 1) { - throw new IAE("Should only have one interval, got[%s]", intervals); - } - this.interval = intervals.get(0); - this.searchQuerySpec = query.getQuery(); this.dimsToSearch = getDimsToSearch(segment.asStorageAdapter().getAvailableDimensions(), query.getDimensions()); this.limit = query.getLimit(); @@ -55,7 +41,7 @@ public SearchQueryExecutor(SearchQuery query, Segment segment) { static Iterable getDimsToSearch(Indexed availableDimensions, List dimensions) { if (dimensions == null || dimensions.isEmpty()) { - return Iterables.transform(availableDimensions, Druids.DIMENSION_IDENTITY); + return ImmutableList.copyOf(Iterables.transform(availableDimensions, Druids.DIMENSION_IDENTITY)); } else { return dimensions; } diff --git a/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java b/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java index 960e2d47d2d0..65b29a2b9caa 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java @@ -19,36 +19,44 @@ package io.druid.query.search.search; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.java.util.common.IAE; +import io.druid.query.filter.Filter; +import io.druid.segment.QueryableIndex; import io.druid.segment.Segment; +import io.druid.segment.filter.Filters; +import org.joda.time.Interval; + +import java.util.List; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -@JsonSubTypes(value = { - @JsonSubTypes.Type(name = "auto", value = AutoStrategy.class), - @JsonSubTypes.Type(name = "indexOnly", value = IndexOnlyStrategy.class), - @JsonSubTypes.Type(name = "cursorBased", value = CursorBasedStrategy.class) -}) public abstract class SearchStrategy { - public abstract SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment); + protected final Filter filter; + protected final Interval interval; - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) { - return false; + protected SearchStrategy(SearchQuery query) + { + this.filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter())); + final List intervals = query.getQuerySegmentSpec().getIntervals(); + if (intervals.size() != 1) { + throw new IAE("Should only have one interval, got[%s]", intervals); } - - return true; + this.interval = intervals.get(0); } - @Override - public int hashCode() { - return toString().hashCode(); - } + public abstract SearchQueryExecutor getExecutionPlan(SearchQuery query, Segment segment); - @Override - public String toString() { - return getClass().getSimpleName(); + public SearchQueryDecisionHelper getDecisionHelper(QueryableIndex index) + { + final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); + if (bitmapFactory.getClass().equals(ConciseBitmapFactory.class)) { + return new ConciseBitmapDecisionHelper(); + } else if (bitmapFactory.getClass().equals(RoaringBitmapFactory.class)) { + return new RoaringBitmapDecisionHelper(); + } else { + throw new IAE("Unknown bitmap type[%s]", bitmapFactory.getClass().getCanonicalName()); + } } } diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 0656bebef1f2..75a69e831a63 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -50,7 +50,6 @@ import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; import io.druid.java.util.common.logger.Logger; -import io.druid.query.search.search.ConciseBitmapDecisionHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnCapabilities; @@ -950,7 +949,6 @@ Column.TIME_COLUMN_NAME, new ColumnBuilder() new ArrayIndexed<>(cols, String.class), index.getAvailableDimensions(), new ConciseBitmapFactory(), - ConciseBitmapDecisionHelper.getInstance(), columns, index.getFileMapper(), null @@ -1033,7 +1031,6 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException cols, dims, segmentBitmapSerdeFactory.getBitmapFactory(), - segmentBitmapSerdeFactory.getDecisionHelper(), columns, smooshedFiles, metadata diff --git a/processing/src/main/java/io/druid/segment/QueryableIndex.java b/processing/src/main/java/io/druid/segment/QueryableIndex.java index 4ab3a78020e8..2ee90efbf1cf 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndex.java @@ -20,7 +20,6 @@ package io.druid.segment; import io.druid.collections.bitmap.BitmapFactory; -import io.druid.query.search.search.SearchQueryDecisionHelper; import io.druid.segment.data.Indexed; import org.joda.time.Interval; @@ -39,8 +38,6 @@ public interface QueryableIndex extends ColumnSelector, Closeable Metadata getMetadata(); Map getDimensionHandlers(); - SearchQueryDecisionHelper getDecisionHelper(); - /** * The close method shouldn't actually be here as this is nasty. We will adjust it in the future. * @throws java.io.IOException if an exception was thrown closing the index diff --git a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java index ab9af66942a0..f91fa7aa0291 100644 --- a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java @@ -23,7 +23,6 @@ import com.google.common.collect.Maps; import io.druid.collections.bitmap.BitmapFactory; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; -import io.druid.query.search.search.SearchQueryDecisionHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.data.Indexed; @@ -44,14 +43,12 @@ public class SimpleQueryableIndex implements QueryableIndex private final SmooshedFileMapper fileMapper; private final Metadata metadata; private final Map dimensionHandlers; - private final SearchQueryDecisionHelper decisionHelper; public SimpleQueryableIndex( Interval dataInterval, Indexed columnNames, Indexed dimNames, BitmapFactory bitmapFactory, - SearchQueryDecisionHelper decisionHelper, Map columns, SmooshedFileMapper fileMapper, Metadata metadata @@ -62,7 +59,6 @@ public SimpleQueryableIndex( this.columnNames = columnNames; this.availableDimensions = dimNames; this.bitmapFactory = bitmapFactory; - this.decisionHelper = decisionHelper; this.columns = columns; this.fileMapper = fileMapper; this.metadata = metadata; @@ -124,12 +120,6 @@ public Map getDimensionHandlers() return dimensionHandlers; } - @Override - public SearchQueryDecisionHelper getDecisionHelper() - { - return decisionHelper; - } - private void initDimensionHandlers() { for (String dim : availableDimensions) { diff --git a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java index 22c32b5a2c5f..4deef2cfd6d9 100644 --- a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.query.search.search.SearchQueryDecisionHelper; /** */ @@ -37,6 +36,4 @@ public interface BitmapSerdeFactory ObjectStrategy getObjectStrategy(); BitmapFactory getBitmapFactory(); - - SearchQueryDecisionHelper getDecisionHelper(); } diff --git a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java index 5fe7c086aa1a..e6626365374b 100644 --- a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java @@ -19,25 +19,21 @@ package io.druid.segment.data; -import java.nio.ByteBuffer; - import com.google.common.collect.Ordering; - import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.WrappedImmutableConciseBitmap; -import io.druid.query.search.search.ConciseBitmapDecisionHelper; -import io.druid.query.search.search.SearchQueryDecisionHelper; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; +import java.nio.ByteBuffer; + /** */ public class ConciseBitmapSerdeFactory implements BitmapSerdeFactory { private static final ObjectStrategy objectStrategy = new ImmutableConciseSetObjectStrategy(); private static final BitmapFactory bitmapFactory = new ConciseBitmapFactory(); - private static final SearchQueryDecisionHelper decisionHelper = ConciseBitmapDecisionHelper.getInstance(); @Override public ObjectStrategy getObjectStrategy() @@ -51,12 +47,6 @@ public BitmapFactory getBitmapFactory() return bitmapFactory; } - @Override - public SearchQueryDecisionHelper getDecisionHelper() - { - return decisionHelper; - } - private static Ordering conciseComparator = new Ordering() { @Override diff --git a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java index 184c6bd5c0a5..9c2ae1d21a85 100644 --- a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java @@ -26,8 +26,6 @@ import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.RoaringBitmapFactory; import io.druid.collections.bitmap.WrappedImmutableRoaringBitmap; -import io.druid.query.search.search.SearchQueryDecisionHelper; -import io.druid.query.search.search.RoaringBitmapDecisionHelper; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; import java.nio.ByteBuffer; @@ -38,7 +36,6 @@ public class RoaringBitmapSerdeFactory implements BitmapSerdeFactory { private static final boolean DEFAULT_COMPRESS_RUN_ON_SERIALIZATION = true; private static final ObjectStrategy objectStrategy = new ImmutableRoaringBitmapObjectStrategy(); - private static final SearchQueryDecisionHelper decisionHelper = RoaringBitmapDecisionHelper.getInstance(); private final boolean compressRunOnSerialization; private final BitmapFactory bitmapFactory; @@ -72,12 +69,6 @@ public BitmapFactory getBitmapFactory() return bitmapFactory; } - @Override - public SearchQueryDecisionHelper getDecisionHelper() - { - return decisionHelper; - } - private static Ordering roaringComparator = new Ordering() { @Override diff --git a/processing/src/test/java/io/druid/query/TestQueryRunners.java b/processing/src/test/java/io/druid/query/TestQueryRunners.java index 382a9fd4b39e..d431e3e62f66 100644 --- a/processing/src/test/java/io/druid/query/TestQueryRunners.java +++ b/processing/src/test/java/io/druid/query/TestQueryRunners.java @@ -23,6 +23,7 @@ import io.druid.collections.StupidPool; import io.druid.query.search.SearchQueryQueryToolChest; import io.druid.query.search.SearchQueryRunnerFactory; +import io.druid.query.search.SearchStrategySelector; import io.druid.query.search.search.SearchQueryConfig; import io.druid.query.timeboundary.TimeBoundaryQueryRunnerFactory; import io.druid.query.timeseries.TimeseriesQueryEngine; @@ -93,8 +94,18 @@ public static QueryRunner makeSearchQueryRunner( Segment adapter ) { - QueryRunnerFactory factory = new SearchQueryRunnerFactory(new SearchQueryQueryToolChest( - new SearchQueryConfig(), + final SearchQueryConfig config = new SearchQueryConfig(); + QueryRunnerFactory factory = new SearchQueryRunnerFactory( + new SearchStrategySelector(new Supplier() + { + @Override + public SearchQueryConfig get() + { + return config; + } + }), + new SearchQueryQueryToolChest( + config, QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator()), QueryRunnerTestHelper.NOOP_QUERYWATCHER); return new FinalizeResultsQueryRunner( diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryQueryToolChestTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryQueryToolChestTest.java index 9787e4f6770c..a3e9b61b9417 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryQueryToolChestTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryQueryToolChestTest.java @@ -59,7 +59,6 @@ public void testCacheStrategy() throws Exception ImmutableList.of(Druids.DIMENSION_IDENTITY.apply("dim1")), new FragmentSearchQuerySpec(ImmutableList.of("a", "b")), null, - null, null ) ); diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 10586137931c..20506fa1783e 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -19,6 +19,7 @@ package io.druid.query.search; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.druid.java.util.common.guava.Sequence; @@ -62,10 +63,19 @@ public class SearchQueryRunnerTest { private static final Logger LOG = new Logger(SearchQueryRunnerTest.class); + private static final SearchQueryConfig config = new SearchQueryConfig(); private static final SearchQueryQueryToolChest toolChest = new SearchQueryQueryToolChest( - new SearchQueryConfig(), + config, QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() ); + private static final SearchStrategySelector selector = new SearchStrategySelector(new Supplier() + { + @Override + public SearchQueryConfig get() + { + return config; + } + }); @Parameterized.Parameters(name="{0}") public static Iterable constructorFeeder() throws IOException @@ -73,6 +83,7 @@ public static Iterable constructorFeeder() throws IOException return QueryRunnerTestHelper.transformToConstructionFeeder( QueryRunnerTestHelper.makeQueryRunners( new SearchQueryRunnerFactory( + selector, toolChest, QueryRunnerTestHelper.NOOP_QUERYWATCHER ) diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java index e94b3f2b5e5c..31e2ff3f58b4 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java @@ -19,6 +19,7 @@ package io.druid.query.search; +import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -27,6 +28,8 @@ import io.druid.query.Druids; import io.druid.query.QueryRunner; import io.druid.query.Result; +import io.druid.query.search.search.CursorBasedStrategy; +import io.druid.query.search.search.IndexOnlyStrategy; import io.druid.query.search.search.SearchHit; import io.druid.query.search.search.SearchQuery; import io.druid.query.search.search.SearchQueryConfig; @@ -68,13 +71,11 @@ public class SearchQueryRunnerWithCaseTest @Parameterized.Parameters public static Iterable constructorFeeder() throws IOException { - SearchQueryRunnerFactory factory = new SearchQueryRunnerFactory( - new SearchQueryQueryToolChest( - new SearchQueryConfig(), - NoopIntervalChunkingQueryRunnerDecorator() - ), - NOOP_QUERYWATCHER - ); + final SearchQueryConfig[] configs = new SearchQueryConfig[2]; + configs[0] = new SearchQueryConfig(); + configs[0].setSearchStrategy(IndexOnlyStrategy.NAME); + configs[1] = new SearchQueryConfig(); + configs[1].setSearchStrategy(CursorBasedStrategy.NAME); CharSource input = CharSource.wrap( "2011-01-12T00:00:00.000Z\tspot\tAutoMotive\tPREFERRED\ta\u0001preferred\t100.000000\n" + @@ -91,14 +92,61 @@ public static Iterable constructorFeeder() throws IOException return transformToConstructionFeeder( Arrays.asList( - makeQueryRunner(factory, "index1", new IncrementalIndexSegment(index1, "index1"), "index1"), - makeQueryRunner(factory, "index2", new IncrementalIndexSegment(index2, "index2"), "index2"), - makeQueryRunner(factory, "index3", new QueryableIndexSegment("index3", index3), "index3"), - makeQueryRunner(factory, "index4", new QueryableIndexSegment("index4", index4), "index4") + makeQueryRunner(makeRunnerFactory(configs[0]), + "index1", + new IncrementalIndexSegment(index1, "index1"), + "index1"), + makeQueryRunner(makeRunnerFactory(configs[0]), + "index2", + new IncrementalIndexSegment(index2, "index2"), + "index2"), + makeQueryRunner(makeRunnerFactory(configs[0]), + "index3", + new QueryableIndexSegment("index3", index3), + "index3"), + makeQueryRunner(makeRunnerFactory(configs[0]), + "index4", + new QueryableIndexSegment("index4", index4), + "index4"), + makeQueryRunner(makeRunnerFactory(configs[1]), + "index1", + new IncrementalIndexSegment(index1, "index1"), + "index1"), + makeQueryRunner(makeRunnerFactory(configs[1]), + "index2", + new IncrementalIndexSegment(index2, "index2"), + "index2"), + makeQueryRunner(makeRunnerFactory(configs[1]), + "index3", + new QueryableIndexSegment("index3", index3), + "index3"), + makeQueryRunner(makeRunnerFactory(configs[1]), + "index4", + new QueryableIndexSegment("index4", index4), + "index4") ) ); } + static SearchQueryRunnerFactory makeRunnerFactory(final SearchQueryConfig config) + { + return new SearchQueryRunnerFactory( + new SearchStrategySelector(new Supplier() + { + @Override + public SearchQueryConfig get() + { + return config; + } + }), + new SearchQueryQueryToolChest( + config, + NoopIntervalChunkingQueryRunnerDecorator() + ), + NOOP_QUERYWATCHER + ); + } + private final QueryRunner runner; public SearchQueryRunnerWithCaseTest( From 91239e6fdc1581dbcc6b6a0c3e0df27e8d523bc0 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 5 Jan 2017 17:34:42 +0900 Subject: [PATCH 4/9] Rename strategies and set UseIndexesStrategy as the default strategy --- .../benchmark/query/SearchBenchmark.java | 10 +---- .../query/search/SearchStrategySelector.java | 39 ++++++++++++++----- .../query/search/search/AutoStrategy.java | 39 ++++++++++++++----- .../search/ConciseBitmapDecisionHelper.java | 27 ++++++++++++- ...dStrategy.java => CursorOnlyStrategy.java} | 25 ++++++++++-- .../search/RoaringBitmapDecisionHelper.java | 27 ++++++++++++- .../search/search/SearchQueryConfig.java | 2 +- .../search/SearchQueryDecisionHelper.java | 19 +++++++++ .../search/search/SearchQueryExecutor.java | 19 +++++++++ .../query/search/search/SearchStrategy.java | 4 +- ...yStrategy.java => UseIndexesStrategy.java} | 29 +++++++++++--- .../main/java/io/druid/segment/IndexIO.java | 2 +- .../java/io/druid/segment/QueryableIndex.java | 14 +++---- .../segment/data/BitmapSerdeFactory.java | 4 +- .../data/ConciseBitmapSerdeFactory.java | 5 ++- .../java/io/druid/query/TestQueryRunners.java | 10 +---- .../query/search/SearchQueryRunnerTest.java | 11 +----- .../search/SearchQueryRunnerWithCaseTest.java | 19 +++------ 18 files changed, 222 insertions(+), 83 deletions(-) rename processing/src/main/java/io/druid/query/search/search/{CursorBasedStrategy.java => CursorOnlyStrategy.java} (80%) rename processing/src/main/java/io/druid/query/search/search/{IndexOnlyStrategy.java => UseIndexesStrategy.java} (88%) diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java index 1866cad41e15..d7eae91d4285 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.hash.Hashing; @@ -258,14 +259,7 @@ public void setup() throws IOException final SearchQueryConfig config = new SearchQueryConfig().withOverrides(query); factory = new SearchQueryRunnerFactory( - new SearchStrategySelector(new Supplier() - { - @Override - public SearchQueryConfig get() - { - return config; - } - }), + new SearchStrategySelector(Suppliers.ofInstance(config)), new SearchQueryQueryToolChest( config, QueryBenchmarkUtil.NoopIntervalChunkingQueryRunnerDecorator() diff --git a/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java b/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java index 02845fc7b692..0fe6bf553edd 100644 --- a/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java +++ b/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java @@ -1,3 +1,22 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.druid.query.search; import com.google.common.base.Supplier; @@ -5,8 +24,8 @@ import com.metamx.emitter.EmittingLogger; import io.druid.java.util.common.ISE; import io.druid.query.search.search.AutoStrategy; -import io.druid.query.search.search.CursorBasedStrategy; -import io.druid.query.search.search.IndexOnlyStrategy; +import io.druid.query.search.search.CursorOnlyStrategy; +import io.druid.query.search.search.UseIndexesStrategy; import io.druid.query.search.search.SearchQuery; import io.druid.query.search.search.SearchQueryConfig; import io.druid.query.search.search.SearchStrategy; @@ -28,16 +47,16 @@ public SearchStrategy strategize(SearchQuery query) switch (strategyString) { case AutoStrategy.NAME: - log.debug("Auto strategy is selected"); + log.debug("Auto strategy is selected, query id [%s]", query.getId()); return new AutoStrategy(query); - case IndexOnlyStrategy.NAME: - log.debug("Index-only execution strategy is selected"); - return new IndexOnlyStrategy(query); - case CursorBasedStrategy.NAME: - log.debug("Cursor-based execution strategy is selected"); - return new CursorBasedStrategy(query); + case UseIndexesStrategy.NAME: + log.debug("Index-only execution strategy is selected, query id [%s]", query.getId()); + return new UseIndexesStrategy(query); + case CursorOnlyStrategy.NAME: + log.debug("Cursor-based execution strategy is selected, query id [%s]", query.getId()); + return new CursorOnlyStrategy(query); default: - throw new ISE("Unknown strategy[%s]", strategyString); + throw new ISE("Unknown strategy[%s], query id [%s]", strategyString, query.getId()); } } } diff --git a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java index 389c8b4e6f85..09f9a21a0b6b 100644 --- a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java @@ -1,3 +1,22 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.druid.query.search.search; import com.metamx.emitter.EmittingLogger; @@ -25,10 +44,10 @@ public List getExecutionPlan(SearchQuery query, Segment seg final QueryableIndex index = segment.asQueryableIndex(); if (index != null) { - final ImmutableBitmap timeFilteredBitmap = IndexOnlyStrategy.makeTimeFilteredBitmap(index, - segment, - filter, - interval); + final ImmutableBitmap timeFilteredBitmap = UseIndexesStrategy.makeTimeFilteredBitmap(index, + segment, + filter, + interval); final Iterable dimsToSearch = getDimsToSearch(index.getAvailableDimensions(), query.getDimensions()); @@ -41,16 +60,16 @@ public List getExecutionPlan(SearchQuery query, Segment seg if (filter == null || helper.hasLowCardinality(index, dimsToSearch) || helper.hasLowSelectivity(index, timeFilteredBitmap)) { - log.debug("Index-only execution strategy is selected"); - return new IndexOnlyStrategy(query, timeFilteredBitmap).getExecutionPlan(query, segment); + log.debug("Index-only execution strategy is selected, query id [%s]", query.getId()); + return new UseIndexesStrategy(query, timeFilteredBitmap).getExecutionPlan(query, segment); } else { - log.debug("Cursor-based execution strategy is selected"); - return new CursorBasedStrategy(query).getExecutionPlan(query, segment); + log.debug("Cursor-based execution strategy is selected, query id [%s]", query.getId()); + return new CursorOnlyStrategy(query).getExecutionPlan(query, segment); } } else { - log.debug("Index doesn't exist. Fall back to cursor-based execution strategy"); - return new CursorBasedStrategy(query).getExecutionPlan(query, segment); + log.debug("Index doesn't exist. Fall back to cursor-based execution strategy, query id [%s]", query.getId()); + return new CursorOnlyStrategy(query).getExecutionPlan(query, segment); } } } diff --git a/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java index 292747ed460e..f30deaeca0f0 100644 --- a/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java +++ b/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java @@ -1,11 +1,36 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.druid.query.search.search; public class ConciseBitmapDecisionHelper extends SearchQueryDecisionHelper { private static final double LOW_FILTER_SELECTIVITY_THRESHOLD = 0.99; private static final int LOW_CARDINALITY_THRESHOLD = 5000; + private static final ConciseBitmapDecisionHelper INSTANCE = new ConciseBitmapDecisionHelper(); + + public static ConciseBitmapDecisionHelper instance() + { + return INSTANCE; + } - public ConciseBitmapDecisionHelper() + private ConciseBitmapDecisionHelper() { super(LOW_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); } diff --git a/processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java similarity index 80% rename from processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java rename to processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java index 1df7b30df38a..ae00915abac3 100644 --- a/processing/src/main/java/io/druid/query/search/search/CursorBasedStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java @@ -1,3 +1,22 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.druid.query.search.search; import com.google.common.collect.ImmutableList; @@ -19,11 +38,11 @@ import java.util.Arrays; import java.util.List; -public class CursorBasedStrategy extends SearchStrategy +public class CursorOnlyStrategy extends SearchStrategy { - public static final String NAME = "cursorBased"; + public static final String NAME = "cursorOnly"; - public CursorBasedStrategy(SearchQuery query) + public CursorOnlyStrategy(SearchQuery query) { super(query); } diff --git a/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java index 4196d2edf853..2c1ff063e218 100644 --- a/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java +++ b/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java @@ -1,11 +1,36 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.druid.query.search.search; public class RoaringBitmapDecisionHelper extends SearchQueryDecisionHelper { private static final double LOW_FILTER_SELECTIVITY_THRESHOLD = 0.65; private static final int LOW_CARDINALITY_THRESHOLD = 1000; + private static final RoaringBitmapDecisionHelper INSTANCE = new RoaringBitmapDecisionHelper(); + + public static RoaringBitmapDecisionHelper instance() + { + return INSTANCE; + } - public RoaringBitmapDecisionHelper() + private RoaringBitmapDecisionHelper() { super(LOW_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); } diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java index 88cd52c765a9..1471036b6ae2 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java @@ -34,7 +34,7 @@ public class SearchQueryConfig private int maxSearchLimit = 1000; @JsonProperty - private String searchStrategy = AutoStrategy.NAME; + private String searchStrategy = UseIndexesStrategy.NAME; public int getMaxSearchLimit() { diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java index 4ec8efd0983d..382d9c40a573 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java @@ -1,3 +1,22 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.druid.query.search.search; import io.druid.collections.bitmap.ImmutableBitmap; diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java index 86bb5ea1de5d..7b66e4aec6c2 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java @@ -1,3 +1,22 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.druid.query.search.search; import io.druid.query.dimension.DimensionSpec; diff --git a/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java b/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java index ef308722e445..528a50faf06f 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchStrategy.java @@ -57,9 +57,9 @@ public SearchQueryDecisionHelper getDecisionHelper(QueryableIndex index) { final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); if (bitmapFactory.getClass().equals(ConciseBitmapFactory.class)) { - return new ConciseBitmapDecisionHelper(); + return ConciseBitmapDecisionHelper.instance(); } else if (bitmapFactory.getClass().equals(RoaringBitmapFactory.class)) { - return new RoaringBitmapDecisionHelper(); + return RoaringBitmapDecisionHelper.instance(); } else { throw new IAE("Unknown bitmap type[%s]", bitmapFactory.getClass().getCanonicalName()); } diff --git a/processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java similarity index 88% rename from processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java rename to processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java index 7eb4004d0411..6b22d7e29bf3 100644 --- a/processing/src/main/java/io/druid/query/search/search/IndexOnlyStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java @@ -1,3 +1,22 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.druid.query.search.search; import com.google.common.base.Preconditions; @@ -12,7 +31,7 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.IdentityExtractionFn; import io.druid.query.filter.Filter; -import io.druid.query.search.search.CursorBasedStrategy.CursorBasedExecutor; +import io.druid.query.search.search.CursorOnlyStrategy.CursorBasedExecutor; import io.druid.segment.ColumnSelectorBitmapIndexSelector; import io.druid.segment.QueryableIndex; import io.druid.segment.Segment; @@ -28,18 +47,18 @@ import java.util.Arrays; import java.util.List; -public class IndexOnlyStrategy extends SearchStrategy +public class UseIndexesStrategy extends SearchStrategy { - public static final String NAME = "indexOnly"; + public static final String NAME = "useIndexes"; private final ImmutableBitmap timeFilteredBitmap; - public IndexOnlyStrategy(SearchQuery query) + public UseIndexesStrategy(SearchQuery query) { this(query, null); } - public IndexOnlyStrategy(SearchQuery query, @Nullable ImmutableBitmap timeFilteredBitmap) + public UseIndexesStrategy(SearchQuery query, @Nullable ImmutableBitmap timeFilteredBitmap) { super(query); this.timeFilteredBitmap = timeFilteredBitmap; diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 75a69e831a63..0fe4c79642a4 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -36,12 +36,12 @@ import com.google.common.io.Files; import com.google.common.primitives.Ints; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.MutableBitmap; import io.druid.collections.spatial.ImmutableRTree; +import com.metamx.emitter.EmittingLogger; import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; diff --git a/processing/src/main/java/io/druid/segment/QueryableIndex.java b/processing/src/main/java/io/druid/segment/QueryableIndex.java index 2ee90efbf1cf..31492b18cc43 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndex.java @@ -31,17 +31,17 @@ */ public interface QueryableIndex extends ColumnSelector, Closeable { - Interval getDataInterval(); - int getNumRows(); - Indexed getAvailableDimensions(); - BitmapFactory getBitmapFactoryForDimensions(); - Metadata getMetadata(); - Map getDimensionHandlers(); + public Interval getDataInterval(); + public int getNumRows(); + public Indexed getAvailableDimensions(); + public BitmapFactory getBitmapFactoryForDimensions(); + public Metadata getMetadata(); + public Map getDimensionHandlers(); /** * The close method shouldn't actually be here as this is nasty. We will adjust it in the future. * @throws java.io.IOException if an exception was thrown closing the index */ //@Deprecated // This is still required for SimpleQueryableIndex. It should not go away unitl SimpleQueryableIndex is fixed - void close() throws IOException; + public void close() throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java index 4deef2cfd6d9..a7c6fe6bcfb2 100644 --- a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java @@ -33,7 +33,7 @@ }) public interface BitmapSerdeFactory { - ObjectStrategy getObjectStrategy(); + public ObjectStrategy getObjectStrategy(); - BitmapFactory getBitmapFactory(); + public BitmapFactory getBitmapFactory(); } diff --git a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java index e6626365374b..c5ef97f594d2 100644 --- a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java @@ -19,15 +19,16 @@ package io.druid.segment.data; +import java.nio.ByteBuffer; + import com.google.common.collect.Ordering; + import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.WrappedImmutableConciseBitmap; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; -import java.nio.ByteBuffer; - /** */ public class ConciseBitmapSerdeFactory implements BitmapSerdeFactory diff --git a/processing/src/test/java/io/druid/query/TestQueryRunners.java b/processing/src/test/java/io/druid/query/TestQueryRunners.java index bfd496843598..87fe81fa35df 100644 --- a/processing/src/test/java/io/druid/query/TestQueryRunners.java +++ b/processing/src/test/java/io/druid/query/TestQueryRunners.java @@ -20,6 +20,7 @@ package io.druid.query; import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import io.druid.collections.StupidPool; import io.druid.query.search.SearchQueryQueryToolChest; import io.druid.query.search.SearchQueryRunnerFactory; @@ -97,14 +98,7 @@ public static QueryRunner makeSearchQueryRunner( { final SearchQueryConfig config = new SearchQueryConfig(); QueryRunnerFactory factory = new SearchQueryRunnerFactory( - new SearchStrategySelector(new Supplier() - { - @Override - public SearchQueryConfig get() - { - return config; - } - }), + new SearchStrategySelector(Suppliers.ofInstance(config)), new SearchQueryQueryToolChest( config, QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator()), diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 540e31e9e63e..1f8aa7b8df97 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -19,7 +19,7 @@ package io.druid.query.search; -import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.druid.java.util.common.guava.Sequence; @@ -70,14 +70,7 @@ public class SearchQueryRunnerTest config, QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() ); - private static final SearchStrategySelector selector = new SearchStrategySelector(new Supplier() - { - @Override - public SearchQueryConfig get() - { - return config; - } - }); + private static final SearchStrategySelector selector = new SearchStrategySelector(Suppliers.ofInstance(config)); @Parameterized.Parameters(name="{0}") public static Iterable constructorFeeder() throws IOException diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java index 31e2ff3f58b4..2bb3155f644d 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java @@ -19,7 +19,7 @@ package io.druid.query.search; -import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -28,8 +28,8 @@ import io.druid.query.Druids; import io.druid.query.QueryRunner; import io.druid.query.Result; -import io.druid.query.search.search.CursorBasedStrategy; -import io.druid.query.search.search.IndexOnlyStrategy; +import io.druid.query.search.search.CursorOnlyStrategy; +import io.druid.query.search.search.UseIndexesStrategy; import io.druid.query.search.search.SearchHit; import io.druid.query.search.search.SearchQuery; import io.druid.query.search.search.SearchQueryConfig; @@ -73,9 +73,9 @@ public static Iterable constructorFeeder() throws IOException { final SearchQueryConfig[] configs = new SearchQueryConfig[2]; configs[0] = new SearchQueryConfig(); - configs[0].setSearchStrategy(IndexOnlyStrategy.NAME); + configs[0].setSearchStrategy(UseIndexesStrategy.NAME); configs[1] = new SearchQueryConfig(); - configs[1].setSearchStrategy(CursorBasedStrategy.NAME); + configs[1].setSearchStrategy(CursorOnlyStrategy.NAME); CharSource input = CharSource.wrap( "2011-01-12T00:00:00.000Z\tspot\tAutoMotive\tPREFERRED\ta\u0001preferred\t100.000000\n" + @@ -131,14 +131,7 @@ public static Iterable constructorFeeder() throws IOException static SearchQueryRunnerFactory makeRunnerFactory(final SearchQueryConfig config) { return new SearchQueryRunnerFactory( - new SearchStrategySelector(new Supplier() - { - @Override - public SearchQueryConfig get() - { - return config; - } - }), + new SearchStrategySelector(Suppliers.ofInstance(config)), new SearchQueryQueryToolChest( config, NoopIntervalChunkingQueryRunnerDecorator() From bb180a7888ca2cea807ece695a34b23cc0d1a60e Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 9 Jan 2017 17:40:26 +0900 Subject: [PATCH 5/9] Add a cost-based planner for auto strategy --- .../benchmark/datagen/BenchmarkSchemas.java | 2 +- .../query/search/search/AutoStrategy.java | 77 ++++++++--- .../search/ConciseBitmapDecisionHelper.java | 7 +- .../search/search/CursorOnlyStrategy.java | 2 +- .../search/RoaringBitmapDecisionHelper.java | 7 +- .../search/SearchQueryDecisionHelper.java | 44 +------ .../search/search/UseIndexesStrategy.java | 120 +++++++++++------- .../io/druid/segment/filter/AndFilter.java | 3 + 8 files changed, 148 insertions(+), 114 deletions(-) diff --git a/benchmarks/src/main/java/io/druid/benchmark/datagen/BenchmarkSchemas.java b/benchmarks/src/main/java/io/druid/benchmark/datagen/BenchmarkSchemas.java index 4222199a2646..e2c8bc176372 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/datagen/BenchmarkSchemas.java +++ b/benchmarks/src/main/java/io/druid/benchmark/datagen/BenchmarkSchemas.java @@ -45,7 +45,7 @@ public class BenchmarkSchemas // dims BenchmarkColumnSchema.makeSequential("dimSequential", ValueType.STRING, false, 1, null, 0, 1000), BenchmarkColumnSchema.makeZipf("dimZipf", ValueType.STRING, false, 1, null, 1, 101, 1.0), - BenchmarkColumnSchema.makeDiscreteUniform("dimUniform", ValueType.STRING, false, 1, null, 1, 1000000), + BenchmarkColumnSchema.makeDiscreteUniform("dimUniform", ValueType.STRING, false, 1, null, 1, 100000), BenchmarkColumnSchema.makeSequential("dimSequentialHalfNull", ValueType.STRING, false, 1, 0.5, 0, 1000), BenchmarkColumnSchema.makeEnumerated( "dimMultivalEnumerated", diff --git a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java index 09f9a21a0b6b..675b66bd50e1 100644 --- a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java @@ -22,8 +22,12 @@ import com.metamx.emitter.EmittingLogger; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.dimension.DimensionSpec; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.segment.ColumnSelectorBitmapIndexSelector; import io.druid.segment.QueryableIndex; import io.druid.segment.Segment; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.column.Column; import java.util.List; @@ -44,32 +48,67 @@ public List getExecutionPlan(SearchQuery query, Segment seg final QueryableIndex index = segment.asQueryableIndex(); if (index != null) { - final ImmutableBitmap timeFilteredBitmap = UseIndexesStrategy.makeTimeFilteredBitmap(index, - segment, - filter, - interval); - final Iterable dimsToSearch = getDimsToSearch(index.getAvailableDimensions(), - query.getDimensions()); + final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( + index.getBitmapFactoryForDimensions(), + index + ); - // Choose a search query execution strategy depending on the query. - // Index-only strategy is selected when - // 1) there is no filter, - // 2) the total cardinality is very low, or - // 3) the filter has a very low selectivity. - final SearchQueryDecisionHelper helper = getDecisionHelper(index); - if (filter == null || - helper.hasLowCardinality(index, dimsToSearch) || - helper.hasLowSelectivity(index, timeFilteredBitmap)) { - log.debug("Index-only execution strategy is selected, query id [%s]", query.getId()); - return new UseIndexesStrategy(query, timeFilteredBitmap).getExecutionPlan(query, segment); + // Index-only plan is used only when any filter is not specified or every filter supports bitmap indexes. + // + // Note: if some filters support bitmap indexes but others are not, the current implementation always employs + // the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index + // from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps. + if (filter == null || filter.supportsBitmapIndex(selector)) { + final ImmutableBitmap timeFilteredBitmap = UseIndexesStrategy.makeTimeFilteredBitmap(index, + segment, + filter, + interval); + final List dimsToSearch = getDimsToSearch(index.getAvailableDimensions(), + query.getDimensions()); + + // Choose a search query execution strategy depending on the query. + // The costs of index-only plan and cursor-based plan can be computed like below. + // + // c_index = (total cardinality of all search dimensions) * (bitmap intersection cost) + // * (search predicate processing cost) + // c_cursor = (# of rows in a segment) * (filter selectivity) * (# of dimensions) + // * (search predicate processing cost) + final SearchQueryDecisionHelper helper = getDecisionHelper(index); + final double useIndexStrategyCost = helper.getBitmapIntersectCost() * computeTotalCard(index, dimsToSearch); + final double cursorOnlyStrategyCost = timeFilteredBitmap.size() * dimsToSearch.size(); + log.debug("useIndexStrategyCost: %f, cursorOnlyStrategyCost: %f", useIndexStrategyCost, cursorOnlyStrategyCost); + + if (useIndexStrategyCost < cursorOnlyStrategyCost) { + log.debug("Use-index execution strategy is selected, query id [%s]", query.getId()); + return new UseIndexesStrategy(query, timeFilteredBitmap).getExecutionPlan(query, segment); + } else { + log.debug("Cursor-only execution strategy is selected, query id [%s]", query.getId()); + return new CursorOnlyStrategy(query).getExecutionPlan(query, segment); + } } else { - log.debug("Cursor-based execution strategy is selected, query id [%s]", query.getId()); + log.debug("Filter doesn't support bitmap index. Fall back to cursor-only execution strategy, query id [%s]", + query.getId()); return new CursorOnlyStrategy(query).getExecutionPlan(query, segment); } } else { - log.debug("Index doesn't exist. Fall back to cursor-based execution strategy, query id [%s]", query.getId()); + log.debug("Index doesn't exist. Fall back to cursor-only execution strategy, query id [%s]", query.getId()); return new CursorOnlyStrategy(query).getExecutionPlan(query, segment); } } + + private static long computeTotalCard(final QueryableIndex index, final Iterable dimensionSpecs) + { + long totalCard = 0; + for (DimensionSpec dimension : dimensionSpecs) { + final Column column = index.getColumn(dimension.getDimension()); + if (column != null) { + final BitmapIndex bitmapIndex = column.getBitmapIndex(); + if (bitmapIndex != null) { + totalCard += bitmapIndex.getCardinality(); + } + } + } + return totalCard; + } } diff --git a/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java index f30deaeca0f0..c0239e34ac9e 100644 --- a/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java +++ b/processing/src/main/java/io/druid/query/search/search/ConciseBitmapDecisionHelper.java @@ -21,8 +21,9 @@ public class ConciseBitmapDecisionHelper extends SearchQueryDecisionHelper { - private static final double LOW_FILTER_SELECTIVITY_THRESHOLD = 0.99; - private static final int LOW_CARDINALITY_THRESHOLD = 5000; + // This value comes from an experiment. + // See the discussion at https://github.com/druid-io/druid/pull/3792#issuecomment-268331804. + private static final double BITMAP_INTERSECT_COST = 7.425; private static final ConciseBitmapDecisionHelper INSTANCE = new ConciseBitmapDecisionHelper(); public static ConciseBitmapDecisionHelper instance() @@ -32,6 +33,6 @@ public static ConciseBitmapDecisionHelper instance() private ConciseBitmapDecisionHelper() { - super(LOW_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); + super(BITMAP_INTERSECT_COST); } } diff --git a/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java index ae00915abac3..f8084de775f7 100644 --- a/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java @@ -102,7 +102,7 @@ public Object2IntRBTreeMap accumulate(Object2IntRBTreeMap return set; } - List> selectorPlusList = Arrays.asList( + final List> selectorPlusList = Arrays.asList( DimensionHandlerUtils.createColumnSelectorPluses( SearchQueryRunner.SEARCH_COLUMN_SELECTOR_STRATEGY_FACTORY, dimsToSearch, diff --git a/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java index 2c1ff063e218..4ca2d98024d4 100644 --- a/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java +++ b/processing/src/main/java/io/druid/query/search/search/RoaringBitmapDecisionHelper.java @@ -21,8 +21,9 @@ public class RoaringBitmapDecisionHelper extends SearchQueryDecisionHelper { - private static final double LOW_FILTER_SELECTIVITY_THRESHOLD = 0.65; - private static final int LOW_CARDINALITY_THRESHOLD = 1000; + // This value comes from an experiment. + // See the discussion at https://github.com/druid-io/druid/pull/3792#issuecomment-268331804. + private static final double BITMAP_INTERSECT_COST = 4.5; private static final RoaringBitmapDecisionHelper INSTANCE = new RoaringBitmapDecisionHelper(); public static RoaringBitmapDecisionHelper instance() @@ -32,6 +33,6 @@ public static RoaringBitmapDecisionHelper instance() private RoaringBitmapDecisionHelper() { - super(LOW_FILTER_SELECTIVITY_THRESHOLD, LOW_CARDINALITY_THRESHOLD); + super(BITMAP_INTERSECT_COST); } } diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java index 382d9c40a573..fc965a432c6b 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryDecisionHelper.java @@ -19,51 +19,17 @@ package io.druid.query.search.search; -import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.query.dimension.DimensionSpec; -import io.druid.segment.QueryableIndex; -import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; - public abstract class SearchQueryDecisionHelper { - protected final double lowFilterSelectivityThreshold; - protected final int lowCardinalityThreshold; - - protected SearchQueryDecisionHelper(final double lowFilterSelectivityThreshold, final int lowCardinalityThreshold) - { - this.lowFilterSelectivityThreshold = lowFilterSelectivityThreshold; - this.lowCardinalityThreshold = lowCardinalityThreshold; - } + private final double bitmapIntersectCost; - public double getLowFilterSelectivityThreshold() + protected SearchQueryDecisionHelper(final double bitmapIntersectCost) { - return lowFilterSelectivityThreshold; - } - - public int getLowCardinalityThreshold() - { - return lowCardinalityThreshold; - } - - public boolean hasLowCardinality(final QueryableIndex index, final Iterable dimensionSpecs) - { - long totalCard = 0; - for (DimensionSpec dimension : dimensionSpecs) { - final Column column = index.getColumn(dimension.getDimension()); - if (column != null) { - final BitmapIndex bitmapIndex = column.getBitmapIndex(); - if (bitmapIndex != null) { - totalCard += bitmapIndex.getCardinality(); - } - } - } - - return totalCard < lowCardinalityThreshold; + this.bitmapIntersectCost = bitmapIntersectCost; } - public boolean hasLowSelectivity(final QueryableIndex index, final ImmutableBitmap bitmap) + public double getBitmapIntersectCost() { - return index.getNumRows() * lowFilterSelectivityThreshold < bitmap.size(); + return bitmapIntersectCost; } } diff --git a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java index 6b22d7e29bf3..25fd3d2f6e14 100644 --- a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java @@ -30,6 +30,7 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.IdentityExtractionFn; +import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.Filter; import io.druid.query.search.search.CursorOnlyStrategy.CursorBasedExecutor; import io.druid.segment.ColumnSelectorBitmapIndexSelector; @@ -67,25 +68,44 @@ public UseIndexesStrategy(SearchQuery query, @Nullable ImmutableBitmap timeFilte @Override public List getExecutionPlan(SearchQuery query, Segment segment) { + final ImmutableList.Builder builder = ImmutableList.builder(); final QueryableIndex index = segment.asQueryableIndex(); final StorageAdapter adapter = segment.asStorageAdapter(); + final List searchDims = getDimsToSearch(adapter.getAvailableDimensions(), query.getDimensions()); - final Pair, List> pair = // pair of bitmap dims and non-bitmap dims - partitionDimensionList(index, adapter, getDimsToSearch(adapter.getAvailableDimensions(), query.getDimensions())); - final List bitmapSuppDims = pair.lhs; - final List nonBitmapSuppDims = pair.rhs; - - final ImmutableList.Builder builder = ImmutableList.builder(); + if (index != null) { + final Pair, List> pair = // pair of bitmap dims and non-bitmap dims + partitionDimensionList(adapter, searchDims); + final List bitmapSuppDims = pair.lhs; + final List nonBitmapSuppDims = pair.rhs; + + if (bitmapSuppDims.size() > 0) { + final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( + index.getBitmapFactoryForDimensions(), + index + ); - if (bitmapSuppDims.size() > 0) { - final ImmutableBitmap timeFilteredBitmap = this.timeFilteredBitmap == null ? - makeTimeFilteredBitmap(index, segment, filter, interval) : - this.timeFilteredBitmap; - builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims)); - } + // Index-only plan is used only when any filter is not specified or every filter supports bitmap indexes. + // + // Note: if some filters support bitmap indexes but others are not, the current implementation always employs + // the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index + // from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps. + if (filter == null || filter.supportsBitmapIndex(selector)) { + final ImmutableBitmap timeFilteredBitmap = this.timeFilteredBitmap == null ? + makeTimeFilteredBitmap(index, segment, filter, interval) : + this.timeFilteredBitmap; + builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims)); + } else { + // Fall back to cursor-based execution strategy + nonBitmapSuppDims.addAll(bitmapSuppDims); + } + } - if (nonBitmapSuppDims.size() > 0) { - builder.add(new CursorBasedExecutor(query, segment, filter, interval, nonBitmapSuppDims)); + if (nonBitmapSuppDims.size() > 0) { + builder.add(new CursorBasedExecutor(query, segment, filter, interval, nonBitmapSuppDims)); + } + } else { + builder.add(new CursorBasedExecutor(query, segment, filter, interval, searchDims)); } return builder.build(); @@ -93,7 +113,6 @@ public List getExecutionPlan(SearchQuery query, Segment seg // Split dimension list into bitmap-supporting list and non-bitmap supporting list private static Pair, List> partitionDimensionList( - QueryableIndex index, StorageAdapter adapter, List dimensions ) @@ -101,24 +120,19 @@ private static Pair, List> partitionDimension final List bitmapDims = Lists.newArrayList(); final List nonBitmapDims = Lists.newArrayList(); final List dimsToSearch = getDimsToSearch(adapter.getAvailableDimensions(), - dimensions); + dimensions); - if (index != null) { - for (DimensionSpec spec : dimsToSearch) { - ColumnCapabilities capabilities = adapter.getColumnCapabilities(spec.getDimension()); - if (capabilities == null) { - continue; - } + for (DimensionSpec spec : dimsToSearch) { + ColumnCapabilities capabilities = adapter.getColumnCapabilities(spec.getDimension()); + if (capabilities == null) { + continue; + } - if (capabilities.hasBitmapIndexes()) { - bitmapDims.add(spec); - } else { - nonBitmapDims.add(spec); - } + if (capabilities.hasBitmapIndexes()) { + bitmapDims.add(spec); + } else { + nonBitmapDims.add(spec); } - } else { - // no QueryableIndex available, so nothing has bitmaps - nonBitmapDims.addAll(dimsToSearch); } return new Pair, List>(ImmutableList.copyOf(bitmapDims), @@ -131,12 +145,21 @@ static ImmutableBitmap makeTimeFilteredBitmap(final QueryableIndex index, final Interval interval) { final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); - final ImmutableBitmap baseFilter = - filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); + final ImmutableBitmap baseFilter; + if (filter == null) { + baseFilter = null; + } else { + final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( + index.getBitmapFactoryForDimensions(), + index + ); + Preconditions.checkArgument(filter.supportsBitmapIndex(selector), "filter[%s] should support bitmap", filter); + baseFilter = filter.getBitmapIndex(selector); + } final ImmutableBitmap timeFilteredBitmap; if (!interval.contains(segment.getDataInterval())) { - MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); + final MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); final Column timeColumn = index.getColumn(Column.TIME_COLUMN_NAME); try (final GenericColumn timeValues = timeColumn.getGenericColumn()) { @@ -221,25 +244,26 @@ public Object2IntRBTreeMap execute(int limit) } final BitmapIndex bitmapIndex = column.getBitmapIndex(); + Preconditions.checkArgument(bitmapIndex != null, + "Dimension [%s] should support bitmap index", dimension.getDimension()); + ExtractionFn extractionFn = dimension.getExtractionFn(); if (extractionFn == null) { extractionFn = IdentityExtractionFn.getInstance(); } - if (bitmapIndex != null) { - for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { - String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); - if (!searchQuerySpec.accept(dimVal)) { - continue; - } - ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); - if (timeFilteredBitmap != null) { - bitmap = bitmapFactory.intersection(Arrays.asList(timeFilteredBitmap, bitmap)); - } - if (bitmap.size() > 0) { - retVal.addTo(new SearchHit(dimension.getOutputName(), dimVal), bitmap.size()); - if (retVal.size() >= limit) { - return retVal; - } + for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { + String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); + if (!searchQuerySpec.accept(dimVal)) { + continue; + } + ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); + if (timeFilteredBitmap != null) { + bitmap = bitmapFactory.intersection(Arrays.asList(timeFilteredBitmap, bitmap)); + } + if (bitmap.size() > 0) { + retVal.addTo(new SearchHit(dimension.getOutputName(), dimVal), bitmap.size()); + if (retVal.size() >= limit) { + return retVal; } } } diff --git a/processing/src/main/java/io/druid/segment/filter/AndFilter.java b/processing/src/main/java/io/druid/segment/filter/AndFilter.java index 5365c151a80a..56ca28974856 100644 --- a/processing/src/main/java/io/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/AndFilter.java @@ -20,6 +20,7 @@ package io.druid.segment.filter; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.filter.BitmapIndexSelector; @@ -55,6 +56,8 @@ public static ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector, List< final List bitmaps = Lists.newArrayListWithCapacity(filters.size()); for (final Filter filter : filters) { + Preconditions.checkArgument(filter.supportsBitmapIndex(selector), + "Filter[%s] does not support bitmap index", filter); final ImmutableBitmap bitmapIndex = filter.getBitmapIndex(selector); if (bitmapIndex.isEmpty()) { // Short-circuit. From ef6c88d6bff630f8bef85fc5cd4a4ef328413526 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 10 Jan 2017 01:05:03 +0900 Subject: [PATCH 6/9] Add document --- .../benchmark/query/SearchBenchmark.java | 1 - docs/content/querying/searchquery.md | 39 +++++++++++++++++++ .../query/search/SearchStrategySelector.java | 10 ++--- .../query/search/search/AutoStrategy.java | 21 ++++++---- .../search/search/CursorOnlyStrategy.java | 7 +++- .../search/search/UseIndexesStrategy.java | 18 +++++++-- 6 files changed, 78 insertions(+), 18 deletions(-) diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java index d7eae91d4285..3d066974bbb3 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java @@ -20,7 +20,6 @@ package io.druid.benchmark.query; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.Lists; import com.google.common.collect.Maps; diff --git a/docs/content/querying/searchquery.md b/docs/content/querying/searchquery.md index 7ebca672bfc8..717b07e82872 100644 --- a/docs/content/querying/searchquery.md +++ b/docs/content/querying/searchquery.md @@ -77,3 +77,42 @@ The format of the result is: } ] ``` + +### Implementation details + +#### Strategies + +Search queries can be executed using two different strategies. The default strategy is determeined by the +"druid.query.search.searchStrategy" runtime property on the broker. This can be overriden using "searchStrategy" in the +query context. If neither the context field nor the property is set, the "useIndexes" strategy will be used. + +- "useIndexes" strategy, the default, first categorizes search dimensions into two groups according to their support for +bitmap indexes. And then, it applies index-only and cursor-based execution plans to the group of dimensions supporting +bitmaps and others, respectively. The index-only plan uses only indexes for search query processing. For each dimension, it reads the bitmap index for +each dimension value, evaluates the search predicate, and finally checks the time interval and filter predicates. For the +cursor-based execution plan, please refer to the "cursorOnly" strategy. The index-only plan shows low performance for +the search dimensions of large cardinality which means most values of search dimensions are unique. + +- "cursorOnly" strategy generates a cursor-based execution plan. This plan creates a cursor which reads a row from a +queryableIndexSegment, and then evaluates search predicates. If some filters support bitmap indexes, the cursor can read +only the rows which satisfy those filters, thereby saving I/O cost. However, it might be slow with filters of low selectivity. + +- "auto" strategy uses a cost-based planner for choosing an optimal search strategy. It estimates the cost of index-only +and cursor-based execution plans, and chooses the optimal one. Currently, its performance is suboptimal due to the large +overhead of cost estimation. + +#### Server configuration + +The following runtime properties apply: + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.query.search.searchStrategy`|Default search query strategy.|useIndexes| + +#### Query context + +The following runtime properties apply: + +|Property|Description| +|--------|-----------| +|`searchStrategy`|Overrides the value of `druid.query.search.searchStrategy` for this query.| diff --git a/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java b/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java index 0fe6bf553edd..b2c79d55f974 100644 --- a/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java +++ b/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java @@ -48,13 +48,13 @@ public SearchStrategy strategize(SearchQuery query) switch (strategyString) { case AutoStrategy.NAME: log.debug("Auto strategy is selected, query id [%s]", query.getId()); - return new AutoStrategy(query); + return AutoStrategy.of(query); case UseIndexesStrategy.NAME: - log.debug("Index-only execution strategy is selected, query id [%s]", query.getId()); - return new UseIndexesStrategy(query); + log.debug("Use-index strategy is selected, query id [%s]", query.getId()); + return UseIndexesStrategy.of(query); case CursorOnlyStrategy.NAME: - log.debug("Cursor-based execution strategy is selected, query id [%s]", query.getId()); - return new CursorOnlyStrategy(query); + log.debug("Cursor-only strategy is selected, query id [%s]", query.getId()); + return CursorOnlyStrategy.of(query); default: throw new ISE("Unknown strategy[%s], query id [%s]", strategyString, query.getId()); } diff --git a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java index 675b66bd50e1..7759988709d8 100644 --- a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java @@ -37,7 +37,12 @@ public class AutoStrategy extends SearchStrategy private static final EmittingLogger log = new EmittingLogger(AutoStrategy.class); - public AutoStrategy(SearchQuery query) + public static AutoStrategy of(SearchQuery query) + { + return new AutoStrategy(query); + } + + private AutoStrategy(SearchQuery query) { super(query); } @@ -75,25 +80,27 @@ public List getExecutionPlan(SearchQuery query, Segment seg // * (search predicate processing cost) final SearchQueryDecisionHelper helper = getDecisionHelper(index); final double useIndexStrategyCost = helper.getBitmapIntersectCost() * computeTotalCard(index, dimsToSearch); - final double cursorOnlyStrategyCost = timeFilteredBitmap.size() * dimsToSearch.size(); - log.debug("useIndexStrategyCost: %f, cursorOnlyStrategyCost: %f", useIndexStrategyCost, cursorOnlyStrategyCost); + final double cursorOnlyStrategyCost = + (timeFilteredBitmap == null ? index.getNumRows() : timeFilteredBitmap.size()) * dimsToSearch.size(); + log.debug("Use-index strategy cost: %f, cursor-only strategy cost: %f", + useIndexStrategyCost, cursorOnlyStrategyCost); if (useIndexStrategyCost < cursorOnlyStrategyCost) { log.debug("Use-index execution strategy is selected, query id [%s]", query.getId()); - return new UseIndexesStrategy(query, timeFilteredBitmap).getExecutionPlan(query, segment); + return UseIndexesStrategy.withTimeFilteredBitmap(query, timeFilteredBitmap).getExecutionPlan(query, segment); } else { log.debug("Cursor-only execution strategy is selected, query id [%s]", query.getId()); - return new CursorOnlyStrategy(query).getExecutionPlan(query, segment); + return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment); } } else { log.debug("Filter doesn't support bitmap index. Fall back to cursor-only execution strategy, query id [%s]", query.getId()); - return new CursorOnlyStrategy(query).getExecutionPlan(query, segment); + return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment); } } else { log.debug("Index doesn't exist. Fall back to cursor-only execution strategy, query id [%s]", query.getId()); - return new CursorOnlyStrategy(query).getExecutionPlan(query, segment); + return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment); } } diff --git a/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java index f8084de775f7..44b8a6878dff 100644 --- a/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java @@ -42,7 +42,12 @@ public class CursorOnlyStrategy extends SearchStrategy { public static final String NAME = "cursorOnly"; - public CursorOnlyStrategy(SearchQuery query) + public static CursorOnlyStrategy of(SearchQuery query) + { + return new CursorOnlyStrategy(query); + } + + private CursorOnlyStrategy(SearchQuery query) { super(query); } diff --git a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java index 25fd3d2f6e14..7cc919cbc5f8 100644 --- a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java @@ -53,15 +53,25 @@ public class UseIndexesStrategy extends SearchStrategy public static final String NAME = "useIndexes"; private final ImmutableBitmap timeFilteredBitmap; + private final boolean needToMakeFilteredBitmap; - public UseIndexesStrategy(SearchQuery query) + public static UseIndexesStrategy of(SearchQuery query) { - this(query, null); + return new UseIndexesStrategy(query, true, null); } - public UseIndexesStrategy(SearchQuery query, @Nullable ImmutableBitmap timeFilteredBitmap) + public static UseIndexesStrategy withTimeFilteredBitmap(SearchQuery query, + @Nullable ImmutableBitmap timeFilteredBitmap) + { + return new UseIndexesStrategy(query, false, timeFilteredBitmap); + } + + private UseIndexesStrategy(SearchQuery query, + boolean needToMakeFilteredBitmap, + @Nullable ImmutableBitmap timeFilteredBitmap) { super(query); + this.needToMakeFilteredBitmap = needToMakeFilteredBitmap; this.timeFilteredBitmap = timeFilteredBitmap; } @@ -91,7 +101,7 @@ public List getExecutionPlan(SearchQuery query, Segment seg // the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index // from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps. if (filter == null || filter.supportsBitmapIndex(selector)) { - final ImmutableBitmap timeFilteredBitmap = this.timeFilteredBitmap == null ? + final ImmutableBitmap timeFilteredBitmap = this.needToMakeFilteredBitmap ? makeTimeFilteredBitmap(index, segment, filter, interval) : this.timeFilteredBitmap; builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims)); From d616106f3ede9983e70fa5952367e78c64fd2555 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 10 Jan 2017 01:13:29 +0900 Subject: [PATCH 7/9] Fix code style --- .../io/druid/query/search/search/CursorOnlyStrategy.java | 3 ++- .../io/druid/query/search/search/SearchQueryConfig.java | 9 ++++++--- .../druid/query/search/search/SearchQueryExecutor.java | 3 ++- .../io/druid/query/search/search/UseIndexesStrategy.java | 3 ++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java index 44b8a6878dff..d29f9defb1d0 100644 --- a/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java @@ -64,7 +64,8 @@ public List getExecutionPlan(SearchQuery query, Segment seg dimensionSpecs)); } - public static class CursorBasedExecutor extends SearchQueryExecutor { + public static class CursorBasedExecutor extends SearchQueryExecutor + { protected Filter filter; protected Interval interval; diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java index 1471036b6ae2..5e2280c0df5d 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java @@ -41,15 +41,18 @@ public int getMaxSearchLimit() return maxSearchLimit; } - public String getSearchStrategy() { + public String getSearchStrategy() + { return searchStrategy; } - public void setSearchStrategy(final String strategy) { + public void setSearchStrategy(final String strategy) + { this.searchStrategy = strategy; } - public SearchQueryConfig withOverrides(final SearchQuery query) { + public SearchQueryConfig withOverrides(final SearchQuery query) + { final SearchQueryConfig newConfig = new SearchQueryConfig(); newConfig.maxSearchLimit = query.getLimit(); newConfig.searchStrategy = query.getContextValue(CTX_KEY_STRATEGY, searchStrategy); diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java index 7b66e4aec6c2..829550c7e7b9 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryExecutor.java @@ -32,7 +32,8 @@ public abstract class SearchQueryExecutor protected final Segment segment; protected final List dimsToSearch; - public SearchQueryExecutor(SearchQuery query, Segment segment, List dimensionSpecs) { + public SearchQueryExecutor(SearchQuery query, Segment segment, List dimensionSpecs) + { this.query = query; this.segment = segment; this.searchQuerySpec = query.getQuery(); diff --git a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java index 7cc919cbc5f8..670ff0068a28 100644 --- a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java @@ -224,7 +224,8 @@ private static int getStartIndexOfTime(GenericColumn timeValues, long time, bool return inclusive ? low : low - 1; } - public static class IndexOnlyExecutor extends SearchQueryExecutor { + public static class IndexOnlyExecutor extends SearchQueryExecutor + { private final ImmutableBitmap timeFilteredBitmap; From e137bc275c677199180147b9005a25309f7ab6b6 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 11 Jan 2017 08:30:38 +0900 Subject: [PATCH 8/9] apply code style --- .../druid/query/search/SearchQueryRunner.java | 12 +-- .../query/search/search/AutoStrategy.java | 25 +++--- .../search/search/CursorOnlyStrategy.java | 22 ++--- .../search/search/UseIndexesStrategy.java | 47 +++++++---- .../main/java/io/druid/segment/IndexIO.java | 8 +- .../io/druid/segment/filter/AndFilter.java | 3 +- .../java/io/druid/query/TestQueryRunners.java | 14 ++-- .../query/search/SearchQueryRunnerTest.java | 7 +- .../search/SearchQueryRunnerWithCaseTest.java | 80 +++++++++++-------- 9 files changed, 129 insertions(+), 89 deletions(-) diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 38243f383aff..1c64a37bd966 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -74,7 +74,7 @@ public SearchColumnSelectorStrategy makeColumnSelectorStrategy( ) { ValueType type = capabilities.getType(); - switch(type) { + switch (type) { case STRING: return new StringSearchColumnSelectorStrategy(); default: @@ -88,17 +88,17 @@ public interface SearchColumnSelectorStrategy * For each row value: * 1. Check if searchQuerySpec accept()s the value * 2. If so, add the value to the result set and increment the counter for that value * 3. If the size of the result set reaches the limit after adding a value, return early. * - * @param outputName Output name for this dimension in the search query being served - * @param dimSelector Dimension value selector + * @param outputName Output name for this dimension in the search query being served + * @param dimSelector Dimension value selector * @param searchQuerySpec Spec for the search query - * @param set The result set of the search query - * @param limit The limit of the search query + * @param set The result set of the search query + * @param limit The limit of the search query */ void updateSearchResultSet( String outputName, diff --git a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java index 7759988709d8..81dc05ca4cf9 100644 --- a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java @@ -64,12 +64,16 @@ public List getExecutionPlan(SearchQuery query, Segment seg // the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index // from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps. if (filter == null || filter.supportsBitmapIndex(selector)) { - final ImmutableBitmap timeFilteredBitmap = UseIndexesStrategy.makeTimeFilteredBitmap(index, - segment, - filter, - interval); - final List dimsToSearch = getDimsToSearch(index.getAvailableDimensions(), - query.getDimensions()); + final ImmutableBitmap timeFilteredBitmap = UseIndexesStrategy.makeTimeFilteredBitmap( + index, + segment, + filter, + interval + ); + final List dimsToSearch = getDimsToSearch( + index.getAvailableDimensions(), + query.getDimensions() + ); // Choose a search query execution strategy depending on the query. // The costs of index-only plan and cursor-based plan can be computed like below. @@ -83,7 +87,8 @@ public List getExecutionPlan(SearchQuery query, Segment seg final double cursorOnlyStrategyCost = (timeFilteredBitmap == null ? index.getNumRows() : timeFilteredBitmap.size()) * dimsToSearch.size(); log.debug("Use-index strategy cost: %f, cursor-only strategy cost: %f", - useIndexStrategyCost, cursorOnlyStrategyCost); + useIndexStrategyCost, cursorOnlyStrategyCost + ); if (useIndexStrategyCost < cursorOnlyStrategyCost) { log.debug("Use-index execution strategy is selected, query id [%s]", query.getId()); @@ -93,8 +98,10 @@ public List getExecutionPlan(SearchQuery query, Segment seg return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment); } } else { - log.debug("Filter doesn't support bitmap index. Fall back to cursor-only execution strategy, query id [%s]", - query.getId()); + log.debug( + "Filter doesn't support bitmap index. Fall back to cursor-only execution strategy, query id [%s]", + query.getId() + ); return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment); } diff --git a/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java index d29f9defb1d0..6bee733e7184 100644 --- a/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/CursorOnlyStrategy.java @@ -57,11 +57,13 @@ public List getExecutionPlan(SearchQuery query, Segment seg { final StorageAdapter adapter = segment.asStorageAdapter(); final List dimensionSpecs = getDimsToSearch(adapter.getAvailableDimensions(), query.getDimensions()); - return ImmutableList.of(new CursorBasedExecutor(query, - segment, - filter, - interval, - dimensionSpecs)); + return ImmutableList.of(new CursorBasedExecutor( + query, + segment, + filter, + interval, + dimensionSpecs + )); } public static class CursorBasedExecutor extends SearchQueryExecutor @@ -70,10 +72,12 @@ public static class CursorBasedExecutor extends SearchQueryExecutor protected Filter filter; protected Interval interval; - public CursorBasedExecutor(SearchQuery query, - Segment segment, - Filter filter, - Interval interval, List dimensionSpecs) + public CursorBasedExecutor( + SearchQuery query, + Segment segment, + Filter filter, + Interval interval, List dimensionSpecs + ) { super(query, segment, dimensionSpecs); diff --git a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java index 670ff0068a28..6283f8bb198d 100644 --- a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java @@ -60,15 +60,19 @@ public static UseIndexesStrategy of(SearchQuery query) return new UseIndexesStrategy(query, true, null); } - public static UseIndexesStrategy withTimeFilteredBitmap(SearchQuery query, - @Nullable ImmutableBitmap timeFilteredBitmap) + public static UseIndexesStrategy withTimeFilteredBitmap( + SearchQuery query, + @Nullable ImmutableBitmap timeFilteredBitmap + ) { return new UseIndexesStrategy(query, false, timeFilteredBitmap); } - private UseIndexesStrategy(SearchQuery query, - boolean needToMakeFilteredBitmap, - @Nullable ImmutableBitmap timeFilteredBitmap) + private UseIndexesStrategy( + SearchQuery query, + boolean needToMakeFilteredBitmap, + @Nullable ImmutableBitmap timeFilteredBitmap + ) { super(query); this.needToMakeFilteredBitmap = needToMakeFilteredBitmap; @@ -129,8 +133,10 @@ private static Pair, List> partitionDimension { final List bitmapDims = Lists.newArrayList(); final List nonBitmapDims = Lists.newArrayList(); - final List dimsToSearch = getDimsToSearch(adapter.getAvailableDimensions(), - dimensions); + final List dimsToSearch = getDimsToSearch( + adapter.getAvailableDimensions(), + dimensions + ); for (DimensionSpec spec : dimsToSearch) { ColumnCapabilities capabilities = adapter.getColumnCapabilities(spec.getDimension()); @@ -145,14 +151,18 @@ private static Pair, List> partitionDimension } } - return new Pair, List>(ImmutableList.copyOf(bitmapDims), - ImmutableList.copyOf(nonBitmapDims)); + return new Pair, List>( + ImmutableList.copyOf(bitmapDims), + ImmutableList.copyOf(nonBitmapDims) + ); } - static ImmutableBitmap makeTimeFilteredBitmap(final QueryableIndex index, - final Segment segment, - final Filter filter, - final Interval interval) + static ImmutableBitmap makeTimeFilteredBitmap( + final QueryableIndex index, + final Segment segment, + final Filter filter, + final Interval interval + ) { final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); final ImmutableBitmap baseFilter; @@ -229,9 +239,11 @@ public static class IndexOnlyExecutor extends SearchQueryExecutor private final ImmutableBitmap timeFilteredBitmap; - public IndexOnlyExecutor(SearchQuery query, Segment segment, - ImmutableBitmap timeFilteredBitmap, - List dimensionSpecs) + public IndexOnlyExecutor( + SearchQuery query, Segment segment, + ImmutableBitmap timeFilteredBitmap, + List dimensionSpecs + ) { super(query, segment, dimensionSpecs); this.timeFilteredBitmap = timeFilteredBitmap; @@ -256,7 +268,8 @@ public Object2IntRBTreeMap execute(int limit) final BitmapIndex bitmapIndex = column.getBitmapIndex(); Preconditions.checkArgument(bitmapIndex != null, - "Dimension [%s] should support bitmap index", dimension.getDimension()); + "Dimension [%s] should support bitmap index", dimension.getDimension() + ); ExtractionFn extractionFn = dimension.getExtractionFn(); if (extractionFn == null) { diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 0fe4c79642a4..80c9b69dd930 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -1027,13 +1027,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException columns.put(Column.TIME_COLUMN_NAME, deserializeColumn(mapper, smooshedFiles.mapFile("__time"))); final QueryableIndex index = new SimpleQueryableIndex( - dataInterval, - cols, - dims, - segmentBitmapSerdeFactory.getBitmapFactory(), - columns, - smooshedFiles, - metadata + dataInterval, cols, dims, segmentBitmapSerdeFactory.getBitmapFactory(), columns, smooshedFiles, metadata ); log.debug("Mapped v9 index[%s] in %,d millis", inDir, System.currentTimeMillis() - startTime); diff --git a/processing/src/main/java/io/druid/segment/filter/AndFilter.java b/processing/src/main/java/io/druid/segment/filter/AndFilter.java index 56ca28974856..5b5d5be363c1 100644 --- a/processing/src/main/java/io/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/AndFilter.java @@ -57,7 +57,8 @@ public static ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector, List< final List bitmaps = Lists.newArrayListWithCapacity(filters.size()); for (final Filter filter : filters) { Preconditions.checkArgument(filter.supportsBitmapIndex(selector), - "Filter[%s] does not support bitmap index", filter); + "Filter[%s] does not support bitmap index", filter + ); final ImmutableBitmap bitmapIndex = filter.getBitmapIndex(selector); if (bitmapIndex.isEmpty()) { // Short-circuit. diff --git a/processing/src/test/java/io/druid/query/TestQueryRunners.java b/processing/src/test/java/io/druid/query/TestQueryRunners.java index 87fe81fa35df..e4daec6ea846 100644 --- a/processing/src/test/java/io/druid/query/TestQueryRunners.java +++ b/processing/src/test/java/io/druid/query/TestQueryRunners.java @@ -65,8 +65,10 @@ public static QueryRunner makeTopNQueryRunner( { QueryRunnerFactory factory = new TopNQueryRunnerFactory( pool, - new TopNQueryQueryToolChest(topNConfig, - QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator()), + new TopNQueryQueryToolChest( + topNConfig, + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ), QueryRunnerTestHelper.NOOP_QUERYWATCHER ); return new FinalizeResultsQueryRunner( @@ -100,9 +102,11 @@ public static QueryRunner makeSearchQueryRunner( QueryRunnerFactory factory = new SearchQueryRunnerFactory( new SearchStrategySelector(Suppliers.ofInstance(config)), new SearchQueryQueryToolChest( - config, - QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator()), - QueryRunnerTestHelper.NOOP_QUERYWATCHER); + config, + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); return new FinalizeResultsQueryRunner( factory.createRunner(adapter), factory.getToolchest() diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 1f8aa7b8df97..3f3a37703255 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -72,7 +72,7 @@ public class SearchQueryRunnerTest ); private static final SearchStrategySelector selector = new SearchStrategySelector(Suppliers.ofInstance(config)); - @Parameterized.Parameters(name="{0}") + @Parameterized.Parameters(name = "{0}") public static Iterable constructorFeeder() throws IOException { return QueryRunnerTestHelper.transformToConstructionFeeder( @@ -95,7 +95,7 @@ public SearchQueryRunnerTest( { this.runner = runner; this.decoratedRunner = toolChest.postMergeQueryDecoration( - toolChest.mergeResults(toolChest.preMergeQueryDecoration(runner))); + toolChest.mergeResults(toolChest.preMergeQueryDecoration(runner))); } @Test @@ -393,7 +393,8 @@ public void testSearchWithSingleFilter1() new AndDimFilter( Arrays.asList( new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "total_market", null), - new SelectorDimFilter(QueryRunnerTestHelper.qualityDimension, "mezzanine", null)))) + new SelectorDimFilter(QueryRunnerTestHelper.qualityDimension, "mezzanine", null) + ))) .intervals(QueryRunnerTestHelper.fullOnInterval) .dimensions(QueryRunnerTestHelper.qualityDimension) .query("a") diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java index 2bb3155f644d..39deba317741 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java @@ -92,38 +92,54 @@ public static Iterable constructorFeeder() throws IOException return transformToConstructionFeeder( Arrays.asList( - makeQueryRunner(makeRunnerFactory(configs[0]), - "index1", - new IncrementalIndexSegment(index1, "index1"), - "index1"), - makeQueryRunner(makeRunnerFactory(configs[0]), - "index2", - new IncrementalIndexSegment(index2, "index2"), - "index2"), - makeQueryRunner(makeRunnerFactory(configs[0]), - "index3", - new QueryableIndexSegment("index3", index3), - "index3"), - makeQueryRunner(makeRunnerFactory(configs[0]), - "index4", - new QueryableIndexSegment("index4", index4), - "index4"), - makeQueryRunner(makeRunnerFactory(configs[1]), - "index1", - new IncrementalIndexSegment(index1, "index1"), - "index1"), - makeQueryRunner(makeRunnerFactory(configs[1]), - "index2", - new IncrementalIndexSegment(index2, "index2"), - "index2"), - makeQueryRunner(makeRunnerFactory(configs[1]), - "index3", - new QueryableIndexSegment("index3", index3), - "index3"), - makeQueryRunner(makeRunnerFactory(configs[1]), - "index4", - new QueryableIndexSegment("index4", index4), - "index4") + makeQueryRunner( + makeRunnerFactory(configs[0]), + "index1", + new IncrementalIndexSegment(index1, "index1"), + "index1" + ), + makeQueryRunner( + makeRunnerFactory(configs[0]), + "index2", + new IncrementalIndexSegment(index2, "index2"), + "index2" + ), + makeQueryRunner( + makeRunnerFactory(configs[0]), + "index3", + new QueryableIndexSegment("index3", index3), + "index3" + ), + makeQueryRunner( + makeRunnerFactory(configs[0]), + "index4", + new QueryableIndexSegment("index4", index4), + "index4" + ), + makeQueryRunner( + makeRunnerFactory(configs[1]), + "index1", + new IncrementalIndexSegment(index1, "index1"), + "index1" + ), + makeQueryRunner( + makeRunnerFactory(configs[1]), + "index2", + new IncrementalIndexSegment(index2, "index2"), + "index2" + ), + makeQueryRunner( + makeRunnerFactory(configs[1]), + "index3", + new QueryableIndexSegment("index3", index3), + "index3" + ), + makeQueryRunner( + makeRunnerFactory(configs[1]), + "index4", + new QueryableIndexSegment("index4", index4), + "index4" + ) ) ); } From 5eaff0050c5a74c7eab432848e56d41e3ee4dc14 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 11 Jan 2017 09:11:56 +0900 Subject: [PATCH 9/9] apply comments --- docs/content/querying/searchquery.md | 19 ++-- .../search/search/UseIndexesStrategy.java | 2 +- .../search/SearchQueryRunnerWithCaseTest.java | 90 ++++++++----------- 3 files changed, 47 insertions(+), 64 deletions(-) diff --git a/docs/content/querying/searchquery.md b/docs/content/querying/searchquery.md index 717b07e82872..b86cd2a76a27 100644 --- a/docs/content/querying/searchquery.md +++ b/docs/content/querying/searchquery.md @@ -82,24 +82,25 @@ The format of the result is: #### Strategies -Search queries can be executed using two different strategies. The default strategy is determeined by the -"druid.query.search.searchStrategy" runtime property on the broker. This can be overriden using "searchStrategy" in the +Search queries can be executed using two different strategies. The default strategy is determined by the +"druid.query.search.searchStrategy" runtime property on the broker. This can be overridden using "searchStrategy" in the query context. If neither the context field nor the property is set, the "useIndexes" strategy will be used. - "useIndexes" strategy, the default, first categorizes search dimensions into two groups according to their support for bitmap indexes. And then, it applies index-only and cursor-based execution plans to the group of dimensions supporting -bitmaps and others, respectively. The index-only plan uses only indexes for search query processing. For each dimension, it reads the bitmap index for -each dimension value, evaluates the search predicate, and finally checks the time interval and filter predicates. For the -cursor-based execution plan, please refer to the "cursorOnly" strategy. The index-only plan shows low performance for -the search dimensions of large cardinality which means most values of search dimensions are unique. +bitmaps and others, respectively. The index-only plan uses only indexes for search query processing. For each dimension, +it reads the bitmap index for each dimension value, evaluates the search predicate, and finally checks the time interval +and filter predicates. For the cursor-based execution plan, please refer to the "cursorOnly" strategy. The index-only +plan shows low performance for the search dimensions of large cardinality which means most values of search dimensions +are unique. - "cursorOnly" strategy generates a cursor-based execution plan. This plan creates a cursor which reads a row from a queryableIndexSegment, and then evaluates search predicates. If some filters support bitmap indexes, the cursor can read only the rows which satisfy those filters, thereby saving I/O cost. However, it might be slow with filters of low selectivity. - "auto" strategy uses a cost-based planner for choosing an optimal search strategy. It estimates the cost of index-only -and cursor-based execution plans, and chooses the optimal one. Currently, its performance is suboptimal due to the large -overhead of cost estimation. +and cursor-based execution plans, and chooses the optimal one. Currently, it is not enabled by default due to the overhead +of cost estimation. #### Server configuration @@ -111,7 +112,7 @@ The following runtime properties apply: #### Query context -The following runtime properties apply: +The following query context parameters apply: |Property|Description| |--------|-----------| diff --git a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java index 6283f8bb198d..fcf6f23016ec 100644 --- a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java @@ -99,7 +99,7 @@ public List getExecutionPlan(SearchQuery query, Segment seg index ); - // Index-only plan is used only when any filter is not specified or every filter supports bitmap indexes. + // Index-only plan is used only when any filter is not specified or the filter supports bitmap indexes. // // Note: if some filters support bitmap indexes but others are not, the current implementation always employs // the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java index 39deba317741..c46e7ce57708 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java @@ -28,11 +28,12 @@ import io.druid.query.Druids; import io.druid.query.QueryRunner; import io.druid.query.Result; +import io.druid.query.search.search.AutoStrategy; import io.druid.query.search.search.CursorOnlyStrategy; -import io.druid.query.search.search.UseIndexesStrategy; import io.druid.query.search.search.SearchHit; import io.druid.query.search.search.SearchQuery; import io.druid.query.search.search.SearchQueryConfig; +import io.druid.query.search.search.UseIndexesStrategy; import io.druid.segment.IncrementalIndexSegment; import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexSegment; @@ -71,11 +72,13 @@ public class SearchQueryRunnerWithCaseTest @Parameterized.Parameters public static Iterable constructorFeeder() throws IOException { - final SearchQueryConfig[] configs = new SearchQueryConfig[2]; + final SearchQueryConfig[] configs = new SearchQueryConfig[3]; configs[0] = new SearchQueryConfig(); configs[0].setSearchStrategy(UseIndexesStrategy.NAME); configs[1] = new SearchQueryConfig(); configs[1].setSearchStrategy(CursorOnlyStrategy.NAME); + configs[2] = new SearchQueryConfig(); + configs[2].setSearchStrategy(AutoStrategy.NAME); CharSource input = CharSource.wrap( "2011-01-12T00:00:00.000Z\tspot\tAutoMotive\tPREFERRED\ta\u0001preferred\t100.000000\n" + @@ -90,58 +93,37 @@ public static Iterable constructorFeeder() throws IOException QueryableIndex index3 = TestIndex.persistRealtimeAndLoadMMapped(index1); QueryableIndex index4 = TestIndex.persistRealtimeAndLoadMMapped(index2); - return transformToConstructionFeeder( - Arrays.asList( - makeQueryRunner( - makeRunnerFactory(configs[0]), - "index1", - new IncrementalIndexSegment(index1, "index1"), - "index1" - ), - makeQueryRunner( - makeRunnerFactory(configs[0]), - "index2", - new IncrementalIndexSegment(index2, "index2"), - "index2" - ), - makeQueryRunner( - makeRunnerFactory(configs[0]), - "index3", - new QueryableIndexSegment("index3", index3), - "index3" - ), - makeQueryRunner( - makeRunnerFactory(configs[0]), - "index4", - new QueryableIndexSegment("index4", index4), - "index4" - ), - makeQueryRunner( - makeRunnerFactory(configs[1]), - "index1", - new IncrementalIndexSegment(index1, "index1"), - "index1" - ), - makeQueryRunner( - makeRunnerFactory(configs[1]), - "index2", - new IncrementalIndexSegment(index2, "index2"), - "index2" - ), - makeQueryRunner( - makeRunnerFactory(configs[1]), - "index3", - new QueryableIndexSegment("index3", index3), - "index3" - ), - makeQueryRunner( - makeRunnerFactory(configs[1]), - "index4", - new QueryableIndexSegment("index4", index4), - "index4" - ) - ) - ); + final List>> runners = Lists.newArrayList(); + for (int i = 0; i < configs.length; i++) { + runners.addAll(Arrays.asList( + makeQueryRunner( + makeRunnerFactory(configs[i]), + "index1", + new IncrementalIndexSegment(index1, "index1"), + "index1" + ), + makeQueryRunner( + makeRunnerFactory(configs[i]), + "index2", + new IncrementalIndexSegment(index2, "index2"), + "index2" + ), + makeQueryRunner( + makeRunnerFactory(configs[i]), + "index3", + new QueryableIndexSegment("index3", index3), + "index3" + ), + makeQueryRunner( + makeRunnerFactory(configs[i]), + "index4", + new QueryableIndexSegment("index4", index4), + "index4" + ) + )); + } + + return transformToConstructionFeeder(runners); } static SearchQueryRunnerFactory makeRunnerFactory(final SearchQueryConfig config)