diff --git a/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java b/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java index bd33d7f97839..32895eeaf88b 100644 --- a/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java +++ b/processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java @@ -25,7 +25,7 @@ import org.apache.druid.collections.bitmap.BitmapFactory; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.bitmap.MutableBitmap; -import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.NonnullPair; import org.apache.druid.query.DefaultBitmapResultFactory; import org.apache.druid.query.Order; import org.apache.druid.query.dimension.DimensionSpec; @@ -35,23 +35,21 @@ import org.apache.druid.query.filter.Filter; import org.apache.druid.query.search.CursorOnlyStrategy.CursorBasedExecutor; import org.apache.druid.segment.ColumnSelectorColumnIndexSelector; -import org.apache.druid.segment.CursorFactory; import org.apache.druid.segment.Cursors; import org.apache.druid.segment.DeprecatedQueryableIndexColumnSelector; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; import org.apache.druid.segment.VirtualColumns; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.NumericColumn; import org.apache.druid.segment.index.BitmapColumnIndex; import org.apache.druid.segment.index.semantic.DictionaryEncodedStringValueIndex; -import org.apache.druid.segment.virtual.VirtualizedColumnInspector; import org.joda.time.Interval; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; public class UseIndexesStrategy extends SearchStrategy @@ -74,67 +72,64 @@ private UseIndexesStrategy( public List getExecutionPlan(SearchQuery query, Segment segment) { final ImmutableList.Builder builder = ImmutableList.builder(); - final QueryableIndex index = segment.as(QueryableIndex.class); - final CursorFactory cursorFactory = segment.asCursorFactory(); final List searchDims = getDimsToSearch(segment, query.getDimensions()); - if (index != null) { - // pair of bitmap dims and non-bitmap dims - final Pair, List> pair = partitionDimensionList( - segment, - cursorFactory, - query.getVirtualColumns(), - searchDims - ); - final List bitmapSuppDims = pair.lhs; - final List nonBitmapSuppDims = pair.rhs; + final QueryableIndex index = segment.as(QueryableIndex.class); + if (index == null) { + return Collections.singletonList(new CursorBasedExecutor(query, segment, searchDims)); + } - if (bitmapSuppDims.size() > 0) { - final ColumnIndexSelector selector = new ColumnSelectorColumnIndexSelector( - index.getBitmapFactoryForDimensions(), + final ColumnIndexSelector selector = new ColumnSelectorColumnIndexSelector( + index.getBitmapFactoryForDimensions(), + query.getVirtualColumns(), + new DeprecatedQueryableIndexColumnSelector(index) + ); + + // pair of bitmap dims and non-bitmap dims + final NonnullPair, List> pair = partitionDimensionList( + segment, + selector, + searchDims + ); + final List bitmapSuppDims = pair.lhs; + final List nonBitmapSuppDims = pair.rhs; + + if (!bitmapSuppDims.isEmpty()) { + // Index-only plan is used only when any filter is not specified or the filter supports bitmap indexes. + // + // Note: if some filters support bitmap indexes but others are not, the current implementation always employs + // the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index + // from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps. + if ((filter == null || filter.getBitmapColumnIndex(selector) != null) + && Cursors.getTimeOrdering(index.getOrdering()) == Order.ASCENDING) { + final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap( + index, + segment, query.getVirtualColumns(), - new DeprecatedQueryableIndexColumnSelector(index) + filter, + interval ); - - // Index-only plan is used only when any filter is not specified or the filter supports bitmap indexes. - // - // Note: if some filters support bitmap indexes but others are not, the current implementation always employs - // the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index - // from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps. - if ((filter == null || filter.getBitmapColumnIndex(selector) != null) - && Cursors.getTimeOrdering(index.getOrdering()) == Order.ASCENDING) { - final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap( - index, - segment, - query.getVirtualColumns(), - filter, - interval - ); - builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims)); - } else { - // Fall back to cursor-based execution strategy - nonBitmapSuppDims.addAll(bitmapSuppDims); - } + builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims)); + } else { + // Fall back to cursor-based execution strategy + nonBitmapSuppDims.addAll(bitmapSuppDims); } + } - if (nonBitmapSuppDims.size() > 0) { - builder.add(new CursorBasedExecutor(query, segment, nonBitmapSuppDims)); - } - } else { - builder.add(new CursorBasedExecutor(query, segment, searchDims)); + if (!nonBitmapSuppDims.isEmpty()) { + builder.add(new CursorBasedExecutor(query, segment, nonBitmapSuppDims)); } return builder.build(); } /** - * Split the given dimensions list into bitmap-supporting dimensions and non-bitmap supporting ones. - * Note that the returned lists are free to modify. + * Split the given dimensions list into columns which provide {@link DictionaryEncodedStringValueIndex} and those + * which do not. Note that the returned lists are free to modify. */ - private static Pair, List> partitionDimensionList( + private static NonnullPair, List> partitionDimensionList( Segment segment, - CursorFactory cursorFactory, - VirtualColumns virtualColumns, + ColumnIndexSelector columnIndexSelector, List dimensions ) { @@ -144,22 +139,21 @@ private static Pair, List> partitionDimension segment, dimensions ); - VirtualizedColumnInspector columnInspector = new VirtualizedColumnInspector(cursorFactory, virtualColumns); - for (DimensionSpec spec : dimsToSearch) { - ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(spec.getDimension()); - if (capabilities == null) { + ColumnIndexSupplier indexSupplier = columnIndexSelector.getIndexSupplier(spec.getDimension()); + if (indexSupplier == null) { + // column doesn't exist, ignore it continue; } - if (capabilities.hasBitmapIndexes()) { + if (indexSupplier.as(DictionaryEncodedStringValueIndex.class) != null) { bitmapDims.add(spec); } else { nonBitmapDims.add(spec); } } - return new Pair<>(bitmapDims, nonBitmapDims); + return new NonnullPair<>(bitmapDims, nonBitmapDims); } static ImmutableBitmap makeTimeFilteredBitmap( @@ -304,6 +298,7 @@ public Object2IntRBTreeMap execute(int limit) } } } else { + // these were checked to be non-null in partitionDimensionList final DictionaryEncodedStringValueIndex bitmapIndex = indexSupplier.as(DictionaryEncodedStringValueIndex.class); for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { diff --git a/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java index fa8fb02ffcb5..1b0012560bb6 100644 --- a/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java @@ -40,6 +40,7 @@ import org.apache.druid.query.context.ResponseContext; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.dimension.ExtractionDimensionSpec; +import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.extraction.JavaScriptExtractionFn; import org.apache.druid.query.extraction.MapLookupExtractor; @@ -60,6 +61,7 @@ import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; +import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.virtual.ListFilteredVirtualColumn; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.SegmentId; @@ -827,6 +829,33 @@ public void testSearchSameValueInMultiDimsVirtualColumns() checkSearchQuery(searchQuery, expectedHits); } + @Test + public void testSearchVirtualColumns() + { + List expectedHits = new ArrayList<>(); + expectedHits.add(new SearchHit("vquality", "a", 93)); + expectedHits.add(new SearchHit("vquality", "not a", 1116)); + + checkSearchQuery( + Druids.newSearchQueryBuilder() + .dataSource(QueryRunnerTestHelper.DATA_SOURCE) + .granularity(QueryRunnerTestHelper.ALL_GRAN) + .virtualColumns( + new ExpressionVirtualColumn( + "vquality", + "case_searched(like(quality,'a%'), 'a', 'not a')", + ColumnType.STRING, + TestExprMacroTable.INSTANCE + ) + ) + .dimensions("vquality") + .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC) + .query("") + .build(), + expectedHits + ); + } + private void checkSearchQuery(Query searchQuery, List expectedResults) { checkSearchQuery(searchQuery, runner, expectedResults);