Skip to content

[Backport] fix search query to check for the specific type of indexes it requires instead of using capabilities #17672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adarshsanjeev merged 1 commit into from
Jan 28, 2025
Merged

[Backport] fix search query to check for the specific type of indexes it requires instead of using capabilities #17672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 51 additions & 56 deletions processing/src/main/java/org/apache/druid/query/search/UseIndexesStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.collections.bitmap.MutableBitmap;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.NonnullPair;
import org.apache.druid.query.DefaultBitmapResultFactory;
import org.apache.druid.query.Order;
import org.apache.druid.query.dimension.DimensionSpec;
Expand All @@ -35,23 +35,21 @@
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.search.CursorOnlyStrategy.CursorBasedExecutor;
import org.apache.druid.segment.ColumnSelectorColumnIndexSelector;
import org.apache.druid.segment.CursorFactory;
import org.apache.druid.segment.Cursors;
import org.apache.druid.segment.DeprecatedQueryableIndexColumnSelector;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.Segment;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.NumericColumn;
import org.apache.druid.segment.index.BitmapColumnIndex;
import org.apache.druid.segment.index.semantic.DictionaryEncodedStringValueIndex;
import org.apache.druid.segment.virtual.VirtualizedColumnInspector;
import org.joda.time.Interval;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class UseIndexesStrategy extends SearchStrategy
Expand All @@ -74,67 +72,64 @@ private UseIndexesStrategy(
public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment segment)
{
final ImmutableList.Builder<SearchQueryExecutor> builder = ImmutableList.builder();
final QueryableIndex index = segment.as(QueryableIndex.class);
final CursorFactory cursorFactory = segment.asCursorFactory();
final List<DimensionSpec> searchDims = getDimsToSearch(segment, query.getDimensions());

if (index != null) {
// pair of bitmap dims and non-bitmap dims
final Pair<List<DimensionSpec>, List<DimensionSpec>> pair = partitionDimensionList(
segment,
cursorFactory,
query.getVirtualColumns(),
searchDims
);
final List<DimensionSpec> bitmapSuppDims = pair.lhs;
final List<DimensionSpec> nonBitmapSuppDims = pair.rhs;
final QueryableIndex index = segment.as(QueryableIndex.class);
if (index == null) {
return Collections.singletonList(new CursorBasedExecutor(query, segment, searchDims));
}

if (bitmapSuppDims.size() > 0) {
final ColumnIndexSelector selector = new ColumnSelectorColumnIndexSelector(
index.getBitmapFactoryForDimensions(),
final ColumnIndexSelector selector = new ColumnSelectorColumnIndexSelector(
index.getBitmapFactoryForDimensions(),
query.getVirtualColumns(),
new DeprecatedQueryableIndexColumnSelector(index)
);

// pair of bitmap dims and non-bitmap dims
final NonnullPair<List<DimensionSpec>, List<DimensionSpec>> pair = partitionDimensionList(
segment,
selector,
searchDims
);
final List<DimensionSpec> bitmapSuppDims = pair.lhs;
final List<DimensionSpec> nonBitmapSuppDims = pair.rhs;

if (!bitmapSuppDims.isEmpty()) {
// Index-only plan is used only when any filter is not specified or the filter supports bitmap indexes.
//
// Note: if some filters support bitmap indexes but others are not, the current implementation always employs
// the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index
// from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps.
if ((filter == null || filter.getBitmapColumnIndex(selector) != null)
&& Cursors.getTimeOrdering(index.getOrdering()) == Order.ASCENDING) {
final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(
index,
segment,
query.getVirtualColumns(),
new DeprecatedQueryableIndexColumnSelector(index)
filter,
interval
);

// Index-only plan is used only when any filter is not specified or the filter supports bitmap indexes.
//
// Note: if some filters support bitmap indexes but others are not, the current implementation always employs
// the cursor-based plan. This can be more optimized. One possible optimization is generating a bitmap index
// from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps.
if ((filter == null || filter.getBitmapColumnIndex(selector) != null)
&& Cursors.getTimeOrdering(index.getOrdering()) == Order.ASCENDING) {
final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(
index,
segment,
query.getVirtualColumns(),
filter,
interval
);
builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims));
} else {
// Fall back to cursor-based execution strategy
nonBitmapSuppDims.addAll(bitmapSuppDims);
}
builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims));
} else {
// Fall back to cursor-based execution strategy
nonBitmapSuppDims.addAll(bitmapSuppDims);
}
}

if (nonBitmapSuppDims.size() > 0) {
builder.add(new CursorBasedExecutor(query, segment, nonBitmapSuppDims));
}
} else {
builder.add(new CursorBasedExecutor(query, segment, searchDims));
if (!nonBitmapSuppDims.isEmpty()) {
builder.add(new CursorBasedExecutor(query, segment, nonBitmapSuppDims));
}

return builder.build();
}

/**
* Split the given dimensions list into bitmap-supporting dimensions and non-bitmap supporting ones.
* Note that the returned lists are free to modify.
* Split the given dimensions list into columns which provide {@link DictionaryEncodedStringValueIndex} and those
* which do not. Note that the returned lists are free to modify.
*/
private static Pair<List<DimensionSpec>, List<DimensionSpec>> partitionDimensionList(
private static NonnullPair<List<DimensionSpec>, List<DimensionSpec>> partitionDimensionList(
Segment segment,
CursorFactory cursorFactory,
VirtualColumns virtualColumns,
ColumnIndexSelector columnIndexSelector,
List<DimensionSpec> dimensions
)
{
Expand All @@ -144,22 +139,21 @@ private static Pair<List<DimensionSpec>, List<DimensionSpec>> partitionDimension
segment,
dimensions
);
VirtualizedColumnInspector columnInspector = new VirtualizedColumnInspector(cursorFactory, virtualColumns);

for (DimensionSpec spec : dimsToSearch) {
ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(spec.getDimension());
if (capabilities == null) {
ColumnIndexSupplier indexSupplier = columnIndexSelector.getIndexSupplier(spec.getDimension());
if (indexSupplier == null) {
// column doesn't exist, ignore it
continue;
}

if (capabilities.hasBitmapIndexes()) {
if (indexSupplier.as(DictionaryEncodedStringValueIndex.class) != null) {
bitmapDims.add(spec);
} else {
nonBitmapDims.add(spec);
}
}

return new Pair<>(bitmapDims, nonBitmapDims);
return new NonnullPair<>(bitmapDims, nonBitmapDims);
}

static ImmutableBitmap makeTimeFilteredBitmap(
Expand Down Expand Up @@ -304,6 +298,7 @@ public Object2IntRBTreeMap<SearchHit> execute(int limit)
}
}
} else {
// these were checked to be non-null in partitionDimensionList
final DictionaryEncodedStringValueIndex bitmapIndex =
indexSupplier.as(DictionaryEncodedStringValueIndex.class);
for (int i = 0; i < bitmapIndex.getCardinality(); ++i) {
Expand Down
29 changes: 29 additions & 0 deletions processing/src/test/java/org/apache/druid/query/search/SearchQueryRunnerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.apache.druid.query.context.ResponseContext;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.dimension.ExtractionDimensionSpec;
import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.extraction.JavaScriptExtractionFn;
import org.apache.druid.query.extraction.MapLookupExtractor;
Expand All @@ -60,6 +61,7 @@
import org.apache.druid.segment.incremental.IncrementalIndex;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.incremental.OnheapIncrementalIndex;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.virtual.ListFilteredVirtualColumn;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.apache.druid.timeline.SegmentId;
Expand Down Expand Up @@ -827,6 +829,33 @@ public void testSearchSameValueInMultiDimsVirtualColumns()
checkSearchQuery(searchQuery, expectedHits);
}

@Test
public void testSearchVirtualColumns()
{
List<SearchHit> expectedHits = new ArrayList<>();
expectedHits.add(new SearchHit("vquality", "a", 93));
expectedHits.add(new SearchHit("vquality", "not a", 1116));

checkSearchQuery(
Druids.newSearchQueryBuilder()
.dataSource(QueryRunnerTestHelper.DATA_SOURCE)
.granularity(QueryRunnerTestHelper.ALL_GRAN)
.virtualColumns(
new ExpressionVirtualColumn(
"vquality",
"case_searched(like(quality,'a%'), 'a', 'not a')",
ColumnType.STRING,
TestExprMacroTable.INSTANCE
)
)
.dimensions("vquality")
.intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
.query("")
.build(),
expectedHits
);
}

private void checkSearchQuery(Query searchQuery, List<SearchHit> expectedResults)
{
checkSearchQuery(searchQuery, runner, expectedResults);
Expand Down