diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java index 1abe6a1a9ae6..75f70ae3d5bf 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java @@ -71,8 +71,8 @@ public class MomentSketchAggregatorFactory extends AggregatorFactory public MomentSketchAggregatorFactory( @JsonProperty("name") final String name, @JsonProperty("fieldName") final String fieldName, - @Nullable @JsonProperty("k") final Integer k, - @Nullable @JsonProperty("compress") final Boolean compress + @JsonProperty("k") @Nullable final Integer k, + @JsonProperty("compress") @Nullable final Boolean compress ) { this(name, fieldName, k, compress, AggregatorUtil.MOMENTS_SKETCH_BUILD_CACHE_TYPE_ID); diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java index b9d34f377d7e..74caf612e92c 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java @@ -76,7 +76,7 @@ public class TDigestSketchAggregatorFactory extends AggregatorFactory public TDigestSketchAggregatorFactory( @JsonProperty("name") final String name, @JsonProperty("fieldName") final String fieldName, - @Nullable @JsonProperty("compression") final Integer compression + @JsonProperty("compression") @Nullable final Integer compression ) { this(name, fieldName, compression, AggregatorUtil.TDIGEST_BUILD_SKETCH_CACHE_TYPE_ID); diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java index 2cc5388a5382..4d9af8c7cbdf 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java @@ -39,10 +39,10 @@ public class SketchMergeAggregatorFactory extends SketchAggregatorFactory public SketchMergeAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("fieldName") String fieldName, - @Nullable @JsonProperty("size") Integer size, - @Nullable @JsonProperty("shouldFinalize") Boolean shouldFinalize, - @Nullable @JsonProperty("isInputThetaSketch") Boolean isInputThetaSketch, - @Nullable @JsonProperty("errorBoundsStdDev") Integer errorBoundsStdDev + @JsonProperty("size") @Nullable Integer size, + @JsonProperty("shouldFinalize") @Nullable Boolean shouldFinalize, + @JsonProperty("isInputThetaSketch") @Nullable Boolean isInputThetaSketch, + @JsonProperty("errorBoundsStdDev") @Nullable Integer errorBoundsStdDev ) { super(name, fieldName, size, AggregatorUtil.SKETCH_MERGE_CACHE_TYPE_ID); diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java index 53839e82f7ad..d158be45a207 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java @@ -71,7 +71,7 @@ public class BloomFilterAggregatorFactory extends AggregatorFactory public BloomFilterAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("field") final DimensionSpec field, - @Nullable @JsonProperty("maxNumEntries") Integer maxNumEntries + @JsonProperty("maxNumEntries") @Nullable Integer maxNumEntries ) { this.name = name; diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java index f235b59c30de..691826931459 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java @@ -20,18 +20,21 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.hash.HashCode; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.segment.filter.DimensionPredicateFilter; -import java.util.HashSet; +import javax.annotation.Nullable; +import java.util.Objects; +import java.util.Set; /** */ @@ -41,13 +44,17 @@ public class BloomDimFilter implements DimFilter private final String dimension; private final BloomKFilter bloomKFilter; private final HashCode hash; + @Nullable private final ExtractionFn extractionFn; + @Nullable + private final FilterTuning filterTuning; @JsonCreator public BloomDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("bloomKFilter") BloomKFilterHolder bloomKFilterHolder, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -56,6 +63,13 @@ public BloomDimFilter( this.bloomKFilter = bloomKFilterHolder.getFilter(); this.hash = bloomKFilterHolder.getFilterHash(); this.extractionFn = extractionFn; + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public BloomDimFilter(String dimension, BloomKFilterHolder bloomKFilterHolder, @Nullable ExtractionFn extractionFn) + { + this(dimension, bloomKFilterHolder, extractionFn, null); } @Override @@ -152,7 +166,8 @@ public boolean applyNull() }; } }, - extractionFn + extractionFn, + filterTuning ); } @@ -168,20 +183,40 @@ public BloomKFilter getBloomKFilter() return bloomKFilter; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public String toString() { - if (extractionFn != null) { - return StringUtils.format("%s(%s) = %s", extractionFn, dimension, hash.toString()); - } else { - return StringUtils.format("%s = %s", dimension, hash.toString()); - } + return new DimFilterToStringBuilder().appendDimension(dimension, extractionFn) + .appendEquals(hash.toString()) + .appendFilterTuning(filterTuning) + .build(); } @Override @@ -193,36 +228,16 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - BloomDimFilter that = (BloomDimFilter) o; - - if (!dimension.equals(that.dimension)) { - return false; - } - if (hash != null ? !hash.equals(that.hash) : that.hash != null) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - } - - @Override - public RangeSet getDimensionRangeSet(String dimension) - { - return null; - } - - @Override - public HashSet getRequiredColumns() - { - return Sets.newHashSet(dimension); + return dimension.equals(that.dimension) && + hash.equals(that.hash) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = dimension.hashCode(); - result = 31 * result + (hash != null ? hash.hashCode() : 0); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; + return Objects.hash(dimension, hash, extractionFn, filterTuning); } } diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java index 60497ff45ec3..64ec379a2e7c 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java @@ -100,7 +100,8 @@ public DimFilter toDruidFilter( return new BloomDimFilter( druidExpression.getSimpleExtraction().getColumn(), holder, - druidExpression.getSimpleExtraction().getExtractionFn() + druidExpression.getSimpleExtraction().getExtractionFn(), + null ); } else if (virtualColumnRegistry != null) { VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( @@ -114,6 +115,7 @@ public DimFilter toDruidFilter( return new BloomDimFilter( virtualColumn.getOutputName(), holder, + null, null ); } else { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java index 793a7fd2073a..bc1295466764 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java @@ -65,7 +65,7 @@ public ApproximateHistogramAggregatorFactory( @JsonProperty("numBuckets") Integer numBuckets, @JsonProperty("lowerLimit") Float lowerLimit, @JsonProperty("upperLimit") Float upperLimit, - @Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary + @JsonProperty("finalizeAsBase64Binary") @Nullable Boolean finalizeAsBase64Binary ) { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java index 7f29cdddb064..becbfe1ae64f 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java @@ -46,7 +46,7 @@ public ApproximateHistogramFoldingAggregatorFactory( @JsonProperty("numBuckets") Integer numBuckets, @JsonProperty("lowerLimit") Float lowerLimit, @JsonProperty("upperLimit") Float upperLimit, - @Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary + @JsonProperty("finalizeAsBase64Binary") @Nullable Boolean finalizeAsBase64Binary ) { super(name, fieldName, resolution, numBuckets, lowerLimit, upperLimit, finalizeAsBase64Binary); diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java index 5c90678db793..b8df5b62e0cb 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java @@ -59,11 +59,11 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory public FixedBucketsHistogramAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("fieldName") String fieldName, - @Nullable @JsonProperty("numBuckets") Integer numBuckets, + @JsonProperty("numBuckets") @Nullable Integer numBuckets, @JsonProperty("lowerLimit") double lowerLimit, @JsonProperty("upperLimit") double upperLimit, @JsonProperty("outlierHandlingMode") FixedBucketsHistogram.OutlierHandlingMode outlierHandlingMode, - @Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary + @JsonProperty("finalizeAsBase64Binary") @Nullable Boolean finalizeAsBase64Binary ) { this.name = name; diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespace.java index 314cd72c1257..ce4eddf526b5 100644 --- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespace.java @@ -59,9 +59,9 @@ public JdbcExtractionNamespace( @NotNull @JsonProperty(value = "table", required = true) final String table, @NotNull @JsonProperty(value = "keyColumn", required = true) final String keyColumn, @NotNull @JsonProperty(value = "valueColumn", required = true) final String valueColumn, - @Nullable @JsonProperty(value = "tsColumn", required = false) final String tsColumn, - @Nullable @JsonProperty(value = "filter", required = false) final String filter, - @Min(0) @Nullable @JsonProperty(value = "pollPeriod", required = false) final Period pollPeriod + @JsonProperty(value = "tsColumn", required = false) @Nullable final String tsColumn, + @JsonProperty(value = "filter", required = false) @Nullable final String filter, + @Min(0) @JsonProperty(value = "pollPeriod", required = false) @Nullable final Period pollPeriod ) { this.connectorConfig = Preconditions.checkNotNull(connectorConfig, "connectorConfig"); diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/UriExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/UriExtractionNamespace.java index d828ca7c3cd1..75de73d6c200 100644 --- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/UriExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/UriExtractionNamespace.java @@ -85,7 +85,7 @@ public UriExtractionNamespace( String fileRegex, @JsonProperty(value = "namespaceParseSpec", required = true) FlatDataParser namespaceParseSpec, - @Min(0) @Nullable @JsonProperty(value = "pollPeriod", required = false) + @Min(0) @JsonProperty(value = "pollPeriod", required = false) @Nullable Period pollPeriod, @Deprecated @JsonProperty(value = "versionRegex", required = false) diff --git a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcHadoopInputRowParser.java b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcHadoopInputRowParser.java index 38b65d1cee2a..9fcefcba2a60 100644 --- a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcHadoopInputRowParser.java +++ b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcHadoopInputRowParser.java @@ -44,7 +44,7 @@ public class OrcHadoopInputRowParser implements InputRowParser @JsonCreator public OrcHadoopInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec, - @Nullable @JsonProperty("binaryAsString") Boolean binaryAsString + @JsonProperty("binaryAsString") @Nullable Boolean binaryAsString ) { this.parseSpec = parseSpec; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java index 1f16f01ca56a..8aee62f4fe33 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java @@ -156,15 +156,15 @@ public CompactionTask( @JsonProperty("id") final String id, @JsonProperty("resource") final TaskResource taskResource, @JsonProperty("dataSource") final String dataSource, - @Nullable @JsonProperty("interval") final Interval interval, - @Nullable @JsonProperty("segments") final List segments, - @Nullable @JsonProperty("dimensions") final DimensionsSpec dimensions, - @Nullable @JsonProperty("dimensionsSpec") final DimensionsSpec dimensionsSpec, - @Nullable @JsonProperty("metricsSpec") final AggregatorFactory[] metricsSpec, - @Nullable @JsonProperty("segmentGranularity") final Granularity segmentGranularity, - @Nullable @JsonProperty("targetCompactionSizeBytes") final Long targetCompactionSizeBytes, - @Nullable @JsonProperty("tuningConfig") final IndexTuningConfig tuningConfig, - @Nullable @JsonProperty("context") final Map context, + @JsonProperty("interval") @Nullable final Interval interval, + @JsonProperty("segments") @Nullable final List segments, + @JsonProperty("dimensions") @Nullable final DimensionsSpec dimensions, + @JsonProperty("dimensionsSpec") @Nullable final DimensionsSpec dimensionsSpec, + @JsonProperty("metricsSpec") @Nullable final AggregatorFactory[] metricsSpec, + @JsonProperty("segmentGranularity") @Nullable final Granularity segmentGranularity, + @JsonProperty("targetCompactionSizeBytes") @Nullable final Long targetCompactionSizeBytes, + @JsonProperty("tuningConfig") @Nullable final IndexTuningConfig tuningConfig, + @JsonProperty("context") @Nullable final Map context, @JacksonInject ObjectMapper jsonMapper, @JacksonInject AuthorizerMapper authorizerMapper, @JacksonInject ChatHandlerProvider chatHandlerProvider, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java b/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java index ccbcb50874bb..53969484128c 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java @@ -101,10 +101,10 @@ public class IngestSegmentFirehoseFactory implements FiniteFirehoseFactory segmentIds, + @JsonProperty("segments") @Nullable List segmentIds, @JsonProperty("filter") DimFilter dimFilter, @JsonProperty("dimensions") List dimensions, @JsonProperty("metrics") List metrics, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfo.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfo.java index 11da1f6733ee..2c077581b04e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfo.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfo.java @@ -51,7 +51,7 @@ public ImmutableWorkerInfo( @JsonProperty("availabilityGroups") Set availabilityGroups, @JsonProperty("runningTasks") Collection runningTasks, @JsonProperty("lastCompletedTaskTime") DateTime lastCompletedTaskTime, - @Nullable @JsonProperty("blacklistedUntil") DateTime blacklistedUntil + @JsonProperty("blacklistedUntil") @Nullable DateTime blacklistedUntil ) { this.worker = worker; diff --git a/processing/src/main/java/org/apache/druid/query/Druids.java b/processing/src/main/java/org/apache/druid/query/Druids.java index 2e35891fafc4..42836b242415 100644 --- a/processing/src/main/java/org/apache/druid/query/Druids.java +++ b/processing/src/main/java/org/apache/druid/query/Druids.java @@ -204,7 +204,7 @@ public TimeseriesQueryBuilder filters(String dimensionName, String value) public TimeseriesQueryBuilder filters(String dimensionName, String value, String... values) { - dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null); + dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null, null); return this; } @@ -361,7 +361,7 @@ public SearchQueryBuilder dataSource(DataSource d) public SearchQueryBuilder filters(String dimensionName, String value) { - dimFilter = new SelectorDimFilter(dimensionName, value, null); + dimFilter = new SelectorDimFilter(dimensionName, value, null, null); return this; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java index 47d175569283..bc35dc2c4d0a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java @@ -63,7 +63,7 @@ public FilteredAggregatorFactory( public FilteredAggregatorFactory( @JsonProperty("aggregator") AggregatorFactory delegate, @JsonProperty("filter") DimFilter dimFilter, - @Nullable @JsonProperty("name") String name + @JsonProperty("name") @Nullable String name ) { Preconditions.checkNotNull(delegate, "aggregator"); @@ -243,7 +243,8 @@ public AggregatorFactory optimizeForSegment(PerSegmentQueryOptimizationContext o new IntervalDimFilter( intervalDimFilter.getDimension(), effectiveFilterIntervals, - intervalDimFilter.getExtractionFn() + intervalDimFilter.getExtractionFn(), + intervalDimFilter.getFilterTuning() ), this.name ); diff --git a/processing/src/main/java/org/apache/druid/query/extraction/BucketExtractionFn.java b/processing/src/main/java/org/apache/druid/query/extraction/BucketExtractionFn.java index 96180f04481c..970116e62169 100644 --- a/processing/src/main/java/org/apache/druid/query/extraction/BucketExtractionFn.java +++ b/processing/src/main/java/org/apache/druid/query/extraction/BucketExtractionFn.java @@ -34,8 +34,8 @@ public class BucketExtractionFn implements ExtractionFn @JsonCreator public BucketExtractionFn( - @Nullable @JsonProperty("size") Double size, - @Nullable @JsonProperty("offset") Double offset + @JsonProperty("size") @Nullable Double size, + @JsonProperty("offset") @Nullable Double offset ) { this.size = size == null ? 1 : size; diff --git a/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java index 9903633609b4..682c2470c30d 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java @@ -32,6 +32,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; /** */ @@ -99,7 +100,7 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { HashSet requiredColumns = new HashSet<>(); fields.forEach(field -> requiredColumns.addAll(field.getRequiredColumns())); diff --git a/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java b/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java index b19327f9d17d..90307eb6380a 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java @@ -38,6 +38,7 @@ public interface BitmapIndexSelector boolean hasMultipleValues(String dimension); int getNumRows(); BitmapFactory getBitmapFactory(); + @Nullable BitmapIndex getBitmapIndex(String dimension); @Nullable ImmutableBitmap getBitmapIndex(String dimension, String value); diff --git a/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java index a919487c1fbf..9bbbdb643eb4 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java @@ -19,9 +19,12 @@ package org.apache.druid.query.filter; +import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import java.util.HashSet; import java.util.List; +import java.util.Set; public interface BooleanFilter extends Filter { @@ -48,4 +51,47 @@ ValueMatcher makeMatcher( ColumnSelectorFactory columnSelectorFactory, RowOffsetMatcherFactory rowOffsetMatcherFactory ); + + @Override + default Set getRequiredColumns() + { + Set allColumns = new HashSet<>(); + for (Filter f : getFilters()) { + allColumns.addAll(f.getRequiredColumns()); + } + return allColumns; + } + + @Override + default boolean supportsBitmapIndex(BitmapIndexSelector selector) + { + for (Filter filter : getFilters()) { + if (!filter.supportsBitmapIndex(selector)) { + return false; + } + } + return true; + } + + @Override + default boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + for (Filter f : getFilters()) { + if (!f.shouldUseBitmapIndex(selector)) { + return false; + } + } + return true; + } + + @Override + default boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) + { + for (Filter filter : getFilters()) { + if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) { + return false; + } + } + return true; + } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java index 38ff78a46bdd..8b7491a54975 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java @@ -20,13 +20,15 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.collect.BoundType; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; @@ -41,32 +43,38 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.Objects; +import java.util.Set; public class BoundDimFilter implements DimFilter { private final String dimension; + @Nullable private final String upper; + @Nullable private final String lower; private final boolean lowerStrict; private final boolean upperStrict; + @Nullable private final ExtractionFn extractionFn; private final StringComparator ordering; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; private final Supplier doublePredicateSupplier; + @Nullable + private final FilterTuning filterTuning; @JsonCreator public BoundDimFilter( @JsonProperty("dimension") String dimension, - @JsonProperty("lower") String lower, - @JsonProperty("upper") String upper, - @JsonProperty("lowerStrict") Boolean lowerStrict, - @JsonProperty("upperStrict") Boolean upperStrict, - @Deprecated @JsonProperty("alphaNumeric") Boolean alphaNumeric, - @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("ordering") StringComparator ordering + @JsonProperty("lower") @Nullable String lower, + @JsonProperty("upper") @Nullable String upper, + @JsonProperty("lowerStrict") @Nullable Boolean lowerStrict, + @JsonProperty("upperStrict") @Nullable Boolean upperStrict, + @Deprecated @JsonProperty("alphaNumeric") @Nullable Boolean alphaNumeric, + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("ordering") @Nullable StringComparator ordering, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be null"); @@ -98,6 +106,22 @@ public BoundDimFilter( this.longPredicateSupplier = makeLongPredicateSupplier(); this.floatPredicateSupplier = makeFloatPredicateSupplier(); this.doublePredicateSupplier = makeDoublePredicateSupplier(); + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public BoundDimFilter( + String dimension, + @Nullable String lower, + @Nullable String upper, + @Nullable Boolean lowerStrict, + @Nullable Boolean upperStrict, + @Nullable Boolean alphaNumeric, + @Nullable ExtractionFn extractionFn, + @Nullable StringComparator ordering + ) + { + this(dimension, lower, upper, lowerStrict, upperStrict, alphaNumeric, extractionFn, ordering, null); } @JsonProperty @@ -106,12 +130,14 @@ public String getDimension() return dimension; } + @Nullable @JsonProperty public String getUpper() { return upper; } + @Nullable @JsonProperty public String getLower() { @@ -140,6 +166,7 @@ public boolean hasUpperBound() return upper != null; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { @@ -152,6 +179,14 @@ public StringComparator getOrdering() return ordering; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + public Supplier getLongPredicateSupplier() { return longPredicateSupplier; @@ -249,9 +284,9 @@ && getExtractionFn() == null } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override @@ -263,49 +298,36 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - BoundDimFilter that = (BoundDimFilter) o; - - if (isLowerStrict() != that.isLowerStrict()) { - return false; - } - if (isUpperStrict() != that.isUpperStrict()) { - return false; - } - if (!getDimension().equals(that.getDimension())) { - return false; - } - if (getUpper() != null ? !getUpper().equals(that.getUpper()) : that.getUpper() != null) { - return false; - } - if (getLower() != null ? !getLower().equals(that.getLower()) : that.getLower() != null) { - return false; - } - if (getExtractionFn() != null - ? !getExtractionFn().equals(that.getExtractionFn()) - : that.getExtractionFn() != null) { - return false; - } - return getOrdering().equals(that.getOrdering()); + return lowerStrict == that.lowerStrict && + upperStrict == that.upperStrict && + dimension.equals(that.dimension) && + Objects.equals(upper, that.upper) && + Objects.equals(lower, that.lower) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(ordering, that.ordering) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = getDimension().hashCode(); - result = 31 * result + (getUpper() != null ? getUpper().hashCode() : 0); - result = 31 * result + (getLower() != null ? getLower().hashCode() : 0); - result = 31 * result + (isLowerStrict() ? 1 : 0); - result = 31 * result + (isUpperStrict() ? 1 : 0); - result = 31 * result + (getExtractionFn() != null ? getExtractionFn().hashCode() : 0); - result = 31 * result + getOrdering().hashCode(); - return result; + return Objects.hash( + dimension, + upper, + lower, + lowerStrict, + upperStrict, + extractionFn, + ordering, + filterTuning + ); } @Override public String toString() { - final StringBuilder builder = new StringBuilder(); + final DimFilterToStringBuilder builder = new DimFilterToStringBuilder(); if (lower != null) { builder.append(lower); @@ -316,11 +338,7 @@ public String toString() } } - if (extractionFn != null) { - builder.append(StringUtils.format("%s(%s)", extractionFn, dimension)); - } else { - builder.append(dimension); - } + builder.appendDimension(dimension, extractionFn); if (!ordering.equals(StringComparators.LEXICOGRAPHIC)) { builder.append(StringUtils.format(" as %s", ordering.toString())); @@ -335,7 +353,7 @@ public String toString() builder.append(upper); } - return builder.toString(); + return builder.appendFilterTuning(filterTuning).build(); } private Supplier makeLongPredicateSupplier() diff --git a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java index 1d7cb502210f..dd97ce2808cf 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java @@ -24,13 +24,13 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.filter.ColumnComparisonFilter; -import java.util.HashSet; import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; /** @@ -94,30 +94,25 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - ColumnComparisonDimFilter that = (ColumnComparisonDimFilter) o; - return dimensions.equals(that.dimensions); } @Override - public RangeSet getDimensionRangeSet(String dimension) + public int hashCode() { - return null; + return Objects.hash(dimensions); } @Override - public HashSet getRequiredColumns() + public RangeSet getDimensionRangeSet(String dimension) { - return Sets.newHashSet(dimensions.stream() - .map(DimensionSpec::getDimension) - .collect(Collectors.toSet()) - ); + return null; } @Override - public int hashCode() + public Set getRequiredColumns() { - return 31 * dimensions.hashCode(); + return dimensions.stream().map(DimensionSpec::getDimension).collect(Collectors.toSet()); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java index 81a4c5a41cac..7688d362b3e8 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java @@ -23,8 +23,10 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.collect.RangeSet; import org.apache.druid.java.util.common.Cacheable; +import org.apache.druid.query.extraction.ExtractionFn; -import java.util.HashSet; +import javax.annotation.Nullable; +import java.util.Set; /** */ @@ -81,5 +83,70 @@ public interface DimFilter extends Cacheable /** * @return a HashSet that represents all columns' name which the DimFilter required to do filter. */ - HashSet getRequiredColumns(); + Set getRequiredColumns(); + + /** + * Wrapper for {@link StringBuilder} to re-use common patterns in custom {@link DimFilter#toString()} implementations + */ + class DimFilterToStringBuilder + { + private final StringBuilder builder; + + public DimFilterToStringBuilder() + { + this.builder = new StringBuilder(); + } + + /** + * Append dimension name OR {@link ExtractionFn#toString()} with dimension wrapped in parenthesis + */ + DimFilterToStringBuilder appendDimension(String dimension, @Nullable ExtractionFn extractionFn) + { + if (extractionFn != null) { + builder.append(extractionFn).append("("); + } + + builder.append(dimension); + + if (extractionFn != null) { + builder.append(")"); + } + return this; + } + + /** + * Add "=" expression + */ + DimFilterToStringBuilder appendEquals(String value) + { + builder.append(" = ").append(value); + return this; + } + + /** + * Add filter tuning to {@link #builder} if tuning exists + */ + DimFilterToStringBuilder appendFilterTuning(@Nullable FilterTuning tuning) + { + if (tuning != null) { + builder.append(" (filterTuning=").append(tuning).append(")"); + } + + return this; + } + + /** + * Generic passthrough to {@link StringBuilder#append} + */ + DimFilterToStringBuilder append(T s) + { + builder.append(s); + return this; + } + + public String build() + { + return builder.toString(); + } + } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java index ec4a6d8fcb0c..d902d598192a 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java @@ -21,41 +21,61 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.Parser; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.filter.ExpressionFilter; -import java.util.HashSet; +import javax.annotation.Nullable; import java.util.Objects; +import java.util.Set; public class ExpressionDimFilter implements DimFilter { private final String expression; private final Supplier parsed; + @Nullable + private final FilterTuning filterTuning; @JsonCreator public ExpressionDimFilter( @JsonProperty("expression") final String expression, + @JsonProperty("filterTuning") @Nullable final FilterTuning filterTuning, @JacksonInject ExprMacroTable macroTable ) { this.expression = expression; + this.filterTuning = filterTuning; this.parsed = Suppliers.memoize(() -> Parser.parse(expression, macroTable)); } + @VisibleForTesting + public ExpressionDimFilter(final String expression, ExprMacroTable macroTable) + { + this(expression, null, macroTable); + } + @JsonProperty public String getExpression() { return expression; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public DimFilter optimize() { @@ -65,7 +85,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - return new ExpressionFilter(parsed); + return new ExpressionFilter(parsed, filterTuning); } @Override @@ -75,9 +95,9 @@ public RangeSet getDimensionRangeSet(final String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(parsed.get().analyzeInputs().getRequiredBindings()); + return parsed.get().analyzeInputs().getRequiredBindings(); } @Override @@ -89,7 +109,16 @@ public byte[] getCacheKey() } @Override - public boolean equals(final Object o) + public String toString() + { + return "ExpressionDimFilter{" + + "expression='" + expression + '\'' + + ", filterTuning=" + filterTuning + + '}'; + } + + @Override + public boolean equals(Object o) { if (this == o) { return true; @@ -97,21 +126,14 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final ExpressionDimFilter that = (ExpressionDimFilter) o; - return Objects.equals(expression, that.expression); + ExpressionDimFilter that = (ExpressionDimFilter) o; + return expression.equals(that.expression) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - return Objects.hash(expression); - } - - @Override - public String toString() - { - return "ExpressionDimFilter{" + - "expression='" + expression + '\'' + - '}'; + return Objects.hash(expression, filterTuning); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/ExtractionDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ExtractionDimFilter.java index 4387ce50f393..fe84a9e340eb 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ExtractionDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ExtractionDimFilter.java @@ -22,13 +22,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.extraction.ExtractionFn; import java.nio.ByteBuffer; -import java.util.HashSet; +import java.util.Set; /** * This class is deprecated, use SelectorDimFilter instead: {@link SelectorDimFilter} @@ -113,9 +113,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java index 120358f30447..27a504709dd2 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java @@ -28,6 +28,8 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import java.util.Set; + public interface Filter { /** @@ -100,8 +102,12 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory } /** - * Indicates whether this filter can return a bitmap index for filtering, based on - * the information provided by the input BitmapIndexSelector. + * Indicates whether this filter can return a bitmap index for filtering, based on the information provided by the + * input {@link BitmapIndexSelector}. + * + * Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)} will return a non-null + * {@link BitmapIndexSelector}, and also that all columns specified in {@link #getRequiredColumns()} have a bitmap + * index retrievable via {@link BitmapIndexSelector#getBitmapIndex(String)}. * * @param selector Object used to retrieve bitmap indexes * @@ -109,6 +115,27 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory */ boolean supportsBitmapIndex(BitmapIndexSelector selector); + /** + * Determine if a filter *should* use a bitmap index based on information collected from the supplied + * {@link BitmapIndexSelector}. This method differs from {@link #supportsBitmapIndex(BitmapIndexSelector)} in that + * the former only indicates if a bitmap index is available and {@link #getBitmapIndex(BitmapIndexSelector)} may be + * used. + * + * If shouldUseFilter(selector) returns true, {@link #supportsBitmapIndex} must also return true when called with the + * same selector object. Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)} + * will return a non-null {@link BitmapIndexSelector}, and also that all columns specified in + * {@link #getRequiredColumns()} have a bitmap index retrievable via + * {@link BitmapIndexSelector#getBitmapIndex(String)}. + * + * Implementations of this methods typically consider a {@link FilterTuning} to make decisions about when to + * use an available index. A "standard" implementation of this is available to all {@link Filter} implementations in + * {@link org.apache.druid.segment.filter.Filters#shouldUseBitmapIndex(Filter, BitmapIndexSelector, FilterTuning)}. + * + * @param selector Object used to retrieve bitmap indexes and provide information about the column + * + * @return true if this Filter should provide a bitmap index using the selector, false otherwise. + */ + boolean shouldUseBitmapIndex(BitmapIndexSelector selector); /** * Indicates whether this filter supports selectivity estimation. @@ -129,4 +156,10 @@ default boolean canVectorizeMatcher() { return false; } + + /** + * Set of columns used by a filter. If {@link #supportsBitmapIndex} returns true, all columns returned by this method + * can be expected to have a bitmap index retrievable via {@link BitmapIndexSelector#getBitmapIndex(String)} + */ + Set getRequiredColumns(); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java new file mode 100644 index 000000000000..586c5f1f7fd3 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; +import java.util.Objects; + +/** + * This class provides a mechansim to influence whether or not indexes are used for a {@link Filter} during processing + * by {@link org.apache.druid.segment.QueryableIndexStorageAdapter#analyzeFilter} (i.e. will a {@link Filter} be a "pre" + * filter in which we union indexes for all values that match the filter to create a + * {@link org.apache.druid.segment.BitmapOffset}/{@link org.apache.druid.segment.vector.BitmapVectorOffset}, or will it + * be used as a "post" filter and evaluated while scanning row values from the + * {@link org.apache.druid.segment.FilteredOffset}/{@link org.apache.druid.segment.vector.FilteredVectorOffset}. + * + * This is currently only manually supplied by the user by adding to a {@link DimFilter} which will pass through to the + * {@link Filter} implementation. The main purpose at this time is to facilitate experimentation so that someday we can + * have {@link Filter} implementations intelligently, automatically use sensible defaults based on things like + * cardinality and who yet knows what additional information. + * + * It can also be used for advanced users to manually control which filters will be "pre" and "post" filters as + * described above to allow skipping indexes in known cases where filters are expensive (mostly high cardinality columns + * with expensive filters). + * + * As such, it is currently undocumented in user facing documentation on purpose, but whatever this turns into once more + * automatic usage of this is in place, should be documented in a future release. + */ +public class FilterTuning +{ + public static FilterTuning createDefault(Filter filter, BitmapIndexSelector selector) + { + return new FilterTuning(filter.supportsBitmapIndex(selector), null, null); + } + + private final boolean useBitmapIndex; + private final int minCardinalityToUseBitmapIndex; + private final int maxCardinalityToUseBitmapIndex; + + @JsonCreator + public FilterTuning( + @JsonProperty("useBitmapIndex") @Nullable Boolean useBitmapIndex, + @JsonProperty("minCardinalityToUseBitmapIndex") @Nullable Integer minCardinalityToUseBitmapIndex, + @JsonProperty("useIndexMaximumCardinalityThreshold") @Nullable Integer maxCardinalityToUseBitmapIndex + ) + { + this.useBitmapIndex = useBitmapIndex != null ? useBitmapIndex : true; + this.minCardinalityToUseBitmapIndex = + minCardinalityToUseBitmapIndex != null ? minCardinalityToUseBitmapIndex : 0; + this.maxCardinalityToUseBitmapIndex = + maxCardinalityToUseBitmapIndex != null ? maxCardinalityToUseBitmapIndex : Integer.MAX_VALUE; + } + + @JsonProperty + public boolean getUseBitmapIndex() + { + return useBitmapIndex; + } + + @JsonProperty + public int getMinCardinalityToUseBitmapIndex() + { + return minCardinalityToUseBitmapIndex; + } + + @JsonProperty + public int getMaxCardinalityToUseBitmapIndex() + { + return maxCardinalityToUseBitmapIndex; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FilterTuning that = (FilterTuning) o; + return Objects.equals(useBitmapIndex, that.useBitmapIndex) && + Objects.equals(minCardinalityToUseBitmapIndex, that.minCardinalityToUseBitmapIndex) && + Objects.equals(maxCardinalityToUseBitmapIndex, that.maxCardinalityToUseBitmapIndex); + } + + @Override + public int hashCode() + { + return Objects.hash(useBitmapIndex, minCardinalityToUseBitmapIndex, maxCardinalityToUseBitmapIndex); + } + + @Override + public String toString() + { + return "FilterTuning{" + + "useBitmapIndex=" + useBitmapIndex + + ", minCardinalityToUseBitmapIndex=" + minCardinalityToUseBitmapIndex + + ", maxCardinalityToUseBitmapIndex=" + maxCardinalityToUseBitmapIndex + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java index 86213eb32166..3eba4e2601a3 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java @@ -20,14 +20,16 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; @@ -45,10 +47,10 @@ import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.filter.InFilter; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -64,7 +66,10 @@ public class InDimFilter implements DimFilter // Values can contain `null` object private final SortedSet values; private final String dimension; + @Nullable private final ExtractionFn extractionFn; + @Nullable + private final FilterTuning filterTuning; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; private final Supplier doublePredicateSupplier; @@ -73,7 +78,8 @@ public class InDimFilter implements DimFilter public InDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("values") Collection values, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkNotNull(dimension, "dimension can not be null"); @@ -85,11 +91,18 @@ public InDimFilter( } this.dimension = dimension; this.extractionFn = extractionFn; + this.filterTuning = filterTuning; this.longPredicateSupplier = getLongPredicateSupplier(); this.floatPredicateSupplier = getFloatPredicateSupplier(); this.doublePredicateSupplier = getDoublePredicateSupplier(); } + @VisibleForTesting + public InDimFilter(String dimension, Collection values, @Nullable ExtractionFn extractionFn) + { + this(dimension, values, extractionFn, null); + } + @JsonProperty public String getDimension() { @@ -102,12 +115,21 @@ public Set getValues() return values; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { @@ -133,7 +155,7 @@ public DimFilter optimize() { InDimFilter inFilter = optimizeLookup(); if (inFilter.values.size() == 1) { - return new SelectorDimFilter(inFilter.dimension, inFilter.values.first(), inFilter.getExtractionFn()); + return new SelectorDimFilter(inFilter.dimension, inFilter.values.first(), inFilter.getExtractionFn(), filterTuning); } return inFilter; } @@ -169,7 +191,7 @@ private InDimFilter optimizeLookup() if (keys.isEmpty()) { return this; } else { - return new InDimFilter(dimension, keys, null); + return new InDimFilter(dimension, keys, null, filterTuning); } } return this; @@ -184,7 +206,8 @@ public Filter toFilter() longPredicateSupplier, floatPredicateSupplier, doublePredicateSupplier, - extractionFn + extractionFn, + filterTuning ); } @@ -210,9 +233,21 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); + } + + @Override + public String toString() + { + final DimFilterToStringBuilder builder = new DimFilterToStringBuilder(); + return builder.appendDimension(dimension, extractionFn) + .append(" IN (") + .append(Joiner.on(", ").join(Iterables.transform(values, StringUtils::nullToEmptyNonDruidDataString))) + .append(")") + .appendFilterTuning(filterTuning) + .build(); } @Override @@ -224,51 +259,17 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - InDimFilter that = (InDimFilter) o; - - if (values != null ? !values.equals(that.values) : that.values != null) { - return false; - } - if (!dimension.equals(that.dimension)) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - + return values.equals(that.values) && + dimension.equals(that.dimension) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = values != null ? values.hashCode() : 0; - result = 31 * result + dimension.hashCode(); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; - } - - @Override - public String toString() - { - final StringBuilder builder = new StringBuilder(); - - if (extractionFn != null) { - builder.append(extractionFn).append("("); - } - - builder.append(dimension); - - if (extractionFn != null) { - builder.append(")"); - } - - builder.append(" IN (") - .append( - Joiner.on(", ").join( - Iterables.transform(values, input -> StringUtils.nullToEmptyNonDruidDataString(input)) - ) - ) - .append(")"); - return builder.toString(); + return Objects.hash(values, dimension, extractionFn, filterTuning); } // As the set of filtered values can be large, parsing them as longs should be done only if needed, and only once. diff --git a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java index eec5ccaa2a41..c1e7ec0d7584 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java @@ -20,10 +20,12 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.primitives.Longs; import org.apache.druid.java.util.common.JodaUtils; import org.apache.druid.java.util.common.Pair; @@ -32,25 +34,31 @@ import org.apache.druid.query.ordering.StringComparators; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; +import java.util.Objects; +import java.util.Set; public class IntervalDimFilter implements DimFilter { private final List intervals; private final List> intervalLongs; private final String dimension; + @Nullable private final ExtractionFn extractionFn; + @Nullable + private final FilterTuning filterTuning; private final OrDimFilter convertedFilter; @JsonCreator public IntervalDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("intervals") List intervals, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkNotNull(dimension, "dimension can not be null"); @@ -59,10 +67,17 @@ public IntervalDimFilter( this.dimension = dimension; this.intervals = Collections.unmodifiableList(JodaUtils.condenseIntervals(intervals)); this.extractionFn = extractionFn; + this.filterTuning = filterTuning; this.intervalLongs = makeIntervalLongs(); this.convertedFilter = new OrDimFilter(makeBoundDimFilters()); } + @VisibleForTesting + public IntervalDimFilter(String dimension, List intervals, @Nullable ExtractionFn extractionFn) + { + this(dimension, intervals, extractionFn, null); + } + @JsonProperty public String getDimension() { @@ -75,12 +90,21 @@ public List getIntervals() return intervals; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { @@ -125,9 +149,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override @@ -139,28 +163,17 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - IntervalDimFilter that = (IntervalDimFilter) o; - - if (!getIntervals().equals(that.getIntervals())) { - return false; - } - if (!getDimension().equals(that.getDimension())) { - return false; - } - return getExtractionFn() != null - ? getExtractionFn().equals(that.getExtractionFn()) - : that.getExtractionFn() == null; - + return intervals.equals(that.intervals) && + dimension.equals(that.dimension) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = getIntervals().hashCode(); - result = 31 * result + getDimension().hashCode(); - result = 31 * result + (getExtractionFn() != null ? getExtractionFn().hashCode() : 0); - return result; + return Objects.hash(intervals, dimension, extractionFn, filterTuning); } @Override @@ -173,7 +186,7 @@ private List> makeIntervalLongs() { List> intervalLongs = new ArrayList<>(); for (Interval interval : intervals) { - intervalLongs.add(new Pair(interval.getStartMillis(), interval.getEndMillis())); + intervalLongs.add(new Pair<>(interval.getStartMillis(), interval.getEndMillis())); } return intervalLongs; } @@ -190,7 +203,8 @@ private List makeBoundDimFilters() true, null, extractionFn, - StringComparators.NUMERIC + StringComparators.NUMERIC, + filterTuning ); boundDimFilters.add(boundDimFilter); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java index b211e38ad78d..ea16c28c63f6 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java @@ -21,11 +21,13 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.js.JavaScriptConfig; import org.apache.druid.query.extraction.ExtractionFn; @@ -36,14 +38,19 @@ import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; +import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; +import java.util.Objects; +import java.util.Set; public class JavaScriptDimFilter implements DimFilter { private final String dimension; private final String function; + @Nullable private final ExtractionFn extractionFn; + @Nullable + private final FilterTuning filterTuning; private final JavaScriptConfig config; /** @@ -61,7 +68,8 @@ public class JavaScriptDimFilter implements DimFilter public JavaScriptDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("function") String function, - @JsonProperty("extractionFn") ExtractionFn extractionFn, + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning, @JacksonInject JavaScriptConfig config ) { @@ -70,9 +78,21 @@ public JavaScriptDimFilter( this.dimension = dimension; this.function = function; this.extractionFn = extractionFn; + this.filterTuning = filterTuning; this.config = config; } + @VisibleForTesting + public JavaScriptDimFilter( + String dimension, + String function, + @Nullable ExtractionFn extractionFn, + JavaScriptConfig config + ) + { + this(dimension, function, extractionFn, null, config); + } + @JsonProperty public String getDimension() { @@ -85,12 +105,21 @@ public String getFunction() return function; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { @@ -118,7 +147,7 @@ public DimFilter optimize() public Filter toFilter() { JavaScriptPredicateFactory predicateFactory = getPredicateFactory(); - return new JavaScriptFilter(dimension, predicateFactory); + return new JavaScriptFilter(dimension, predicateFactory, filterTuning); } /** @@ -152,19 +181,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); - } - - @Override - public String toString() - { - return "JavaScriptDimFilter{" + - "dimension='" + dimension + '\'' + - ", function='" + function + '\'' + - ", extractionFn='" + extractionFn + '\'' + - '}'; + return ImmutableSet.of(dimension); } @Override @@ -173,29 +192,31 @@ public boolean equals(Object o) if (this == o) { return true; } - if (!(o instanceof JavaScriptDimFilter)) { + if (o == null || getClass() != o.getClass()) { return false; } - JavaScriptDimFilter that = (JavaScriptDimFilter) o; - - if (!dimension.equals(that.dimension)) { - return false; - } - if (!function.equals(that.function)) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - + return dimension.equals(that.dimension) && + function.equals(that.function) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = dimension.hashCode(); - result = 31 * result + function.hashCode(); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; + return Objects.hash(dimension, function, extractionFn, filterTuning); + } + + @Override + public String toString() + { + return "JavaScriptDimFilter{" + + "dimension='" + dimension + '\'' + + ", function='" + function + '\'' + + ", extractionFn=" + extractionFn + + ", filterTuning=" + filterTuning + + '}'; } public static class JavaScriptPredicateFactory implements DruidPredicateFactory diff --git a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java index e7ec2fcc2d62..34749b2d2cfa 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java @@ -20,11 +20,13 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; import org.apache.druid.common.config.NullHandling; @@ -35,7 +37,8 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; +import java.util.Objects; +import java.util.Set; import java.util.regex.Pattern; public class LikeDimFilter implements DimFilter @@ -47,21 +50,27 @@ public class LikeDimFilter implements DimFilter private final String dimension; private final String pattern; + @Nullable private final Character escapeChar; + @Nullable private final ExtractionFn extractionFn; + @Nullable + private final FilterTuning filterTuning; private final LikeMatcher likeMatcher; @JsonCreator public LikeDimFilter( @JsonProperty("dimension") final String dimension, @JsonProperty("pattern") final String pattern, - @JsonProperty("escape") final String escape, - @JsonProperty("extractionFn") final ExtractionFn extractionFn + @JsonProperty("escape") @Nullable final String escape, + @JsonProperty("extractionFn") @Nullable final ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable final FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension"); this.pattern = Preconditions.checkNotNull(pattern, "pattern"); this.extractionFn = extractionFn; + this.filterTuning = filterTuning; if (escape != null && escape.length() != 1) { throw new IllegalArgumentException("Escape must be null or a single character"); @@ -72,6 +81,129 @@ public LikeDimFilter( this.likeMatcher = LikeMatcher.from(pattern, this.escapeChar); } + @VisibleForTesting + public LikeDimFilter( + final String dimension, + final String pattern, + @Nullable final String escape, + @Nullable final ExtractionFn extractionFn + ) + { + this(dimension, pattern, escape, extractionFn, null); + } + + @JsonProperty + public String getDimension() + { + return dimension; + } + + @JsonProperty + public String getPattern() + { + return pattern; + } + + @Nullable + @JsonProperty + public String getEscape() + { + return escapeChar != null ? escapeChar.toString() : null; + } + + @Nullable + @JsonProperty + public ExtractionFn getExtractionFn() + { + return extractionFn; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + + @Override + public byte[] getCacheKey() + { + final byte[] dimensionBytes = StringUtils.toUtf8(dimension); + final byte[] patternBytes = StringUtils.toUtf8(pattern); + final byte[] escapeBytes = escapeChar == null ? new byte[0] : Chars.toByteArray(escapeChar); + final byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); + final int sz = 4 + dimensionBytes.length + patternBytes.length + escapeBytes.length + extractionFnBytes.length; + return ByteBuffer.allocate(sz) + .put(DimFilterUtils.LIKE_CACHE_ID) + .put(dimensionBytes) + .put(DimFilterUtils.STRING_SEPARATOR) + .put(patternBytes) + .put(DimFilterUtils.STRING_SEPARATOR) + .put(escapeBytes) + .put(DimFilterUtils.STRING_SEPARATOR) + .put(extractionFnBytes) + .array(); + } + + @Override + public DimFilter optimize() + { + return this; + } + + @Override + public Filter toFilter() + { + return new LikeFilter(dimension, extractionFn, likeMatcher, filterTuning); + } + + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LikeDimFilter that = (LikeDimFilter) o; + return dimension.equals(that.dimension) && + pattern.equals(that.pattern) && + Objects.equals(escapeChar, that.escapeChar) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); + } + + @Override + public int hashCode() + { + return Objects.hash(dimension, pattern, escapeChar, extractionFn, filterTuning); + } + + @Override + public String toString() + { + final DimFilterToStringBuilder builder = new DimFilterToStringBuilder(); + builder.appendDimension(dimension, extractionFn).append(" LIKE '").append(pattern).append("'"); + if (escapeChar != null) { + builder.append(" ESCAPE '").append(escapeChar).append("'"); + } + return builder.appendFilterTuning(filterTuning).build(); + } + public static class LikeMatcher { public enum SuffixMatch @@ -232,131 +364,4 @@ public SuffixMatch getSuffixMatch() return suffixMatch; } } - - @JsonProperty - public String getDimension() - { - return dimension; - } - - @JsonProperty - public String getPattern() - { - return pattern; - } - - @JsonProperty - public String getEscape() - { - return escapeChar != null ? escapeChar.toString() : null; - } - - @JsonProperty - public ExtractionFn getExtractionFn() - { - return extractionFn; - } - - @Override - public byte[] getCacheKey() - { - final byte[] dimensionBytes = StringUtils.toUtf8(dimension); - final byte[] patternBytes = StringUtils.toUtf8(pattern); - final byte[] escapeBytes = escapeChar == null ? new byte[0] : Chars.toByteArray(escapeChar); - final byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - final int sz = 4 + dimensionBytes.length + patternBytes.length + escapeBytes.length + extractionFnBytes.length; - return ByteBuffer.allocate(sz) - .put(DimFilterUtils.LIKE_CACHE_ID) - .put(dimensionBytes) - .put(DimFilterUtils.STRING_SEPARATOR) - .put(patternBytes) - .put(DimFilterUtils.STRING_SEPARATOR) - .put(escapeBytes) - .put(DimFilterUtils.STRING_SEPARATOR) - .put(extractionFnBytes) - .array(); - } - - @Override - public DimFilter optimize() - { - return this; - } - - @Override - public Filter toFilter() - { - return new LikeFilter(dimension, extractionFn, likeMatcher); - } - - @Override - public RangeSet getDimensionRangeSet(String dimension) - { - return null; - } - - @Override - public HashSet getRequiredColumns() - { - return Sets.newHashSet(dimension); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - LikeDimFilter that = (LikeDimFilter) o; - - if (dimension != null ? !dimension.equals(that.dimension) : that.dimension != null) { - return false; - } - if (pattern != null ? !pattern.equals(that.pattern) : that.pattern != null) { - return false; - } - if (escapeChar != null ? !escapeChar.equals(that.escapeChar) : that.escapeChar != null) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - - } - - @Override - public int hashCode() - { - int result = dimension != null ? dimension.hashCode() : 0; - result = 31 * result + (pattern != null ? pattern.hashCode() : 0); - result = 31 * result + (escapeChar != null ? escapeChar.hashCode() : 0); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; - } - - @Override - public String toString() - { - final StringBuilder builder = new StringBuilder(); - - if (extractionFn != null) { - builder.append(extractionFn).append("("); - } - - builder.append(dimension); - - if (extractionFn != null) { - builder.append(")"); - } - - builder.append(" LIKE '").append(pattern).append("'"); - - if (escapeChar != null) { - builder.append(" ESCAPE '").append(escapeChar).append("'"); - } - - return builder.toString(); - } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java index 23f968074c7b..039667eb7edb 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java @@ -27,8 +27,8 @@ import org.apache.druid.segment.filter.NotFilter; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.List; +import java.util.Set; /** */ @@ -102,7 +102,7 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { return field.getRequiredColumns(); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java index d2ed4e77bbde..762ad72ea5fe 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java @@ -33,6 +33,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; /** */ @@ -107,7 +108,7 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { HashSet requiredColumns = new HashSet<>(); fields.forEach(field -> requiredColumns.addAll(field.getRequiredColumns())); diff --git a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java index adb52ba614bc..b696ad9ff9dd 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java @@ -20,16 +20,20 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.segment.filter.RegexFilter; +import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; +import java.util.Objects; +import java.util.Set; import java.util.regex.Pattern; /** @@ -38,7 +42,10 @@ public class RegexDimFilter implements DimFilter { private final String dimension; private final String pattern; + @Nullable private final ExtractionFn extractionFn; + @Nullable + private final FilterTuning filterTuning; private final Pattern compiledPattern; @@ -46,7 +53,8 @@ public class RegexDimFilter implements DimFilter public RegexDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("pattern") String pattern, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -55,6 +63,13 @@ public RegexDimFilter( this.pattern = pattern; this.extractionFn = extractionFn; this.compiledPattern = Pattern.compile(pattern); + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public RegexDimFilter(String dimension, String pattern, @Nullable ExtractionFn extractionFn) + { + this(dimension, pattern, extractionFn, null); } @JsonProperty @@ -69,12 +84,21 @@ public String getPattern() return pattern; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { @@ -101,7 +125,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - return new RegexFilter(dimension, compiledPattern, extractionFn); + return new RegexFilter(dimension, compiledPattern, extractionFn, filterTuning); } @Override @@ -111,9 +135,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override @@ -123,6 +147,7 @@ public String toString() "dimension='" + dimension + '\'' + ", pattern='" + pattern + '\'' + ", extractionFn='" + extractionFn + '\'' + + ", filterTuning=" + filterTuning + '}'; } @@ -132,28 +157,19 @@ public boolean equals(Object o) if (this == o) { return true; } - if (!(o instanceof RegexDimFilter)) { + if (o == null || getClass() != o.getClass()) { return false; } - RegexDimFilter that = (RegexDimFilter) o; - - if (!dimension.equals(that.dimension)) { - return false; - } - if (!pattern.equals(that.pattern)) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - + return dimension.equals(that.dimension) && + pattern.equals(that.pattern) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = dimension.hashCode(); - result = 31 * result + pattern.hashCode(); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; + return Objects.hash(dimension, pattern, extractionFn, filterTuning); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java index 67d78b6f87fc..757cc7955875 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java @@ -19,17 +19,21 @@ package org.apache.druid.query.filter; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.search.SearchQuerySpec; import org.apache.druid.segment.filter.SearchQueryFilter; +import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; +import java.util.Objects; +import java.util.Set; /** */ @@ -37,12 +41,16 @@ public class SearchQueryDimFilter implements DimFilter { private final String dimension; private final SearchQuerySpec query; + @Nullable private final ExtractionFn extractionFn; + @Nullable + private final FilterTuning filterTuning; public SearchQueryDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("query") SearchQuerySpec query, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -51,6 +59,17 @@ public SearchQueryDimFilter( this.dimension = dimension; this.query = query; this.extractionFn = extractionFn; + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public SearchQueryDimFilter( + String dimension, + SearchQuerySpec query, + @Nullable ExtractionFn extractionFn + ) + { + this(dimension, query, extractionFn, null); } @JsonProperty @@ -65,12 +84,21 @@ public SearchQuerySpec getQuery() return query; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { @@ -97,7 +125,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - return new SearchQueryFilter(dimension, query, extractionFn); + return new SearchQueryFilter(dimension, query, extractionFn, filterTuning); } @Override @@ -107,9 +135,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override @@ -119,6 +147,7 @@ public String toString() "dimension='" + dimension + '\'' + ", query=" + query + ", extractionFn='" + extractionFn + '\'' + + ", filterTuning=" + filterTuning + '}'; } @@ -128,28 +157,19 @@ public boolean equals(Object o) if (this == o) { return true; } - if (!(o instanceof SearchQueryDimFilter)) { + if (o == null || getClass() != o.getClass()) { return false; } - SearchQueryDimFilter that = (SearchQueryDimFilter) o; - - if (!dimension.equals(that.dimension)) { - return false; - } - if (!query.equals(that.query)) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - + return dimension.equals(that.dimension) && + query.equals(that.query) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = dimension.hashCode(); - result = 31 * result + query.hashCode(); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; + return Objects.hash(dimension, query, extractionFn, filterTuning); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java index 2c71ccc89608..ef351a5c41e0 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java @@ -20,19 +20,19 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.guava.GuavaUtils; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.segment.filter.DimensionPredicateFilter; @@ -40,8 +40,8 @@ import javax.annotation.Nullable; import java.util.Collections; -import java.util.HashSet; import java.util.Objects; +import java.util.Set; /** */ @@ -51,7 +51,10 @@ public class SelectorDimFilter implements DimFilter @Nullable private final String value; + @Nullable private final ExtractionFn extractionFn; + @Nullable + private final FilterTuning filterTuning; private final Object initLock = new Object(); @@ -63,7 +66,8 @@ public class SelectorDimFilter implements DimFilter public SelectorDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("value") String value, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -71,6 +75,12 @@ public SelectorDimFilter( this.dimension = dimension; this.value = NullHandling.emptyToNullIfNeeded(value); this.extractionFn = extractionFn; + this.filterTuning = filterTuning; + } + + public SelectorDimFilter(String dimension, String value, @Nullable ExtractionFn extractionFn) + { + this(dimension, value, extractionFn, null); } @Override @@ -90,14 +100,14 @@ public byte[] getCacheKey() @Override public DimFilter optimize() { - return new InDimFilter(dimension, Collections.singletonList(value), extractionFn).optimize(); + return new InDimFilter(dimension, Collections.singletonList(value), extractionFn, filterTuning).optimize(); } @Override public Filter toFilter() { if (extractionFn == null) { - return new SelectorFilter(dimension, value); + return new SelectorFilter(dimension, value, filterTuning); } else { final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() @@ -129,7 +139,7 @@ public DruidDoublePredicate makeDoublePredicate() return druidDoublePredicate; } }; - return new DimensionPredicateFilter(dimension, predicateFactory, extractionFn); + return new DimensionPredicateFilter(dimension, predicateFactory, extractionFn, filterTuning); } } @@ -139,26 +149,35 @@ public String getDimension() return dimension; } + @Nullable @JsonProperty public String getValue() { return value; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public String toString() { - if (extractionFn != null) { - return StringUtils.format("%s(%s) = %s", extractionFn, dimension, value); - } else { - return StringUtils.format("%s = %s", dimension, value); - } + return new DimFilterToStringBuilder().appendDimension(dimension, extractionFn) + .appendEquals(value) + .appendFilterTuning(filterTuning) + .build(); } @Override @@ -170,16 +189,17 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - SelectorDimFilter that = (SelectorDimFilter) o; + return dimension.equals(that.dimension) && + Objects.equals(value, that.value) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); + } - if (!dimension.equals(that.dimension)) { - return false; - } - if (value != null ? !value.equals(that.value) : that.value != null) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; + @Override + public int hashCode() + { + return Objects.hash(dimension, value, extractionFn, filterTuning); } @Override @@ -201,18 +221,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() - { - return Sets.newHashSet(dimension); - } - - @Override - public int hashCode() + public Set getRequiredColumns() { - int result = dimension.hashCode(); - result = 31 * result + (value != null ? value.hashCode() : 0); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; + return ImmutableSet.of(dimension); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java index 0d12ac5979e8..c7a124820f11 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java @@ -20,16 +20,20 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.collections.spatial.search.Bound; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.segment.filter.SpatialFilter; +import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; +import java.util.Objects; +import java.util.Set; /** */ @@ -37,11 +41,14 @@ public class SpatialDimFilter implements DimFilter { private final String dimension; private final Bound bound; + @Nullable + private final FilterTuning filterTuning; @JsonCreator public SpatialDimFilter( @JsonProperty("dimension") String dimension, - @JsonProperty("bound") Bound bound + @JsonProperty("bound") Bound bound, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -49,6 +56,13 @@ public SpatialDimFilter( this.dimension = dimension; this.bound = bound; + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public SpatialDimFilter(String dimension, Bound bound) + { + this(dimension, bound, null); } @Override @@ -83,10 +97,18 @@ public Bound getBound() return bound; } + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public Filter toFilter() { - return new SpatialFilter(dimension, bound); + return new SpatialFilter(dimension, bound, filterTuning); } @Override @@ -96,9 +118,19 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + + @Override + public String toString() { - return Sets.newHashSet(dimension); + return "SpatialDimFilter{" + + "dimension='" + dimension + '\'' + + ", bound=" + bound + + ", filterTuning=" + filterTuning + + '}'; } @Override @@ -110,33 +142,15 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - SpatialDimFilter that = (SpatialDimFilter) o; - - if (bound != null ? !bound.equals(that.bound) : that.bound != null) { - return false; - } - if (dimension != null ? !dimension.equals(that.dimension) : that.dimension != null) { - return false; - } - - return true; + return dimension.equals(that.dimension) && + bound.equals(that.bound) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = dimension != null ? dimension.hashCode() : 0; - result = 31 * result + (bound != null ? bound.hashCode() : 0); - return result; - } - - @Override - public String toString() - { - return "SpatialDimFilter{" + - "dimension='" + dimension + '\'' + - ", bound=" + bound + - '}'; + return Objects.hash(dimension, bound, filterTuning); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java index 405bece232bd..d10e6d9b8b5b 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java @@ -23,14 +23,13 @@ import org.apache.druid.segment.filter.TrueFilter; import java.nio.ByteBuffer; -import java.util.HashSet; +import java.util.Collections; +import java.util.Set; /** */ public class TrueDimFilter implements DimFilter { - private static HashSet REQUIRED_COLUMNS = new HashSet<>(); - @Override public byte[] getCacheKey() { @@ -56,8 +55,8 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return REQUIRED_COLUMNS; + return Collections.emptySet(); } } diff --git a/processing/src/main/java/org/apache/druid/query/lookup/LookupExtractionFn.java b/processing/src/main/java/org/apache/druid/query/lookup/LookupExtractionFn.java index 9266cf2da621..fa918af1c755 100644 --- a/processing/src/main/java/org/apache/druid/query/lookup/LookupExtractionFn.java +++ b/processing/src/main/java/org/apache/druid/query/lookup/LookupExtractionFn.java @@ -40,9 +40,9 @@ public class LookupExtractionFn extends FunctionalExtraction public LookupExtractionFn( @JsonProperty("lookup") final LookupExtractor lookup, @JsonProperty("retainMissingValue") final boolean retainMissingValue, - @Nullable @JsonProperty("replaceMissingValueWith") final String replaceMissingValueWith, - @Nullable @JsonProperty("injective") final Boolean injective, - @Nullable @JsonProperty("optimize") final Boolean optimize + @JsonProperty("replaceMissingValueWith") @Nullable final String replaceMissingValueWith, + @JsonProperty("injective") @Nullable final Boolean injective, + @JsonProperty("optimize") @Nullable final Boolean optimize ) { super( diff --git a/processing/src/main/java/org/apache/druid/query/lookup/RegisteredLookupExtractionFn.java b/processing/src/main/java/org/apache/druid/query/lookup/RegisteredLookupExtractionFn.java index 9ffe0fbc602a..d9b0b676e79b 100644 --- a/processing/src/main/java/org/apache/druid/query/lookup/RegisteredLookupExtractionFn.java +++ b/processing/src/main/java/org/apache/druid/query/lookup/RegisteredLookupExtractionFn.java @@ -47,8 +47,8 @@ public RegisteredLookupExtractionFn( @JacksonInject LookupExtractorFactoryContainerProvider manager, @JsonProperty("lookup") String lookup, @JsonProperty("retainMissingValue") final boolean retainMissingValue, - @Nullable @JsonProperty("replaceMissingValueWith") final String replaceMissingValueWith, - @Nullable @JsonProperty("injective") final Boolean injective, + @JsonProperty("replaceMissingValueWith") @Nullable final String replaceMissingValueWith, + @JsonProperty("injective") @Nullable final Boolean injective, @JsonProperty("optimize") Boolean optimize ) { diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanResultValue.java b/processing/src/main/java/org/apache/druid/query/scan/ScanResultValue.java index 8673b3479b03..aa368102dd4f 100644 --- a/processing/src/main/java/org/apache/druid/query/scan/ScanResultValue.java +++ b/processing/src/main/java/org/apache/druid/query/scan/ScanResultValue.java @@ -48,7 +48,7 @@ public class ScanResultValue implements Comparable @JsonCreator public ScanResultValue( - @Nullable @JsonProperty("segmentId") String segmentId, + @JsonProperty("segmentId") @Nullable String segmentId, @JsonProperty("columns") List columns, @JsonProperty("events") Object events ) diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java index 45227b93b739..d50c5657686a 100644 --- a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java +++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java @@ -230,7 +230,7 @@ public TopNQueryBuilder filters(String dimensionName, String value) public TopNQueryBuilder filters(String dimensionName, String value, String... values) { - dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null); + dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null, null); return this; } diff --git a/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java b/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java index d387c65c6a10..a6c50d5f6396 100644 --- a/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java +++ b/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java @@ -56,7 +56,7 @@ public final class FilteredOffset extends Offset rowOffsetMatcherFactory ); } else { - if (postFilter.supportsBitmapIndex(bitmapIndexSelector)) { + if (postFilter.shouldUseBitmapIndex(bitmapIndexSelector)) { filterMatcher = rowOffsetMatcherFactory.makeRowOffsetMatcher( postFilter.getBitmapIndex(bitmapIndexSelector) ); diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java index d055b4bb2f64..87faef84d554 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.java.util.common.DateTimes; @@ -212,7 +213,7 @@ public boolean canVectorize( { if (filter != null) { final boolean filterCanVectorize = - filter.supportsBitmapIndex(makeBitmapIndexSelector(virtualColumns)) + filter.shouldUseBitmapIndex(makeBitmapIndexSelector(virtualColumns)) || filter.canVectorizeMatcher(); if (!filterCanVectorize) { @@ -347,7 +348,8 @@ private Interval computeCursorInterval(final Granularity gran, final Interval in return interval.overlap(dataInterval); } - private ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualColumns virtualColumns) + @VisibleForTesting + public ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualColumns virtualColumns) { return new ColumnSelectorBitmapIndexSelector( index.getBitmapFactoryForDimensions(), @@ -356,9 +358,10 @@ private ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualC ); } - private FilterAnalysis analyzeFilter( + @VisibleForTesting + public FilterAnalysis analyzeFilter( @Nullable final Filter filter, - ColumnSelectorBitmapIndexSelector bitmapIndexSelector, + ColumnSelectorBitmapIndexSelector indexSelector, @Nullable QueryMetrics queryMetrics ) { @@ -389,7 +392,9 @@ private FilterAnalysis analyzeFilter( if (filter instanceof AndFilter) { // If we get an AndFilter, we can split the subfilters across both filtering stages for (Filter subfilter : ((AndFilter) filter).getFilters()) { - if (subfilter.supportsBitmapIndex(bitmapIndexSelector)) { + + if (subfilter.supportsBitmapIndex(indexSelector) && subfilter.shouldUseBitmapIndex(indexSelector)) { + preFilters.add(subfilter); } else { postFilters.add(subfilter); @@ -397,7 +402,7 @@ private FilterAnalysis analyzeFilter( } } else { // If we get an OrFilter or a single filter, handle the filter in one stage - if (filter.supportsBitmapIndex(bitmapIndexSelector)) { + if (filter.supportsBitmapIndex(indexSelector) && filter.shouldUseBitmapIndex(indexSelector)) { preFilters.add(filter); } else { postFilters.add(filter); @@ -411,15 +416,15 @@ private FilterAnalysis analyzeFilter( } else { if (queryMetrics != null) { BitmapResultFactory bitmapResultFactory = - queryMetrics.makeBitmapResultFactory(bitmapIndexSelector.getBitmapFactory()); + queryMetrics.makeBitmapResultFactory(indexSelector.getBitmapFactory()); long bitmapConstructionStartNs = System.nanoTime(); // Use AndFilter.getBitmapResult to intersect the preFilters to get its short-circuiting behavior. - preFilterBitmap = AndFilter.getBitmapIndex(bitmapIndexSelector, bitmapResultFactory, preFilters); + preFilterBitmap = AndFilter.getBitmapIndex(indexSelector, bitmapResultFactory, preFilters); preFilteredRows = preFilterBitmap.size(); queryMetrics.reportBitmapConstructionTime(System.nanoTime() - bitmapConstructionStartNs); } else { - BitmapResultFactory bitmapResultFactory = new DefaultBitmapResultFactory(bitmapIndexSelector.getBitmapFactory()); - preFilterBitmap = AndFilter.getBitmapIndex(bitmapIndexSelector, bitmapResultFactory, preFilters); + BitmapResultFactory bitmapResultFactory = new DefaultBitmapResultFactory(indexSelector.getBitmapFactory()); + preFilterBitmap = AndFilter.getBitmapIndex(indexSelector, bitmapResultFactory, preFilters); } } @@ -442,7 +447,8 @@ private FilterAnalysis analyzeFilter( return new FilterAnalysis(preFilterBitmap, postFilter); } - private static class FilterAnalysis + @VisibleForTesting + public static class FilterAnalysis { private final Filter postFilter; private final ImmutableBitmap preFilterBitmap; diff --git a/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java index 76fd626c9d45..2192340f7fc3 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java @@ -34,7 +34,6 @@ import org.apache.druid.query.filter.vector.ReadableVectorMatch; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; @@ -159,31 +158,6 @@ public List getFilters() return filters; } - @Override - public boolean supportsBitmapIndex(BitmapIndexSelector selector) - { - for (Filter filter : filters) { - if (!filter.supportsBitmapIndex(selector)) { - return false; - } - } - return true; - } - - @Override - public boolean supportsSelectivityEstimation( - final ColumnSelector columnSelector, - final BitmapIndexSelector indexSelector - ) - { - for (Filter filter : filters) { - if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) { - return false; - } - } - return true; - } - @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java index 28b8548f9d73..f0f8a6b08ad8 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java @@ -34,6 +34,7 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer; @@ -46,12 +47,14 @@ import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import java.util.Comparator; +import java.util.Set; public class BoundFilter implements Filter { private final BoundDimFilter boundDimFilter; private final Comparator comparator; private final ExtractionFn extractionFn; + private final FilterTuning filterTuning; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; @@ -65,6 +68,7 @@ public BoundFilter(final BoundDimFilter boundDimFilter) this.longPredicateSupplier = boundDimFilter.getLongPredicateSupplier(); this.floatPredicateSupplier = boundDimFilter.getFloatPredicateSupplier(); this.doublePredicateSupplier = boundDimFilter.getDoublePredicateSupplier(); + this.filterTuning = boundDimFilter.getFilterTuning(); } @Override @@ -149,6 +153,11 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return selector.getBitmapIndex(boundDimFilter.getDimension()) != null; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); + } @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) @@ -156,6 +165,12 @@ public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, Bitm return Filters.supportsSelectivityEstimation(this, boundDimFilter.getDimension(), columnSelector, indexSelector); } + @Override + public Set getRequiredColumns() + { + return boundDimFilter.getRequiredColumns(); + } + private static Pair getStartEndIndexes( final BoundDimFilter boundDimFilter, final BitmapIndex bitmapIndex diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java index eea5bcd74dce..5bcb350a9b59 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java @@ -37,6 +37,8 @@ import javax.annotation.Nullable; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** */ @@ -141,12 +143,24 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return false; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return false; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return false; } + @Override + public Set getRequiredColumns() + { + return dimensions.stream().map(DimensionSpec::getDimension).collect(Collectors.toSet()); + } + @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java index fbcff2a2dafa..6029db54e545 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.extraction.ExtractionFn; @@ -30,6 +31,7 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer; @@ -38,6 +40,8 @@ import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import java.util.Set; + /** */ public class DimensionPredicateFilter implements Filter @@ -46,17 +50,29 @@ public class DimensionPredicateFilter implements Filter private final DruidPredicateFactory predicateFactory; private final String basePredicateString; private final ExtractionFn extractionFn; + private final FilterTuning filterTuning; public DimensionPredicateFilter( final String dimension, final DruidPredicateFactory predicateFactory, final ExtractionFn extractionFn ) + { + this(dimension, predicateFactory, extractionFn, null); + } + + public DimensionPredicateFilter( + final String dimension, + final DruidPredicateFactory predicateFactory, + final ExtractionFn extractionFn, + final FilterTuning filterTuning + ) { Preconditions.checkNotNull(predicateFactory, "predicateFactory"); this.dimension = Preconditions.checkNotNull(dimension, "dimension"); this.basePredicateString = predicateFactory.toString(); this.extractionFn = extractionFn; + this.filterTuning = filterTuning; if (extractionFn == null) { this.predicateFactory = predicateFactory; @@ -120,12 +136,24 @@ public boolean canVectorizeMatcher() return true; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return selector.getBitmapIndex(dimension) != null; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java index c721e9e0fb5d..a8dd18fef434 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java @@ -30,6 +30,7 @@ import org.apache.druid.query.expression.ExprUtils; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.ColumnSelector; @@ -44,11 +45,13 @@ public class ExpressionFilter implements Filter { private final Supplier expr; private final Supplier> requiredBindings; + private final FilterTuning filterTuning; - public ExpressionFilter(final Supplier expr) + public ExpressionFilter(final Supplier expr, final FilterTuning filterTuning) { this.expr = expr; this.requiredBindings = Suppliers.memoize(() -> expr.get().analyzeInputs().getRequiredBindings()); + this.filterTuning = filterTuning; } @Override @@ -108,6 +111,12 @@ public boolean supportsBitmapIndex(final BitmapIndexSelector selector) } } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); + } + @Override public T getBitmapResult(final BitmapIndexSelector selector, final BitmapResultFactory bitmapResultFactory) { @@ -151,4 +160,10 @@ public double estimateSelectivity(final BitmapIndexSelector indexSelector) // Selectivity estimation not supported. throw new UnsupportedOperationException(); } + + @Override + public Set getRequiredColumns() + { + return requiredBindings.get(); + } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java index a81eb60a8fea..4d3ebd952be5 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java @@ -39,6 +39,7 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.ValueMatcherColumnSelectorStrategy; import org.apache.druid.query.filter.ValueMatcherColumnSelectorStrategyFactory; @@ -651,4 +652,33 @@ private static void generateAllCombinations( generateAllCombinations(result, andList.subList(1, andList.size()), nonAndList); } } + + /** + * This method provides a "standard" implementation of {@link Filter#shouldUseBitmapIndex(BitmapIndexSelector)} which takes + * a {@link Filter}, a {@link BitmapIndexSelector}, and {@link FilterTuning} to determine if: + * a) the filter supports bitmap indexes for all required columns + * b) the filter tuning specifies that it should use the index + * c) the cardinality of the column is above the minimum threshold and below the maximum threshold to use the index + * + * If all these things are true, {@link org.apache.druid.segment.QueryableIndexStorageAdapter} will utilize the + * indexes. + */ + public static boolean shouldUseBitmapIndex( + Filter filter, + BitmapIndexSelector indexSelector, + @Nullable FilterTuning filterTuning + ) + { + final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector); + if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseBitmapIndex()) { + return filter.getRequiredColumns().stream().allMatch(column -> { + final BitmapIndex index = indexSelector.getBitmapIndex(column); + Preconditions.checkNotNull(index, "Column does not have a bitmap index"); + final int cardinality = index.getCardinality(); + return cardinality >= tuning.getMinCardinalityToUseBitmapIndex() + && cardinality <= tuning.getMaxCardinalityToUseBitmapIndex(); + }); + } + return false; + } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java index 8186781ee8a0..4594126b37a2 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java @@ -21,6 +21,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.ints.IntIterable; import it.unimi.dsi.fastutil.ints.IntIterator; import org.apache.druid.collections.bitmap.ImmutableBitmap; @@ -32,6 +33,7 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer; @@ -52,6 +54,7 @@ public class InFilter implements Filter private final String dimension; private final Set values; private final ExtractionFn extractionFn; + private final FilterTuning filterTuning; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; private final Supplier doublePredicateSupplier; @@ -62,12 +65,14 @@ public InFilter( Supplier longPredicateSupplier, Supplier floatPredicateSupplier, Supplier doublePredicateSupplier, - ExtractionFn extractionFn + ExtractionFn extractionFn, + FilterTuning filterTuning ) { this.dimension = dimension; this.values = values; this.extractionFn = extractionFn; + this.filterTuning = filterTuning; this.longPredicateSupplier = longPredicateSupplier; this.floatPredicateSupplier = floatPredicateSupplier; this.doublePredicateSupplier = doublePredicateSupplier; @@ -162,12 +167,24 @@ public boolean canVectorizeMatcher() return true; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return selector.getBitmapIndex(dimension) != null; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java index eedebe32e035..952f269f4724 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java @@ -20,27 +20,34 @@ package org.apache.druid.segment.filter; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.JavaScriptDimFilter; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.mozilla.javascript.Context; +import java.util.Set; + public class JavaScriptFilter implements Filter { private final String dimension; private final JavaScriptDimFilter.JavaScriptPredicateFactory predicateFactory; + private final FilterTuning filterTuning; public JavaScriptFilter( String dimension, - JavaScriptDimFilter.JavaScriptPredicateFactory predicate + JavaScriptDimFilter.JavaScriptPredicateFactory predicate, + FilterTuning filterTuning ) { this.dimension = dimension; this.predicateFactory = predicate; + this.filterTuning = filterTuning; } @Override @@ -92,9 +99,21 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return selector.getBitmapIndex(dimension) != null; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector); } + + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java index 765bcb430fdb..6d148e41c388 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.segment.filter; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.ints.IntIterable; import it.unimi.dsi.fastutil.ints.IntIterator; import org.apache.druid.collections.bitmap.ImmutableBitmap; @@ -28,6 +29,7 @@ import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.LikeDimFilter; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; @@ -43,22 +45,26 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.NoSuchElementException; +import java.util.Set; public class LikeFilter implements Filter { private final String dimension; private final ExtractionFn extractionFn; private final LikeDimFilter.LikeMatcher likeMatcher; + private final FilterTuning filterTuning; public LikeFilter( final String dimension, final ExtractionFn extractionFn, - final LikeDimFilter.LikeMatcher likeMatcher + final LikeDimFilter.LikeMatcher likeMatcher, + final FilterTuning filterTuning ) { this.dimension = dimension; this.extractionFn = extractionFn; this.likeMatcher = likeMatcher; + this.filterTuning = filterTuning; } @Override @@ -95,12 +101,24 @@ public boolean canVectorizeMatcher() return true; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return selector.getBitmapIndex(dimension) != null; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java index ef12e1693347..3a39e2f1370c 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java @@ -32,6 +32,8 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import java.util.Set; + /** */ public class NotFilter implements Filter @@ -103,12 +105,24 @@ public boolean canVectorizeMatcher() return baseFilter.canVectorizeMatcher(); } + @Override + public Set getRequiredColumns() + { + return baseFilter.getRequiredColumns(); + } + @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return baseFilter.supportsBitmapIndex(selector); } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return baseFilter.shouldUseBitmapIndex(selector); + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java index 01d82298e401..1c54de1b3471 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java @@ -34,7 +34,6 @@ import org.apache.druid.query.filter.vector.VectorMatch; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; @@ -133,28 +132,6 @@ public List getFilters() return filters; } - @Override - public boolean supportsBitmapIndex(BitmapIndexSelector selector) - { - for (Filter filter : filters) { - if (!filter.supportsBitmapIndex(selector)) { - return false; - } - } - return true; - } - - @Override - public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) - { - for (Filter filter : filters) { - if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) { - return false; - } - } - return true; - } - @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/RegexFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/RegexFilter.java index 6ebc46ca1599..c71841ad08d2 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/RegexFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/RegexFilter.java @@ -25,6 +25,7 @@ import org.apache.druid.query.filter.DruidFloatPredicate; import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; +import org.apache.druid.query.filter.FilterTuning; import java.util.regex.Pattern; @@ -35,7 +36,8 @@ public class RegexFilter extends DimensionPredicateFilter public RegexFilter( final String dimension, final Pattern pattern, - final ExtractionFn extractionFn + final ExtractionFn extractionFn, + final FilterTuning filterTuning ) { super( @@ -74,7 +76,8 @@ public String toString() '}'; } }, - extractionFn + extractionFn, + filterTuning ); } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SearchQueryFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SearchQueryFilter.java index 9063cd7818a2..81424fe76ce8 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SearchQueryFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SearchQueryFilter.java @@ -27,6 +27,7 @@ import org.apache.druid.query.filter.DruidFloatPredicate; import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.search.SearchQuerySpec; /** @@ -37,7 +38,8 @@ public class SearchQueryFilter extends DimensionPredicateFilter public SearchQueryFilter( @JsonProperty("dimension") final String dimension, @JsonProperty("query") final SearchQuerySpec query, - @JsonProperty("extractionFn") final ExtractionFn extractionFn + @JsonProperty("extractionFn") final ExtractionFn extractionFn, + @JsonProperty("filterTuning") final FilterTuning filterTuning ) { super( @@ -68,7 +70,8 @@ public DruidDoublePredicate makeDoublePredicate() return input -> query.accept(String.valueOf(input)); } }, - extractionFn + extractionFn, + filterTuning ); } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java index 3640b7d3a512..641e768faeff 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java @@ -19,10 +19,12 @@ package org.apache.druid.segment.filter; +import com.google.common.collect.ImmutableSet; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer; @@ -31,20 +33,35 @@ import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import javax.annotation.Nullable; +import java.util.Set; + /** */ public class SelectorFilter implements Filter { private final String dimension; private final String value; + @Nullable + private final FilterTuning filterTuning; public SelectorFilter( String dimension, String value ) + { + this(dimension, value, null); + } + + public SelectorFilter( + String dimension, + String value, + @Nullable FilterTuning filterTuning + ) { this.dimension = dimension; this.value = value; + this.filterTuning = filterTuning; } @Override @@ -75,6 +92,12 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return selector.getBitmapIndex(dimension) != null; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { @@ -93,6 +116,12 @@ public boolean canVectorizeMatcher() return true; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public String toString() { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java index 37b228d0842e..e07c56d20d66 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.spatial.search.Bound; import org.apache.druid.query.BitmapResultFactory; @@ -30,25 +31,31 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.incremental.SpatialDimensionRowTransformer; +import java.util.Set; + /** */ public class SpatialFilter implements Filter { private final String dimension; private final Bound bound; + private final FilterTuning filterTuning; public SpatialFilter( String dimension, - Bound bound + Bound bound, + FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension"); this.bound = Preconditions.checkNotNull(bound, "bound"); + this.filterTuning = filterTuning; } @Override @@ -69,17 +76,12 @@ public ValueMatcher makeMatcher(ColumnSelectorFactory factory) @Override public Predicate makeStringPredicate() { - return new Predicate() - { - @Override - public boolean apply(String input) - { - if (input == null) { - return false; - } - final float[] coordinate = SpatialDimensionRowTransformer.decode(input); - return bound.contains(coordinate); + return input -> { + if (input == null) { + return false; } + final float[] coordinate = SpatialDimensionRowTransformer.decode(input); + return bound.contains(coordinate); }; } @@ -113,12 +115,24 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return selector.getBitmapIndex(dimension) != null; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return false; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java index 965fc1b659a6..2ae0edfb19e6 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java @@ -26,6 +26,9 @@ import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import java.util.Collections; +import java.util.Set; + /** */ public class TrueFilter implements Filter @@ -52,12 +55,24 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return true; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return true; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return true; } + @Override + public Set getRequiredColumns() + { + return Collections.emptySet(); + } + @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 7f0bc3da3bef..a0a733ec6f62 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -98,7 +98,7 @@ public byte asByte() @JsonCreator public static DictionaryEncodedColumnPartSerde createDeserializer( - @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory, + @JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory, @NotNull @JsonProperty("byteOrder") ByteOrder byteOrder ) { diff --git a/processing/src/main/java/org/apache/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java b/processing/src/main/java/org/apache/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java index 2262184d9f51..e2a5cd213528 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java @@ -42,7 +42,7 @@ public class DoubleNumericColumnPartSerdeV2 implements ColumnPartSerde @JsonCreator public static DoubleNumericColumnPartSerdeV2 getDoubleGenericColumnPartSerde( @JsonProperty("byteOrder") ByteOrder byteOrder, - @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + @JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory ) { return new DoubleNumericColumnPartSerdeV2( diff --git a/processing/src/main/java/org/apache/druid/segment/serde/FloatNumericColumnPartSerdeV2.java b/processing/src/main/java/org/apache/druid/segment/serde/FloatNumericColumnPartSerdeV2.java index b3c71db94a55..77d900dbe2ab 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/FloatNumericColumnPartSerdeV2.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/FloatNumericColumnPartSerdeV2.java @@ -40,7 +40,7 @@ public class FloatNumericColumnPartSerdeV2 implements ColumnPartSerde @JsonCreator public static FloatNumericColumnPartSerdeV2 createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder, - @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + @JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory ) { return new FloatNumericColumnPartSerdeV2( diff --git a/processing/src/main/java/org/apache/druid/segment/serde/LongNumericColumnPartSerdeV2.java b/processing/src/main/java/org/apache/druid/segment/serde/LongNumericColumnPartSerdeV2.java index 7a46c51eb394..75344ec9340c 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/LongNumericColumnPartSerdeV2.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/LongNumericColumnPartSerdeV2.java @@ -40,7 +40,7 @@ public class LongNumericColumnPartSerdeV2 implements ColumnPartSerde @JsonCreator public static LongNumericColumnPartSerdeV2 createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder, - @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + @JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory ) { return new LongNumericColumnPartSerdeV2( diff --git a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java index 9e550903799f..691769aa7ba4 100644 --- a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java +++ b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java @@ -37,7 +37,7 @@ public class InDimFilterSerDesrTest { private static ObjectMapper mapper; - private final String actualInFilter = + private final String serializedFilter = "{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[\"bad\",\"good\"],\"extractionFn\":null}"; @Before @@ -50,7 +50,7 @@ public void setUp() @Test public void testDeserialization() throws IOException { - final InDimFilter actualInDimFilter = mapper.readerFor(DimFilter.class).readValue(actualInFilter); + final InDimFilter actualInDimFilter = mapper.readerFor(DimFilter.class).readValue(serializedFilter); final InDimFilter expectedInDimFilter = new InDimFilter("dimTest", Arrays.asList("good", "bad"), null); Assert.assertEquals(expectedInDimFilter, actualInDimFilter); } @@ -59,8 +59,8 @@ public void testDeserialization() throws IOException public void testSerialization() throws IOException { final InDimFilter dimInFilter = new InDimFilter("dimTest", Arrays.asList("good", "bad"), null); - final String expectedInFilter = mapper.writeValueAsString(dimInFilter); - Assert.assertEquals(expectedInFilter, actualInFilter); + final String actualSerializedFilter = mapper.writeValueAsString(dimInFilter); + Assert.assertEquals(serializedFilter, actualSerializedFilter); } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java index a77aa8e894d3..db4863ad7952 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java @@ -81,13 +81,15 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; public abstract class BaseFilterTest { - private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create( + static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create( ImmutableList.of( new ExpressionVirtualColumn("expr", "1.0 + 0.1", ValueType.FLOAT, TestExprMacroTable.INSTANCE), new ExpressionVirtualColumn("exprDouble", "1.0 + 1.1", ValueType.DOUBLE, TestExprMacroTable.INSTANCE), @@ -396,12 +398,24 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return false; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return false; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return false; } + @Override + public Set getRequiredColumns() + { + return Collections.emptySet(); + } + @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { @@ -458,6 +472,12 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return false; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return false; + } + @Override public VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory) { @@ -470,6 +490,12 @@ public boolean canVectorizeMatcher() return theFilter.canVectorizeMatcher(); } + @Override + public Set getRequiredColumns() + { + return null; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java index caa0f8ab86ac..286b1b632d51 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java @@ -269,6 +269,6 @@ public void testGetRequiredColumn() private static ExpressionDimFilter edf(final String expression) { - return new ExpressionDimFilter(expression, TestExprMacroTable.INSTANCE); + return new ExpressionDimFilter(expression, null, TestExprMacroTable.INSTANCE); } } diff --git a/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java index 1d48b377338c..966b418656bf 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java @@ -43,9 +43,12 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.OrDimFilter; import org.apache.druid.query.filter.SelectorDimFilter; +import org.apache.druid.segment.ColumnSelectorBitmapIndexSelector; import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.QueryableIndexStorageAdapter; import org.apache.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Assert; @@ -72,6 +75,15 @@ public NoBitmapSelectorFilter( super(dimension, value); } + public NoBitmapSelectorFilter( + String dimension, + String value, + FilterTuning filterTuning + ) + { + super(dimension, value, filterTuning); + } + @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) { @@ -727,4 +739,84 @@ public void testDistributeOrCNFExtractionFn() ImmutableList.of("2", "3", "4", "6", "7", "9") ); } + + @Test + public void testAnalyze() + { + if (!(adapter instanceof QueryableIndexStorageAdapter)) { + return; + } + QueryableIndexStorageAdapter storageAdapter = (QueryableIndexStorageAdapter) adapter; + final ColumnSelectorBitmapIndexSelector bitmapIndexSelector = storageAdapter.makeBitmapIndexSelector(BaseFilterTest.VIRTUAL_COLUMNS); + + // has bitmap index, will use it by default + Filter normalFilter = new SelectorFilter("dim1", "HELLO"); + QueryableIndexStorageAdapter.FilterAnalysis filterAnalysisNormal = + storageAdapter.analyzeFilter(normalFilter, bitmapIndexSelector, null); + Assert.assertTrue(filterAnalysisNormal.getPreFilterBitmap() != null); + Assert.assertTrue(filterAnalysisNormal.getPostFilter() == null); + + + // no bitmap index, should be a post filter + Filter noBitmapFilter = new NoBitmapSelectorFilter("dim1", "HELLO"); + QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterAnalysis = + storageAdapter.analyzeFilter(noBitmapFilter, bitmapIndexSelector, null); + Assert.assertTrue(noBitmapFilterAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(noBitmapFilterAnalysis.getPostFilter() != null); + + // this column has a bitmap index, but is forced to not use it + Filter bitmapFilterWithForceNoIndexTuning = new SelectorFilter( + "dim1", + "HELLO", + new FilterTuning(false, null, null) + ); + QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithForceNoIndexTuningAnalysis = + storageAdapter.analyzeFilter(bitmapFilterWithForceNoIndexTuning, bitmapIndexSelector, null); + Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPostFilter() != null); + + // this max cardinality is too low to use bitmap index + Filter bitmapFilterWithCardinalityMax = new SelectorFilter( + "dim1", + "HELLO", + new FilterTuning(true, 0, 3) + ); + QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMaxAnalysis = + storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax, bitmapIndexSelector, null); + Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPostFilter() != null); + + // this max cardinality is high enough that we can still use bitmap index + Filter bitmapFilterWithCardinalityMax2 = new SelectorFilter( + "dim1", + "HELLO", + new FilterTuning(true, 0, 1000) + ); + QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMax2Analysis = + storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax2, bitmapIndexSelector, null); + Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPreFilterBitmap() != null); + Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPostFilter() == null); + + // this min cardinality is too high, will not use bitmap index + Filter bitmapFilterWithCardinalityMin = new SelectorFilter( + "dim1", + "HELLO", + new FilterTuning(true, 1000, null) + ); + QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMinAnalysis = + storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMin, bitmapIndexSelector, null); + Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPostFilter() != null); + + // cannot force using bitmap if there are no bitmaps + Filter noBitmapFilterWithForceUse = new NoBitmapSelectorFilter( + "dim1", + "HELLO", + new FilterTuning(true, null, null) + ); + QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterWithForceUseAnalysis = + storageAdapter.analyzeFilter(noBitmapFilterWithForceUse, bitmapIndexSelector, null); + Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPostFilter() != null); + } } diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index b3302f84fafe..2015690dde87 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -77,6 +77,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; /** @@ -693,12 +694,24 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return true; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return true; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return true; } + @Override + public Set getRequiredColumns() + { + return Collections.emptySet(); + } + private class DictionaryRaceTestFilterDruidPredicateFactory implements DruidPredicateFactory { @Override diff --git a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQuery.java b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQuery.java index 1a09c451e14d..e29cd60ed272 100644 --- a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQuery.java +++ b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQuery.java @@ -46,8 +46,8 @@ public class ClientCompactQuery implements ClientQuery @JsonCreator public ClientCompactQuery( @JsonProperty("dataSource") String dataSource, - @Nullable @JsonProperty("interval") final Interval interval, - @Nullable @JsonProperty("segments") final List segments, + @JsonProperty("interval") @Nullable final Interval interval, + @JsonProperty("segments") @Nullable final List segments, @JsonProperty("targetCompactionSizeBytes") @Nullable Long targetCompactionSizeBytes, @JsonProperty("tuningConfig") ClientCompactQueryTuningConfig tuningConfig, @JsonProperty("context") Map context diff --git a/server/src/main/java/org/apache/druid/client/indexing/IndexingWorkerInfo.java b/server/src/main/java/org/apache/druid/client/indexing/IndexingWorkerInfo.java index b60fbc4218a1..8aa43439dd15 100644 --- a/server/src/main/java/org/apache/druid/client/indexing/IndexingWorkerInfo.java +++ b/server/src/main/java/org/apache/druid/client/indexing/IndexingWorkerInfo.java @@ -46,7 +46,7 @@ public IndexingWorkerInfo( @JsonProperty("availabilityGroups") Set availabilityGroups, @JsonProperty("runningTasks") Collection runningTasks, @JsonProperty("lastCompletedTaskTime") DateTime lastCompletedTaskTime, - @Nullable @JsonProperty("blacklistedUntil") DateTime blacklistedUntil + @JsonProperty("blacklistedUntil") @Nullable DateTime blacklistedUntil ) { this.worker = worker; diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java index 3a904b9e0f74..e11b017e5dca 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java @@ -59,9 +59,9 @@ public NoopSupervisorSpec( @JsonCreator public NoopSupervisorSpec( - @Nullable @JsonProperty("id") String id, - @Nullable @JsonProperty("dataSources") List datasources, - @Nullable @JsonProperty("suspended") Boolean suspended + @JsonProperty("id") @Nullable String id, + @JsonProperty("dataSources") @Nullable List datasources, + @JsonProperty("suspended") @Nullable Boolean suspended ) { this.id = id; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java index 2af0baf33321..b41833d7b332 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java @@ -53,7 +53,8 @@ public static BoundDimFilter not(final BoundDimFilter bound) false, null, bound.getExtractionFn(), - bound.getOrdering() + bound.getOrdering(), + bound.getFilterTuning() ); } else { // bound.getLower() != null @@ -65,7 +66,8 @@ public static BoundDimFilter not(final BoundDimFilter bound) !bound.isLowerStrict(), null, bound.getExtractionFn(), - bound.getOrdering() + bound.getOrdering(), + bound.getFilterTuning() ); } } @@ -113,7 +115,8 @@ public static BoundDimFilter toFilter(final BoundRefKey boundRefKey, final Range range.hasUpperBound() && range.upperBoundType() == BoundType.OPEN, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -127,7 +130,8 @@ public static BoundDimFilter equalTo(final BoundRefKey boundRefKey, final String false, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -141,7 +145,8 @@ public static BoundDimFilter greaterThan(final BoundRefKey boundRefKey, final St false, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -155,7 +160,8 @@ public static BoundDimFilter greaterThanOrEqualTo(final BoundRefKey boundRefKey, false, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -169,7 +175,8 @@ public static BoundDimFilter lessThan(final BoundRefKey boundRefKey, final Strin true, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -183,7 +190,8 @@ public static BoundDimFilter lessThanOrEqualTo(final BoundRefKey boundRefKey, fi false, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -202,7 +210,8 @@ public static BoundDimFilter interval(final BoundRefKey boundRefKey, final Inter true, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java index a594bbf0b6d4..04067db79047 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java @@ -89,7 +89,7 @@ public DimFilter process(DimFilter filter) } } - children.add(new InDimFilter(entry.getKey().getDimension(), values, entry.getKey().getExtractionFn())); + children.add(new InDimFilter(entry.getKey().getDimension(), values, entry.getKey().getExtractionFn(), null)); } }