From 4177dc3f3a07930a9ccd1735d071bb3ea72e03ad Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 31 Jul 2019 16:33:45 -0700 Subject: [PATCH 01/24] add support for mechanism to control filter optimization in historical query processing --- .../druid/query/filter/BloomDimFilter.java | 46 +-- .../sql/BloomFilterOperatorConversion.java | 4 +- .../java/org/apache/druid/query/Druids.java | 2 +- .../FilteredAggregatorFactory.java | 3 +- .../druid/query/filter/BooleanFilter.java | 18 ++ .../druid/query/filter/BoundDimFilter.java | 76 ++--- .../filter/ColumnComparisonDimFilter.java | 39 ++- .../query/filter/ExpressionDimFilter.java | 47 +++- .../org/apache/druid/query/filter/Filter.java | 52 +++- .../druid/query/filter/FilterTuning.java | 99 +++++++ .../druid/query/filter/InDimFilter.java | 82 +++--- .../druid/query/filter/IntervalDimFilter.java | 49 ++-- .../query/filter/JavaScriptDimFilter.java | 66 +++-- .../druid/query/filter/LikeDimFilter.java | 266 +++++++++--------- .../druid/query/filter/RegexDimFilter.java | 41 +-- .../query/filter/SearchQueryDimFilter.java | 47 ++-- .../druid/query/filter/SelectorDimFilter.java | 54 ++-- .../druid/query/filter/SpatialDimFilter.java | 54 ++-- .../segment/QueryableIndexStorageAdapter.java | 16 +- .../druid/segment/filter/BoundFilter.java | 18 ++ .../filter/ColumnComparisonFilter.java | 21 +- .../filter/DimensionPredicateFilter.java | 30 ++ .../segment/filter/ExpressionFilter.java | 19 +- .../apache/druid/segment/filter/InFilter.java | 21 +- .../segment/filter/JavaScriptFilter.java | 23 +- .../druid/segment/filter/LikeFilter.java | 22 +- .../druid/segment/filter/NotFilter.java | 17 ++ .../druid/segment/filter/RegexFilter.java | 7 +- .../segment/filter/SearchQueryFilter.java | 7 +- .../druid/segment/filter/SelectorFilter.java | 29 ++ .../druid/segment/filter/SpatialFilter.java | 23 +- .../druid/segment/filter/TrueFilter.java | 16 ++ .../query/filter/InDimFilterSerDesrTest.java | 2 +- .../druid/segment/filter/BaseFilterTest.java | 31 +- .../segment/filter/ExpressionFilterTest.java | 2 +- .../segment/filter/FilterPartitionTest.java | 91 ++++++ .../IncrementalIndexStorageAdapterTest.java | 16 ++ .../druid/sql/calcite/CalciteQueryTest.java | 4 +- 38 files changed, 1075 insertions(+), 385 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java index f235b59c30de..4c5270825804 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.RangeSet; @@ -32,6 +33,7 @@ import org.apache.druid.segment.filter.DimensionPredicateFilter; import java.util.HashSet; +import java.util.Objects; /** */ @@ -42,12 +44,14 @@ public class BloomDimFilter implements DimFilter private final BloomKFilter bloomKFilter; private final HashCode hash; private final ExtractionFn extractionFn; + private final FilterTuning filterTuning; @JsonCreator public BloomDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("bloomKFilter") BloomKFilterHolder bloomKFilterHolder, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") ExtractionFn extractionFn, + @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -56,6 +60,17 @@ public BloomDimFilter( this.bloomKFilter = bloomKFilterHolder.getFilter(); this.hash = bloomKFilterHolder.getFilterHash(); this.extractionFn = extractionFn; + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public BloomDimFilter( + String dimension, + BloomKFilterHolder bloomKFilterHolder, + ExtractionFn extractionFn + ) + { + this(dimension, bloomKFilterHolder, extractionFn, null); } @Override @@ -152,7 +167,8 @@ public boolean applyNull() }; } }, - extractionFn + extractionFn, + filterTuning ); } @@ -193,16 +209,17 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - BloomDimFilter that = (BloomDimFilter) o; + return dimension.equals(that.dimension) && + hash.equals(that.hash) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); + } - if (!dimension.equals(that.dimension)) { - return false; - } - if (hash != null ? !hash.equals(that.hash) : that.hash != null) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; + @Override + public int hashCode() + { + return Objects.hash(dimension, hash, extractionFn, filterTuning); } @Override @@ -216,13 +233,4 @@ public HashSet getRequiredColumns() { return Sets.newHashSet(dimension); } - - @Override - public int hashCode() - { - int result = dimension.hashCode(); - result = 31 * result + (hash != null ? hash.hashCode() : 0); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; - } } diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java index 60497ff45ec3..64ec379a2e7c 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java @@ -100,7 +100,8 @@ public DimFilter toDruidFilter( return new BloomDimFilter( druidExpression.getSimpleExtraction().getColumn(), holder, - druidExpression.getSimpleExtraction().getExtractionFn() + druidExpression.getSimpleExtraction().getExtractionFn(), + null ); } else if (virtualColumnRegistry != null) { VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( @@ -114,6 +115,7 @@ public DimFilter toDruidFilter( return new BloomDimFilter( virtualColumn.getOutputName(), holder, + null, null ); } else { diff --git a/processing/src/main/java/org/apache/druid/query/Druids.java b/processing/src/main/java/org/apache/druid/query/Druids.java index 47e3ede9a339..e0599663a7d5 100644 --- a/processing/src/main/java/org/apache/druid/query/Druids.java +++ b/processing/src/main/java/org/apache/druid/query/Druids.java @@ -204,7 +204,7 @@ public TimeseriesQueryBuilder filters(String dimensionName, String value) public TimeseriesQueryBuilder filters(String dimensionName, String value, String... values) { - dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null); + dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null, null); return this; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java index 47d175569283..e5a3ef7e28c3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java @@ -243,7 +243,8 @@ public AggregatorFactory optimizeForSegment(PerSegmentQueryOptimizationContext o new IntervalDimFilter( intervalDimFilter.getDimension(), effectiveFilterIntervals, - intervalDimFilter.getExtractionFn() + intervalDimFilter.getExtractionFn(), + intervalDimFilter.getFilterTuning() ), this.name ); diff --git a/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java index a919487c1fbf..2d3fe2ecc071 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java @@ -21,7 +21,9 @@ import org.apache.druid.segment.ColumnSelectorFactory; +import java.util.HashSet; import java.util.List; +import java.util.Set; public interface BooleanFilter extends Filter { @@ -48,4 +50,20 @@ ValueMatcher makeMatcher( ColumnSelectorFactory columnSelectorFactory, RowOffsetMatcherFactory rowOffsetMatcherFactory ); + + @Override + default FilterTuning getManualTuning() + { + return null; + } + + @Override + default Set getRequiredColumns() + { + Set allColumns = new HashSet<>(); + for (Filter f : getFilters()) { + allColumns.addAll(f.getRequiredColumns()); + } + return allColumns; + } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java index 38ff78a46bdd..ef5d798a2d0c 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.collect.BoundType; @@ -56,6 +57,7 @@ public class BoundDimFilter implements DimFilter private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; private final Supplier doublePredicateSupplier; + private final FilterTuning filterTuning; @JsonCreator public BoundDimFilter( @@ -66,7 +68,8 @@ public BoundDimFilter( @JsonProperty("upperStrict") Boolean upperStrict, @Deprecated @JsonProperty("alphaNumeric") Boolean alphaNumeric, @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("ordering") StringComparator ordering + @JsonProperty("ordering") StringComparator ordering, + @JsonProperty("filterTuning") FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be null"); @@ -98,6 +101,22 @@ public BoundDimFilter( this.longPredicateSupplier = makeLongPredicateSupplier(); this.floatPredicateSupplier = makeFloatPredicateSupplier(); this.doublePredicateSupplier = makeDoublePredicateSupplier(); + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public BoundDimFilter( + String dimension, + String lower, + String upper, + Boolean lowerStrict, + Boolean upperStrict, + Boolean alphaNumeric, + ExtractionFn extractionFn, + StringComparator ordering + ) + { + this(dimension, lower, upper, lowerStrict, upperStrict, alphaNumeric, extractionFn, ordering, null); } @JsonProperty @@ -152,6 +171,12 @@ public StringComparator getOrdering() return ordering; } + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + public Supplier getLongPredicateSupplier() { return longPredicateSupplier; @@ -263,43 +288,30 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - BoundDimFilter that = (BoundDimFilter) o; - - if (isLowerStrict() != that.isLowerStrict()) { - return false; - } - if (isUpperStrict() != that.isUpperStrict()) { - return false; - } - if (!getDimension().equals(that.getDimension())) { - return false; - } - if (getUpper() != null ? !getUpper().equals(that.getUpper()) : that.getUpper() != null) { - return false; - } - if (getLower() != null ? !getLower().equals(that.getLower()) : that.getLower() != null) { - return false; - } - if (getExtractionFn() != null - ? !getExtractionFn().equals(that.getExtractionFn()) - : that.getExtractionFn() != null) { - return false; - } - return getOrdering().equals(that.getOrdering()); + return lowerStrict == that.lowerStrict && + upperStrict == that.upperStrict && + dimension.equals(that.dimension) && + Objects.equals(upper, that.upper) && + Objects.equals(lower, that.lower) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(ordering, that.ordering) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = getDimension().hashCode(); - result = 31 * result + (getUpper() != null ? getUpper().hashCode() : 0); - result = 31 * result + (getLower() != null ? getLower().hashCode() : 0); - result = 31 * result + (isLowerStrict() ? 1 : 0); - result = 31 * result + (isUpperStrict() ? 1 : 0); - result = 31 * result + (getExtractionFn() != null ? getExtractionFn().hashCode() : 0); - result = 31 * result + getOrdering().hashCode(); - return result; + return Objects.hash( + dimension, + upper, + lower, + lowerStrict, + upperStrict, + extractionFn, + ordering, + filterTuning + ); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java index 1d7cb502210f..8fe5dfbced47 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.RangeSet; @@ -31,6 +32,7 @@ import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -40,14 +42,25 @@ public class ColumnComparisonDimFilter implements DimFilter private static final Joiner COMMA_JOINER = Joiner.on(", "); private final List dimensions; + private final FilterTuning filterTuning; @JsonCreator public ColumnComparisonDimFilter( - @JsonProperty("dimensions") List dimensions + @JsonProperty("dimensions") List dimensions, + @JsonProperty("filterTuning") FilterTuning filterTuning ) { this.dimensions = Preconditions.checkNotNull(dimensions, "dimensions"); Preconditions.checkArgument(dimensions.size() >= 2, "dimensions must have a least 2 dimensions"); + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public ColumnComparisonDimFilter( + List dimensions + ) + { + this(dimensions, null); } @Override @@ -68,7 +81,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - return new ColumnComparisonFilter(dimensions); + return new ColumnComparisonFilter(dimensions, filterTuning); } @JsonProperty @@ -77,11 +90,18 @@ public List getDimensions() return dimensions; } + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public String toString() { return "ColumnComparisonDimFilter{" + "dimensions=[" + COMMA_JOINER.join(dimensions) + "]" + + ", filterTuning=" + filterTuning + "}"; } @@ -94,10 +114,15 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - ColumnComparisonDimFilter that = (ColumnComparisonDimFilter) o; + return dimensions.equals(that.dimensions) && + Objects.equals(filterTuning, that.filterTuning); + } - return dimensions.equals(that.dimensions); + @Override + public int hashCode() + { + return Objects.hash(dimensions, filterTuning); } @Override @@ -114,10 +139,4 @@ public HashSet getRequiredColumns() .collect(Collectors.toSet()) ); } - - @Override - public int hashCode() - { - return 31 * dimensions.hashCode(); - } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java index 0a67652ff6e6..a64689991648 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.RangeSet; @@ -39,23 +40,41 @@ public class ExpressionDimFilter implements DimFilter { private final String expression; private final Supplier parsed; + private final FilterTuning filterTuning; @JsonCreator public ExpressionDimFilter( @JsonProperty("expression") final String expression, + @JsonProperty("filterTuning") final FilterTuning filterTuning, @JacksonInject ExprMacroTable macroTable ) { this.expression = expression; + this.filterTuning = filterTuning; this.parsed = Suppliers.memoize(() -> Parser.parse(expression, macroTable)); } + @VisibleForTesting + public ExpressionDimFilter( + final String expression, + ExprMacroTable macroTable + ) + { + this(expression, null, macroTable); + } + @JsonProperty public String getExpression() { return expression; } + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public DimFilter optimize() { @@ -65,7 +84,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - return new ExpressionFilter(parsed); + return new ExpressionFilter(parsed, filterTuning); } @Override @@ -89,7 +108,16 @@ public byte[] getCacheKey() } @Override - public boolean equals(final Object o) + public String toString() + { + return "ExpressionDimFilter{" + + "expression='" + expression + '\'' + + ", filterTuning=" + filterTuning + + '}'; + } + + @Override + public boolean equals(Object o) { if (this == o) { return true; @@ -97,21 +125,14 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final ExpressionDimFilter that = (ExpressionDimFilter) o; - return Objects.equals(expression, that.expression); + ExpressionDimFilter that = (ExpressionDimFilter) o; + return expression.equals(that.expression) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - return Objects.hash(expression); - } - - @Override - public String toString() - { - return "ExpressionDimFilter{" + - "expression='" + expression + '\'' + - '}'; + return Objects.hash(expression, filterTuning); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java index 120358f30447..9b518fb1b029 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java @@ -28,6 +28,9 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import javax.annotation.Nullable; +import java.util.Set; + public interface Filter { /** @@ -109,7 +112,6 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory */ boolean supportsBitmapIndex(BitmapIndexSelector selector); - /** * Indicates whether this filter supports selectivity estimation. * A filter supports selectivity estimation if it supports bitmap index and @@ -129,4 +131,52 @@ default boolean canVectorizeMatcher() { return false; } + + /** + * Set of columns used by a filter + */ + Set getRequiredColumns(); + + /** + * "Manual" {@link FilterTuning} to allow explicit control of filter optimization behavior. This will likely be + * supplied by the {@link DimFilter} that is creating this {@link Filter} + */ + @Nullable + FilterTuning getManualTuning(); + + /** + * This method allows a filter to automatically compute a {@link FilterTuning} based on information it can gather + * from a {@link BitmapIndexSelector}. This method makes sense to override if the default implementation of + * {@link #shouldUseIndex(BitmapIndexSelector)} is sufficient for filter optimization. By default, creates a + * {@link FilterTuning} with no limits that will always use a bitmap index if + * {@link #supportsBitmapIndex(BitmapIndexSelector)} is true, unless a {@link FilterTuning} is provided by + * {@link #getManualTuning()} + */ + default FilterTuning computeTuning(BitmapIndexSelector selector) + { + final FilterTuning manual = getManualTuning(); + return manual != null ? manual : FilterTuning.createDefault(supportsBitmapIndex(selector)); + } + + /** + * Determine if a filter *should* use a bitmap index based on information collected from the supplied + * {@link BitmapIndexSelector}. This method differs from {@link #supportsBitmapIndex(BitmapIndexSelector)} in that + * the former only indicates if a bitmap index is available and {@link #getBitmapIndex(BitmapIndexSelector)} may be + * used. This method, by default, will consider a {@link FilterTuning} to make decisions about when to use an + * available index. Override this method in a {@link Filter} implementation when {@link FilterTuning} alone is not + * adequate for making this decision. + */ + default boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + { + if (supportsBitmapIndex(bitmapIndexSelector)) { + final FilterTuning tuning = computeTuning(bitmapIndexSelector); + return tuning.getUseIndex() + && (getRequiredColumns().size() == 0 || getRequiredColumns().stream().allMatch(column -> { + final int cardinality = bitmapIndexSelector.getBitmapIndex(column).getCardinality(); + return cardinality >= tuning.getUseIndexMinCardinalityThreshold() + && cardinality <= tuning.getUseIndexMaxCardinalityThreshold(); + })); + } + return false; + } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java new file mode 100644 index 000000000000..6ce30b174e62 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; +import java.util.Objects; + +public class FilterTuning +{ + public static FilterTuning createDefault(boolean useIndex) + { + return new FilterTuning(useIndex, 0, Integer.MAX_VALUE); + } + + private final Boolean useIndex; + private final Integer useIndexMinCardinalityThreshold; + private final Integer useIndexMaxCardinalityThreshold; + + @JsonCreator + public FilterTuning( + @Nullable @JsonProperty("useIndex") Boolean useIndex, + @Nullable @JsonProperty("useIndexMinCardinalityThreshold") Integer useIndexMinCardinalityThreshold, + @Nullable @JsonProperty("useIndexMaximumCardinalityThreshold") Integer useIndexMaxCardinalityThreshold + ) + { + this.useIndex = useIndex; + this.useIndexMinCardinalityThreshold = useIndexMinCardinalityThreshold; + this.useIndexMaxCardinalityThreshold = useIndexMaxCardinalityThreshold; + } + + @JsonProperty + public Boolean getUseIndex() + { + return useIndex; + } + + @JsonProperty + public Integer getUseIndexMinCardinalityThreshold() + { + return useIndexMinCardinalityThreshold != null ? useIndexMinCardinalityThreshold : 0; + } + + @JsonProperty + public Integer getUseIndexMaxCardinalityThreshold() + { + return useIndexMaxCardinalityThreshold != null ? useIndexMaxCardinalityThreshold : Integer.MAX_VALUE; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FilterTuning that = (FilterTuning) o; + return Objects.equals(useIndex, that.useIndex) && + Objects.equals(useIndexMinCardinalityThreshold, that.useIndexMinCardinalityThreshold) && + Objects.equals(useIndexMaxCardinalityThreshold, that.useIndexMaxCardinalityThreshold); + } + + @Override + public int hashCode() + { + return Objects.hash(useIndex, useIndexMinCardinalityThreshold, useIndexMaxCardinalityThreshold); + } + + @Override + public String toString() + { + return "FilterTuning{" + + "useIndex=" + useIndex + + ", useIndexMinCardinalityThreshold=" + useIndexMinCardinalityThreshold + + ", useIndexMaxCardinalityThreshold=" + useIndexMaxCardinalityThreshold + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java index 86213eb32166..d1ba394cf50a 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; @@ -65,6 +66,7 @@ public class InDimFilter implements DimFilter private final SortedSet values; private final String dimension; private final ExtractionFn extractionFn; + private final FilterTuning filterTuning; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; private final Supplier doublePredicateSupplier; @@ -73,7 +75,8 @@ public class InDimFilter implements DimFilter public InDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("values") Collection values, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") ExtractionFn extractionFn, + @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkNotNull(dimension, "dimension can not be null"); @@ -85,11 +88,22 @@ public InDimFilter( } this.dimension = dimension; this.extractionFn = extractionFn; + this.filterTuning = filterTuning; this.longPredicateSupplier = getLongPredicateSupplier(); this.floatPredicateSupplier = getFloatPredicateSupplier(); this.doublePredicateSupplier = getDoublePredicateSupplier(); } + @VisibleForTesting + public InDimFilter( + String dimension, + Collection values, + ExtractionFn extractionFn + ) + { + this(dimension, values, extractionFn, null); + } + @JsonProperty public String getDimension() { @@ -108,6 +122,12 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { @@ -133,7 +153,7 @@ public DimFilter optimize() { InDimFilter inFilter = optimizeLookup(); if (inFilter.values.size() == 1) { - return new SelectorDimFilter(inFilter.dimension, inFilter.values.first(), inFilter.getExtractionFn()); + return new SelectorDimFilter(inFilter.dimension, inFilter.values.first(), inFilter.getExtractionFn(), filterTuning); } return inFilter; } @@ -169,7 +189,7 @@ private InDimFilter optimizeLookup() if (keys.isEmpty()) { return this; } else { - return new InDimFilter(dimension, keys, null); + return new InDimFilter(dimension, keys, null, filterTuning); } } return this; @@ -184,7 +204,8 @@ public Filter toFilter() longPredicateSupplier, floatPredicateSupplier, doublePredicateSupplier, - extractionFn + extractionFn, + filterTuning ); } @@ -215,37 +236,6 @@ public HashSet getRequiredColumns() return Sets.newHashSet(dimension); } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - InDimFilter that = (InDimFilter) o; - - if (values != null ? !values.equals(that.values) : that.values != null) { - return false; - } - if (!dimension.equals(that.dimension)) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - - } - - @Override - public int hashCode() - { - int result = values != null ? values.hashCode() : 0; - result = 31 * result + dimension.hashCode(); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; - } - @Override public String toString() { @@ -271,6 +261,28 @@ public String toString() return builder.toString(); } + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + InDimFilter that = (InDimFilter) o; + return values.equals(that.values) && + dimension.equals(that.dimension) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); + } + + @Override + public int hashCode() + { + return Objects.hash(values, dimension, extractionFn, filterTuning); + } + // As the set of filtered values can be large, parsing them as longs should be done only if needed, and only once. // Pass in a common long predicate supplier to all filters created by .toFilter(), so that // we only compute the long hashset/array once per query. diff --git a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java index eec5ccaa2a41..9e4510360b5f 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; @@ -37,6 +38,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; public class IntervalDimFilter implements DimFilter { @@ -45,12 +47,14 @@ public class IntervalDimFilter implements DimFilter private final String dimension; private final ExtractionFn extractionFn; private final OrDimFilter convertedFilter; + private final FilterTuning filterTuning; @JsonCreator public IntervalDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("intervals") List intervals, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") ExtractionFn extractionFn, + @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkNotNull(dimension, "dimension can not be null"); @@ -59,10 +63,21 @@ public IntervalDimFilter( this.dimension = dimension; this.intervals = Collections.unmodifiableList(JodaUtils.condenseIntervals(intervals)); this.extractionFn = extractionFn; + this.filterTuning = filterTuning; this.intervalLongs = makeIntervalLongs(); this.convertedFilter = new OrDimFilter(makeBoundDimFilters()); } + @VisibleForTesting + public IntervalDimFilter( + String dimension, + List intervals, + ExtractionFn extractionFn + ) + { + this(dimension, intervals, extractionFn, null); + } + @JsonProperty public String getDimension() { @@ -81,6 +96,12 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { @@ -139,28 +160,17 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - IntervalDimFilter that = (IntervalDimFilter) o; - - if (!getIntervals().equals(that.getIntervals())) { - return false; - } - if (!getDimension().equals(that.getDimension())) { - return false; - } - return getExtractionFn() != null - ? getExtractionFn().equals(that.getExtractionFn()) - : that.getExtractionFn() == null; - + return intervals.equals(that.intervals) && + dimension.equals(that.dimension) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = getIntervals().hashCode(); - result = 31 * result + getDimension().hashCode(); - result = 31 * result + (getExtractionFn() != null ? getExtractionFn().hashCode() : 0); - return result; + return Objects.hash(intervals, dimension, extractionFn, filterTuning); } @Override @@ -173,7 +183,7 @@ private List> makeIntervalLongs() { List> intervalLongs = new ArrayList<>(); for (Interval interval : intervals) { - intervalLongs.add(new Pair(interval.getStartMillis(), interval.getEndMillis())); + intervalLongs.add(new Pair<>(interval.getStartMillis(), interval.getEndMillis())); } return intervalLongs; } @@ -190,7 +200,8 @@ private List makeBoundDimFilters() true, null, extractionFn, - StringComparators.NUMERIC + StringComparators.NUMERIC, + filterTuning ); boundDimFilters.add(boundDimFilter); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java index b211e38ad78d..c322b8aa3027 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.RangeSet; @@ -38,6 +39,7 @@ import java.nio.ByteBuffer; import java.util.HashSet; +import java.util.Objects; public class JavaScriptDimFilter implements DimFilter { @@ -45,6 +47,7 @@ public class JavaScriptDimFilter implements DimFilter private final String function; private final ExtractionFn extractionFn; private final JavaScriptConfig config; + private final FilterTuning filterTuning; /** * The field is declared volatile in order to ensure safe publication of the object @@ -62,6 +65,7 @@ public JavaScriptDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("function") String function, @JsonProperty("extractionFn") ExtractionFn extractionFn, + @JsonProperty("filterTuning") FilterTuning filterTuning, @JacksonInject JavaScriptConfig config ) { @@ -70,9 +74,21 @@ public JavaScriptDimFilter( this.dimension = dimension; this.function = function; this.extractionFn = extractionFn; + this.filterTuning = filterTuning; this.config = config; } + @VisibleForTesting + public JavaScriptDimFilter( + String dimension, + String function, + ExtractionFn extractionFn, + JavaScriptConfig config + ) + { + this(dimension, function, extractionFn, null, config); + } + @JsonProperty public String getDimension() { @@ -91,6 +107,12 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { @@ -118,7 +140,7 @@ public DimFilter optimize() public Filter toFilter() { JavaScriptPredicateFactory predicateFactory = getPredicateFactory(); - return new JavaScriptFilter(dimension, predicateFactory); + return new JavaScriptFilter(dimension, predicateFactory, filterTuning); } /** @@ -157,45 +179,37 @@ public HashSet getRequiredColumns() return Sets.newHashSet(dimension); } - @Override - public String toString() - { - return "JavaScriptDimFilter{" + - "dimension='" + dimension + '\'' + - ", function='" + function + '\'' + - ", extractionFn='" + extractionFn + '\'' + - '}'; - } - @Override public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof JavaScriptDimFilter)) { + if (o == null || getClass() != o.getClass()) { return false; } - JavaScriptDimFilter that = (JavaScriptDimFilter) o; - - if (!dimension.equals(that.dimension)) { - return false; - } - if (!function.equals(that.function)) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - + return dimension.equals(that.dimension) && + function.equals(that.function) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = dimension.hashCode(); - result = 31 * result + function.hashCode(); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; + return Objects.hash(dimension, function, extractionFn, filterTuning); + } + + @Override + public String toString() + { + return "JavaScriptDimFilter{" + + "dimension='" + dimension + '\'' + + ", function='" + function + '\'' + + ", extractionFn=" + extractionFn + + ", filterTuning=" + filterTuning + + '}'; } public static class JavaScriptPredicateFactory implements DruidPredicateFactory diff --git a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java index e7ec2fcc2d62..cc2c536f6ec2 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.RangeSet; @@ -36,6 +37,7 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.HashSet; +import java.util.Objects; import java.util.regex.Pattern; public class LikeDimFilter implements DimFilter @@ -50,18 +52,21 @@ public class LikeDimFilter implements DimFilter private final Character escapeChar; private final ExtractionFn extractionFn; private final LikeMatcher likeMatcher; + private final FilterTuning filterTuning; @JsonCreator public LikeDimFilter( @JsonProperty("dimension") final String dimension, @JsonProperty("pattern") final String pattern, @JsonProperty("escape") final String escape, - @JsonProperty("extractionFn") final ExtractionFn extractionFn + @JsonProperty("extractionFn") final ExtractionFn extractionFn, + @JsonProperty("filterTuning") final FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension"); this.pattern = Preconditions.checkNotNull(pattern, "pattern"); this.extractionFn = extractionFn; + this.filterTuning = filterTuning; if (escape != null && escape.length() != 1) { throw new IllegalArgumentException("Escape must be null or a single character"); @@ -72,6 +77,138 @@ public LikeDimFilter( this.likeMatcher = LikeMatcher.from(pattern, this.escapeChar); } + @VisibleForTesting + public LikeDimFilter( + final String dimension, + final String pattern, + final String escape, + final ExtractionFn extractionFn + ) + { + this(dimension, pattern, escape, extractionFn, null); + } + + @JsonProperty + public String getDimension() + { + return dimension; + } + + @JsonProperty + public String getPattern() + { + return pattern; + } + + @JsonProperty + public String getEscape() + { + return escapeChar != null ? escapeChar.toString() : null; + } + + @JsonProperty + public ExtractionFn getExtractionFn() + { + return extractionFn; + } + + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + + @Override + public byte[] getCacheKey() + { + final byte[] dimensionBytes = StringUtils.toUtf8(dimension); + final byte[] patternBytes = StringUtils.toUtf8(pattern); + final byte[] escapeBytes = escapeChar == null ? new byte[0] : Chars.toByteArray(escapeChar); + final byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); + final int sz = 4 + dimensionBytes.length + patternBytes.length + escapeBytes.length + extractionFnBytes.length; + return ByteBuffer.allocate(sz) + .put(DimFilterUtils.LIKE_CACHE_ID) + .put(dimensionBytes) + .put(DimFilterUtils.STRING_SEPARATOR) + .put(patternBytes) + .put(DimFilterUtils.STRING_SEPARATOR) + .put(escapeBytes) + .put(DimFilterUtils.STRING_SEPARATOR) + .put(extractionFnBytes) + .array(); + } + + @Override + public DimFilter optimize() + { + return this; + } + + @Override + public Filter toFilter() + { + return new LikeFilter(dimension, extractionFn, likeMatcher, filterTuning); + } + + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + + @Override + public HashSet getRequiredColumns() + { + return Sets.newHashSet(dimension); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LikeDimFilter that = (LikeDimFilter) o; + return dimension.equals(that.dimension) && + pattern.equals(that.pattern) && + Objects.equals(escapeChar, that.escapeChar) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); + } + + @Override + public int hashCode() + { + return Objects.hash(dimension, pattern, escapeChar, extractionFn, filterTuning); + } + + @Override + public String toString() + { + final StringBuilder builder = new StringBuilder(); + + if (extractionFn != null) { + builder.append(extractionFn).append("("); + } + + builder.append(dimension); + + if (extractionFn != null) { + builder.append(")"); + } + + builder.append(" LIKE '").append(pattern).append("'"); + + if (escapeChar != null) { + builder.append(" ESCAPE '").append(escapeChar).append("'"); + } + + return builder.toString(); + } + public static class LikeMatcher { public enum SuffixMatch @@ -232,131 +369,4 @@ public SuffixMatch getSuffixMatch() return suffixMatch; } } - - @JsonProperty - public String getDimension() - { - return dimension; - } - - @JsonProperty - public String getPattern() - { - return pattern; - } - - @JsonProperty - public String getEscape() - { - return escapeChar != null ? escapeChar.toString() : null; - } - - @JsonProperty - public ExtractionFn getExtractionFn() - { - return extractionFn; - } - - @Override - public byte[] getCacheKey() - { - final byte[] dimensionBytes = StringUtils.toUtf8(dimension); - final byte[] patternBytes = StringUtils.toUtf8(pattern); - final byte[] escapeBytes = escapeChar == null ? new byte[0] : Chars.toByteArray(escapeChar); - final byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - final int sz = 4 + dimensionBytes.length + patternBytes.length + escapeBytes.length + extractionFnBytes.length; - return ByteBuffer.allocate(sz) - .put(DimFilterUtils.LIKE_CACHE_ID) - .put(dimensionBytes) - .put(DimFilterUtils.STRING_SEPARATOR) - .put(patternBytes) - .put(DimFilterUtils.STRING_SEPARATOR) - .put(escapeBytes) - .put(DimFilterUtils.STRING_SEPARATOR) - .put(extractionFnBytes) - .array(); - } - - @Override - public DimFilter optimize() - { - return this; - } - - @Override - public Filter toFilter() - { - return new LikeFilter(dimension, extractionFn, likeMatcher); - } - - @Override - public RangeSet getDimensionRangeSet(String dimension) - { - return null; - } - - @Override - public HashSet getRequiredColumns() - { - return Sets.newHashSet(dimension); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - LikeDimFilter that = (LikeDimFilter) o; - - if (dimension != null ? !dimension.equals(that.dimension) : that.dimension != null) { - return false; - } - if (pattern != null ? !pattern.equals(that.pattern) : that.pattern != null) { - return false; - } - if (escapeChar != null ? !escapeChar.equals(that.escapeChar) : that.escapeChar != null) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - - } - - @Override - public int hashCode() - { - int result = dimension != null ? dimension.hashCode() : 0; - result = 31 * result + (pattern != null ? pattern.hashCode() : 0); - result = 31 * result + (escapeChar != null ? escapeChar.hashCode() : 0); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; - } - - @Override - public String toString() - { - final StringBuilder builder = new StringBuilder(); - - if (extractionFn != null) { - builder.append(extractionFn).append("("); - } - - builder.append(dimension); - - if (extractionFn != null) { - builder.append(")"); - } - - builder.append(" LIKE '").append(pattern).append("'"); - - if (escapeChar != null) { - builder.append(" ESCAPE '").append(escapeChar).append("'"); - } - - return builder.toString(); - } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java index adb52ba614bc..95312553e938 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; @@ -30,6 +31,7 @@ import java.nio.ByteBuffer; import java.util.HashSet; +import java.util.Objects; import java.util.regex.Pattern; /** @@ -39,6 +41,7 @@ public class RegexDimFilter implements DimFilter private final String dimension; private final String pattern; private final ExtractionFn extractionFn; + private final FilterTuning filterTuning; private final Pattern compiledPattern; @@ -46,7 +49,8 @@ public class RegexDimFilter implements DimFilter public RegexDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("pattern") String pattern, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") ExtractionFn extractionFn, + @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -55,6 +59,17 @@ public RegexDimFilter( this.pattern = pattern; this.extractionFn = extractionFn; this.compiledPattern = Pattern.compile(pattern); + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public RegexDimFilter( + String dimension, + String pattern, + ExtractionFn extractionFn + ) + { + this(dimension, pattern, extractionFn, null); } @JsonProperty @@ -101,7 +116,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - return new RegexFilter(dimension, compiledPattern, extractionFn); + return new RegexFilter(dimension, compiledPattern, extractionFn, filterTuning); } @Override @@ -123,6 +138,7 @@ public String toString() "dimension='" + dimension + '\'' + ", pattern='" + pattern + '\'' + ", extractionFn='" + extractionFn + '\'' + + ", filterTuning=" + filterTuning + '}'; } @@ -132,28 +148,19 @@ public boolean equals(Object o) if (this == o) { return true; } - if (!(o instanceof RegexDimFilter)) { + if (o == null || getClass() != o.getClass()) { return false; } - RegexDimFilter that = (RegexDimFilter) o; - - if (!dimension.equals(that.dimension)) { - return false; - } - if (!pattern.equals(that.pattern)) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - + return dimension.equals(that.dimension) && + pattern.equals(that.pattern) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = dimension.hashCode(); - result = 31 * result + pattern.hashCode(); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; + return Objects.hash(dimension, pattern, extractionFn, filterTuning); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java index 67d78b6f87fc..417f56ff2dbc 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; @@ -30,6 +31,7 @@ import java.nio.ByteBuffer; import java.util.HashSet; +import java.util.Objects; /** */ @@ -38,11 +40,13 @@ public class SearchQueryDimFilter implements DimFilter private final String dimension; private final SearchQuerySpec query; private final ExtractionFn extractionFn; + private final FilterTuning filterTuning; public SearchQueryDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("query") SearchQuerySpec query, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") ExtractionFn extractionFn, + @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -51,6 +55,17 @@ public SearchQueryDimFilter( this.dimension = dimension; this.query = query; this.extractionFn = extractionFn; + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public SearchQueryDimFilter( + String dimension, + SearchQuerySpec query, + ExtractionFn extractionFn + ) + { + this(dimension, query, extractionFn, null); } @JsonProperty @@ -71,6 +86,12 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { @@ -97,7 +118,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - return new SearchQueryFilter(dimension, query, extractionFn); + return new SearchQueryFilter(dimension, query, extractionFn, filterTuning); } @Override @@ -119,6 +140,7 @@ public String toString() "dimension='" + dimension + '\'' + ", query=" + query + ", extractionFn='" + extractionFn + '\'' + + ", filterTuning=" + filterTuning + '}'; } @@ -128,28 +150,19 @@ public boolean equals(Object o) if (this == o) { return true; } - if (!(o instanceof SearchQueryDimFilter)) { + if (o == null || getClass() != o.getClass()) { return false; } - SearchQueryDimFilter that = (SearchQueryDimFilter) o; - - if (!dimension.equals(that.dimension)) { - return false; - } - if (!query.equals(that.query)) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - + return dimension.equals(that.dimension) && + query.equals(that.query) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = dimension.hashCode(); - result = 31 * result + query.hashCode(); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; + return Objects.hash(dimension, query, extractionFn, filterTuning); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java index 2c71ccc89608..d917cb345cd8 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; @@ -52,6 +53,7 @@ public class SelectorDimFilter implements DimFilter @Nullable private final String value; private final ExtractionFn extractionFn; + private final FilterTuning filterTuning; private final Object initLock = new Object(); @@ -63,7 +65,8 @@ public class SelectorDimFilter implements DimFilter public SelectorDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("value") String value, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("extractionFn") ExtractionFn extractionFn, + @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -71,6 +74,17 @@ public SelectorDimFilter( this.dimension = dimension; this.value = NullHandling.emptyToNullIfNeeded(value); this.extractionFn = extractionFn; + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public SelectorDimFilter( + String dimension, + String value, + ExtractionFn extractionFn + ) + { + this(dimension, value, extractionFn, null); } @Override @@ -90,14 +104,14 @@ public byte[] getCacheKey() @Override public DimFilter optimize() { - return new InDimFilter(dimension, Collections.singletonList(value), extractionFn).optimize(); + return new InDimFilter(dimension, Collections.singletonList(value), extractionFn, filterTuning).optimize(); } @Override public Filter toFilter() { if (extractionFn == null) { - return new SelectorFilter(dimension, value); + return new SelectorFilter(dimension, value, filterTuning); } else { final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() @@ -129,7 +143,7 @@ public DruidDoublePredicate makeDoublePredicate() return druidDoublePredicate; } }; - return new DimensionPredicateFilter(dimension, predicateFactory, extractionFn); + return new DimensionPredicateFilter(dimension, predicateFactory, extractionFn, filterTuning); } } @@ -151,6 +165,12 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public String toString() { @@ -170,16 +190,17 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - SelectorDimFilter that = (SelectorDimFilter) o; + return dimension.equals(that.dimension) && + Objects.equals(value, that.value) && + Objects.equals(extractionFn, that.extractionFn) && + Objects.equals(filterTuning, that.filterTuning); + } - if (!dimension.equals(that.dimension)) { - return false; - } - if (value != null ? !value.equals(that.value) : that.value != null) { - return false; - } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; + @Override + public int hashCode() + { + return Objects.hash(dimension, value, extractionFn, filterTuning); } @Override @@ -206,15 +227,6 @@ public HashSet getRequiredColumns() return Sets.newHashSet(dimension); } - @Override - public int hashCode() - { - int result = dimension.hashCode(); - result = 31 * result + (value != null ? value.hashCode() : 0); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); - return result; - } - private void initLongPredicate() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java index 0d12ac5979e8..ae9b36cdb83f 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; @@ -30,6 +31,7 @@ import java.nio.ByteBuffer; import java.util.HashSet; +import java.util.Objects; /** */ @@ -37,11 +39,13 @@ public class SpatialDimFilter implements DimFilter { private final String dimension; private final Bound bound; + private final FilterTuning filterTuning; @JsonCreator public SpatialDimFilter( @JsonProperty("dimension") String dimension, - @JsonProperty("bound") Bound bound + @JsonProperty("bound") Bound bound, + @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -49,6 +53,16 @@ public SpatialDimFilter( this.dimension = dimension; this.bound = bound; + this.filterTuning = filterTuning; + } + + @VisibleForTesting + public SpatialDimFilter( + String dimension, + Bound bound + ) + { + this(dimension, bound, null); } @Override @@ -86,7 +100,7 @@ public Bound getBound() @Override public Filter toFilter() { - return new SpatialFilter(dimension, bound); + return new SpatialFilter(dimension, bound, filterTuning); } @Override @@ -101,6 +115,16 @@ public HashSet getRequiredColumns() return Sets.newHashSet(dimension); } + @Override + public String toString() + { + return "SpatialDimFilter{" + + "dimension='" + dimension + '\'' + + ", bound=" + bound + + ", filterTuning=" + filterTuning + + '}'; + } + @Override public boolean equals(Object o) { @@ -110,33 +134,15 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - SpatialDimFilter that = (SpatialDimFilter) o; - - if (bound != null ? !bound.equals(that.bound) : that.bound != null) { - return false; - } - if (dimension != null ? !dimension.equals(that.dimension) : that.dimension != null) { - return false; - } - - return true; + return dimension.equals(that.dimension) && + bound.equals(that.bound) && + Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - int result = dimension != null ? dimension.hashCode() : 0; - result = 31 * result + (bound != null ? bound.hashCode() : 0); - return result; - } - - @Override - public String toString() - { - return "SpatialDimFilter{" + - "dimension='" + dimension + '\'' + - ", bound=" + bound + - '}'; + return Objects.hash(dimension, bound, filterTuning); } } diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java index d055b4bb2f64..58f215550785 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.java.util.common.DateTimes; @@ -347,7 +348,8 @@ private Interval computeCursorInterval(final Granularity gran, final Interval in return interval.overlap(dataInterval); } - private ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualColumns virtualColumns) + @VisibleForTesting + public ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualColumns virtualColumns) { return new ColumnSelectorBitmapIndexSelector( index.getBitmapFactoryForDimensions(), @@ -356,7 +358,8 @@ private ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualC ); } - private FilterAnalysis analyzeFilter( + @VisibleForTesting + public FilterAnalysis analyzeFilter( @Nullable final Filter filter, ColumnSelectorBitmapIndexSelector bitmapIndexSelector, @Nullable QueryMetrics queryMetrics @@ -389,7 +392,9 @@ private FilterAnalysis analyzeFilter( if (filter instanceof AndFilter) { // If we get an AndFilter, we can split the subfilters across both filtering stages for (Filter subfilter : ((AndFilter) filter).getFilters()) { - if (subfilter.supportsBitmapIndex(bitmapIndexSelector)) { + + if (subfilter.shouldUseIndex(bitmapIndexSelector)) { + preFilters.add(subfilter); } else { postFilters.add(subfilter); @@ -397,7 +402,7 @@ private FilterAnalysis analyzeFilter( } } else { // If we get an OrFilter or a single filter, handle the filter in one stage - if (filter.supportsBitmapIndex(bitmapIndexSelector)) { + if (filter.shouldUseIndex(bitmapIndexSelector)) { preFilters.add(filter); } else { postFilters.add(filter); @@ -442,7 +447,8 @@ private FilterAnalysis analyzeFilter( return new FilterAnalysis(preFilterBitmap, postFilter); } - private static class FilterAnalysis + @VisibleForTesting + public static class FilterAnalysis { private final Filter postFilter; private final ImmutableBitmap preFilterBitmap; diff --git a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java index 28b8548f9d73..87194ef80689 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java @@ -34,6 +34,7 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer; @@ -45,13 +46,16 @@ import org.apache.druid.segment.column.BitmapIndex; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import javax.annotation.Nullable; import java.util.Comparator; +import java.util.Set; public class BoundFilter implements Filter { private final BoundDimFilter boundDimFilter; private final Comparator comparator; private final ExtractionFn extractionFn; + private final FilterTuning manualFilterTuning; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; @@ -65,6 +69,7 @@ public BoundFilter(final BoundDimFilter boundDimFilter) this.longPredicateSupplier = boundDimFilter.getLongPredicateSupplier(); this.floatPredicateSupplier = boundDimFilter.getFloatPredicateSupplier(); this.doublePredicateSupplier = boundDimFilter.getDoublePredicateSupplier(); + this.manualFilterTuning = boundDimFilter.getFilterTuning(); } @Override @@ -150,6 +155,19 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return selector.getBitmapIndex(boundDimFilter.getDimension()) != null; } + @Override + public Set getRequiredColumns() + { + return boundDimFilter.getRequiredColumns(); + } + + @Nullable + @Override + public FilterTuning getManualTuning() + { + return manualFilterTuning; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java index eea5bcd74dce..c0359dc816e6 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java @@ -25,6 +25,7 @@ import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueGetter; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.ValueMatcherColumnSelectorStrategy; @@ -37,18 +38,23 @@ import javax.annotation.Nullable; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** */ public class ColumnComparisonFilter implements Filter { private final List dimensions; + private final FilterTuning manualFilterTuning; public ColumnComparisonFilter( - final List dimensions + final List dimensions, + final FilterTuning filterTuning ) { this.dimensions = Preconditions.checkNotNull(dimensions, "dimensions"); + this.manualFilterTuning = filterTuning; } @Override @@ -141,12 +147,25 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return false; } + @Nullable + @Override + public FilterTuning getManualTuning() + { + return manualFilterTuning; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return false; } + @Override + public Set getRequiredColumns() + { + return dimensions.stream().map(dim -> dim.getDimension()).collect(Collectors.toSet()); + } + @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java index fbcff2a2dafa..826418f61283 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.extraction.ExtractionFn; @@ -30,6 +31,7 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer; @@ -38,6 +40,9 @@ import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import javax.annotation.Nullable; +import java.util.Set; + /** */ public class DimensionPredicateFilter implements Filter @@ -46,17 +51,29 @@ public class DimensionPredicateFilter implements Filter private final DruidPredicateFactory predicateFactory; private final String basePredicateString; private final ExtractionFn extractionFn; + private final FilterTuning manualFilterTuning; public DimensionPredicateFilter( final String dimension, final DruidPredicateFactory predicateFactory, final ExtractionFn extractionFn ) + { + this(dimension, predicateFactory, extractionFn, null); + } + + public DimensionPredicateFilter( + final String dimension, + final DruidPredicateFactory predicateFactory, + final ExtractionFn extractionFn, + final FilterTuning filterTuning + ) { Preconditions.checkNotNull(predicateFactory, "predicateFactory"); this.dimension = Preconditions.checkNotNull(dimension, "dimension"); this.basePredicateString = predicateFactory.toString(); this.extractionFn = extractionFn; + this.manualFilterTuning = filterTuning; if (extractionFn == null) { this.predicateFactory = predicateFactory; @@ -120,12 +137,25 @@ public boolean canVectorizeMatcher() return true; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return selector.getBitmapIndex(dimension) != null; } + @Nullable + @Override + public FilterTuning getManualTuning() + { + return manualFilterTuning; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java index f46cb8db4f33..1064e9a0f58e 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java @@ -30,6 +30,7 @@ import org.apache.druid.query.expression.ExprUtils; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.ColumnSelector; @@ -37,6 +38,7 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.virtual.ExpressionSelectors; +import javax.annotation.Nullable; import java.util.Arrays; import java.util.Set; @@ -44,11 +46,13 @@ public class ExpressionFilter implements Filter { private final Supplier expr; private final Supplier> requiredBindings; + private final FilterTuning manualFilterTuning; - public ExpressionFilter(final Supplier expr) + public ExpressionFilter(final Supplier expr, final FilterTuning filterTuning) { this.expr = expr; this.requiredBindings = Suppliers.memoize(() -> expr.get().analyzeInputs().getRequiredColumns()); + this.manualFilterTuning = filterTuning; } @Override @@ -108,6 +112,13 @@ public boolean supportsBitmapIndex(final BitmapIndexSelector selector) } } + @Nullable + @Override + public FilterTuning getManualTuning() + { + return manualFilterTuning; + } + @Override public T getBitmapResult(final BitmapIndexSelector selector, final BitmapResultFactory bitmapResultFactory) { @@ -151,4 +162,10 @@ public double estimateSelectivity(final BitmapIndexSelector indexSelector) // Selectivity estimation not supported. throw new UnsupportedOperationException(); } + + @Override + public Set getRequiredColumns() + { + return requiredBindings.get(); + } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java index 8186781ee8a0..41753cada3a5 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java @@ -21,6 +21,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.ints.IntIterable; import it.unimi.dsi.fastutil.ints.IntIterator; import org.apache.druid.collections.bitmap.ImmutableBitmap; @@ -32,6 +33,7 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer; @@ -42,6 +44,7 @@ import org.apache.druid.segment.column.BitmapIndex; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import javax.annotation.Nullable; import java.util.Iterator; import java.util.Set; @@ -52,6 +55,7 @@ public class InFilter implements Filter private final String dimension; private final Set values; private final ExtractionFn extractionFn; + private final FilterTuning manualFilterTuning; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; private final Supplier doublePredicateSupplier; @@ -62,12 +66,14 @@ public InFilter( Supplier longPredicateSupplier, Supplier floatPredicateSupplier, Supplier doublePredicateSupplier, - ExtractionFn extractionFn + ExtractionFn extractionFn, + FilterTuning filterTuning ) { this.dimension = dimension; this.values = values; this.extractionFn = extractionFn; + this.manualFilterTuning = filterTuning; this.longPredicateSupplier = longPredicateSupplier; this.floatPredicateSupplier = floatPredicateSupplier; this.doublePredicateSupplier = doublePredicateSupplier; @@ -162,12 +168,25 @@ public boolean canVectorizeMatcher() return true; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return selector.getBitmapIndex(dimension) != null; } + @Nullable + @Override + public FilterTuning getManualTuning() + { + return manualFilterTuning; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java index eedebe32e035..29552676c3ba 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java @@ -20,27 +20,35 @@ package org.apache.druid.segment.filter; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.JavaScriptDimFilter; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.mozilla.javascript.Context; +import javax.annotation.Nullable; +import java.util.Set; + public class JavaScriptFilter implements Filter { private final String dimension; private final JavaScriptDimFilter.JavaScriptPredicateFactory predicateFactory; + private final FilterTuning manualFilterTuning; public JavaScriptFilter( String dimension, - JavaScriptDimFilter.JavaScriptPredicateFactory predicate + JavaScriptDimFilter.JavaScriptPredicateFactory predicate, + FilterTuning filterTuning ) { this.dimension = dimension; this.predicateFactory = predicate; + this.manualFilterTuning = filterTuning; } @Override @@ -97,4 +105,17 @@ public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, Bitm { return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector); } + + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + + @Nullable + @Override + public FilterTuning getManualTuning() + { + return manualFilterTuning; + } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java index 765bcb430fdb..b750ee1a6395 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.segment.filter; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.ints.IntIterable; import it.unimi.dsi.fastutil.ints.IntIterator; import org.apache.druid.collections.bitmap.ImmutableBitmap; @@ -28,6 +29,7 @@ import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.LikeDimFilter; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; @@ -40,25 +42,30 @@ import org.apache.druid.segment.data.Indexed; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import javax.annotation.Nullable; import java.io.IOException; import java.io.UncheckedIOException; import java.util.NoSuchElementException; +import java.util.Set; public class LikeFilter implements Filter { private final String dimension; private final ExtractionFn extractionFn; private final LikeDimFilter.LikeMatcher likeMatcher; + private final FilterTuning manualFilterTuning; public LikeFilter( final String dimension, final ExtractionFn extractionFn, - final LikeDimFilter.LikeMatcher likeMatcher + final LikeDimFilter.LikeMatcher likeMatcher, + final FilterTuning filterTuning ) { this.dimension = dimension; this.extractionFn = extractionFn; this.likeMatcher = likeMatcher; + this.manualFilterTuning = filterTuning; } @Override @@ -95,12 +102,25 @@ public boolean canVectorizeMatcher() return true; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return selector.getBitmapIndex(dimension) != null; } + @Nullable + @Override + public FilterTuning getManualTuning() + { + return manualFilterTuning; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java index ef12e1693347..8b0af0e59c54 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java @@ -22,6 +22,7 @@ import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.BaseVectorValueMatcher; import org.apache.druid.query.filter.vector.ReadableVectorMatch; @@ -32,6 +33,9 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import javax.annotation.Nullable; +import java.util.Set; + /** */ public class NotFilter implements Filter @@ -103,12 +107,25 @@ public boolean canVectorizeMatcher() return baseFilter.canVectorizeMatcher(); } + @Override + public Set getRequiredColumns() + { + return baseFilter.getRequiredColumns(); + } + @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return baseFilter.supportsBitmapIndex(selector); } + @Nullable + @Override + public FilterTuning getManualTuning() + { + return baseFilter.getManualTuning(); + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/RegexFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/RegexFilter.java index 6ebc46ca1599..c71841ad08d2 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/RegexFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/RegexFilter.java @@ -25,6 +25,7 @@ import org.apache.druid.query.filter.DruidFloatPredicate; import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; +import org.apache.druid.query.filter.FilterTuning; import java.util.regex.Pattern; @@ -35,7 +36,8 @@ public class RegexFilter extends DimensionPredicateFilter public RegexFilter( final String dimension, final Pattern pattern, - final ExtractionFn extractionFn + final ExtractionFn extractionFn, + final FilterTuning filterTuning ) { super( @@ -74,7 +76,8 @@ public String toString() '}'; } }, - extractionFn + extractionFn, + filterTuning ); } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SearchQueryFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SearchQueryFilter.java index 9063cd7818a2..81424fe76ce8 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SearchQueryFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SearchQueryFilter.java @@ -27,6 +27,7 @@ import org.apache.druid.query.filter.DruidFloatPredicate; import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.search.SearchQuerySpec; /** @@ -37,7 +38,8 @@ public class SearchQueryFilter extends DimensionPredicateFilter public SearchQueryFilter( @JsonProperty("dimension") final String dimension, @JsonProperty("query") final SearchQuerySpec query, - @JsonProperty("extractionFn") final ExtractionFn extractionFn + @JsonProperty("extractionFn") final ExtractionFn extractionFn, + @JsonProperty("filterTuning") final FilterTuning filterTuning ) { super( @@ -68,7 +70,8 @@ public DruidDoublePredicate makeDoublePredicate() return input -> query.accept(String.valueOf(input)); } }, - extractionFn + extractionFn, + filterTuning ); } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java index 3640b7d3a512..ecf3e887c6a8 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java @@ -19,10 +19,12 @@ package org.apache.druid.segment.filter; +import com.google.common.collect.ImmutableSet; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnStrategizer; @@ -31,20 +33,34 @@ import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import javax.annotation.Nullable; +import java.util.Set; + /** */ public class SelectorFilter implements Filter { private final String dimension; private final String value; + private final FilterTuning manualFilterTuning; public SelectorFilter( String dimension, String value ) + { + this(dimension, value, null); + } + + public SelectorFilter( + String dimension, + String value, + @Nullable FilterTuning filterTuning + ) { this.dimension = dimension; this.value = value; + this.manualFilterTuning = filterTuning; } @Override @@ -75,6 +91,13 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return selector.getBitmapIndex(dimension) != null; } + @Nullable + @Override + public FilterTuning getManualTuning() + { + return manualFilterTuning; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { @@ -93,6 +116,12 @@ public boolean canVectorizeMatcher() return true; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public String toString() { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java index 37b228d0842e..ee0fccc9f33b 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.spatial.search.Bound; import org.apache.druid.query.BitmapResultFactory; @@ -30,25 +31,32 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.incremental.SpatialDimensionRowTransformer; +import javax.annotation.Nullable; +import java.util.Set; + /** */ public class SpatialFilter implements Filter { private final String dimension; private final Bound bound; + private final FilterTuning manualFilterTuning; public SpatialFilter( String dimension, - Bound bound + Bound bound, + FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension"); this.bound = Preconditions.checkNotNull(bound, "bound"); + this.manualFilterTuning = filterTuning; } @Override @@ -119,6 +127,19 @@ public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, Bitm return false; } + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + + @Nullable + @Override + public FilterTuning getManualTuning() + { + return manualFilterTuning; + } + @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java index 965fc1b659a6..aec150fb1093 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java @@ -22,10 +22,14 @@ import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import java.util.Collections; +import java.util.Set; + /** */ public class TrueFilter implements Filter @@ -52,12 +56,24 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return true; } + @Override + public FilterTuning getManualTuning() + { + return null; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return true; } + @Override + public Set getRequiredColumns() + { + return Collections.emptySet(); + } + @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java index 9e550903799f..7f9f9e872ec2 100644 --- a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java +++ b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java @@ -38,7 +38,7 @@ public class InDimFilterSerDesrTest private static ObjectMapper mapper; private final String actualInFilter = - "{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[\"bad\",\"good\"],\"extractionFn\":null}"; + "{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[\"bad\",\"good\"],\"extractionFn\":null,\"filterTuning\":null}"; @Before public void setUp() diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java index 335ca490319e..a0169546b295 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java @@ -43,6 +43,7 @@ import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.groupby.RowBasedColumnSelectorFactory; @@ -76,6 +77,7 @@ import org.junit.rules.TemporaryFolder; import org.junit.runners.Parameterized; +import javax.annotation.Nullable; import java.io.Closeable; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -84,10 +86,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; public abstract class BaseFilterTest { - private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create( + static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create( ImmutableList.of( new ExpressionVirtualColumn("expr", "1.0 + 0.1", ValueType.FLOAT, TestExprMacroTable.INSTANCE), new ExpressionVirtualColumn("exprDouble", "1.0 + 1.1", ValueType.DOUBLE, TestExprMacroTable.INSTANCE), @@ -401,12 +404,25 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return false; } + @Nullable + @Override + public FilterTuning getManualTuning() + { + return null; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return false; } + @Override + public Set getRequiredColumns() + { + return null; + } + @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { @@ -480,6 +496,19 @@ public boolean canVectorizeMatcher() return theFilter.canVectorizeMatcher(); } + @Override + public Set getRequiredColumns() + { + return null; + } + + @Nullable + @Override + public FilterTuning getManualTuning() + { + return null; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java index 578e4d2af0c4..b4d78cf4e720 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java @@ -268,6 +268,6 @@ public void testGetRequiredColumn() private static ExpressionDimFilter edf(final String expression) { - return new ExpressionDimFilter(expression, TestExprMacroTable.INSTANCE); + return new ExpressionDimFilter(expression, null, TestExprMacroTable.INSTANCE); } } diff --git a/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java index 1d48b377338c..5bbd728deb4b 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java @@ -43,9 +43,12 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.OrDimFilter; import org.apache.druid.query.filter.SelectorDimFilter; +import org.apache.druid.segment.ColumnSelectorBitmapIndexSelector; import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.QueryableIndexStorageAdapter; import org.apache.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Assert; @@ -71,6 +74,14 @@ public NoBitmapSelectorFilter( { super(dimension, value); } + public NoBitmapSelectorFilter( + String dimension, + String value, + FilterTuning filterTuning + ) + { + super(dimension, value, filterTuning); + } @Override public boolean supportsBitmapIndex(BitmapIndexSelector selector) @@ -727,4 +738,84 @@ public void testDistributeOrCNFExtractionFn() ImmutableList.of("2", "3", "4", "6", "7", "9") ); } + + @Test + public void testAnalyze() + { + if (!(adapter instanceof QueryableIndexStorageAdapter)) { + return; + } + QueryableIndexStorageAdapter storageAdapter = (QueryableIndexStorageAdapter) adapter; + final ColumnSelectorBitmapIndexSelector bitmapIndexSelector = storageAdapter.makeBitmapIndexSelector(BaseFilterTest.VIRTUAL_COLUMNS); + + // has bitmap index, will use it by default + Filter normalFilter = new SelectorFilter("dim1", "HELLO"); + QueryableIndexStorageAdapter.FilterAnalysis filterAnalysisNormal = + storageAdapter.analyzeFilter(normalFilter, bitmapIndexSelector, null); + Assert.assertTrue(filterAnalysisNormal.getPreFilterBitmap() != null); + Assert.assertTrue(filterAnalysisNormal.getPostFilter() == null); + + + // no bitmap index, should be a post filter + Filter noBitmapFilter = new NoBitmapSelectorFilter("dim1", "HELLO"); + QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterAnalysis = + storageAdapter.analyzeFilter(noBitmapFilter, bitmapIndexSelector, null); + Assert.assertTrue(noBitmapFilterAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(noBitmapFilterAnalysis.getPostFilter() != null); + + // this column has a bitmap index, but is forced to not use it + Filter bitmapFilterWithForceNoIndexTuning = new SelectorFilter( + "dim1", + "HELLO", + new FilterTuning(false, null, null) + ); + QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithForceNoIndexTuningAnalysis = + storageAdapter.analyzeFilter(bitmapFilterWithForceNoIndexTuning, bitmapIndexSelector, null); + Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPostFilter() != null); + + // this max cardinality is too low to use bitmap index + Filter bitmapFilterWithCardinalityMax = new SelectorFilter( + "dim1", + "HELLO", + new FilterTuning(true, 0, 3) + ); + QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMaxAnalysis = + storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax, bitmapIndexSelector, null); + Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPostFilter() != null); + + // this max cardinality is high enough that we can still use bitmap index + Filter bitmapFilterWithCardinalityMax2 = new SelectorFilter( + "dim1", + "HELLO", + new FilterTuning(true, 0, 1000) + ); + QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMax2Analysis = + storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax2, bitmapIndexSelector, null); + Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPreFilterBitmap() != null); + Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPostFilter() == null); + + // this min cardinality is too high, will not use bitmap index + Filter bitmapFilterWithCardinalityMin = new SelectorFilter( + "dim1", + "HELLO", + new FilterTuning(true, 1000, null) + ); + QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMinAnalysis = + storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMin, bitmapIndexSelector, null); + Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPostFilter() != null); + + // cannot force using bitmap if there are no bitmaps + Filter noBitmapFilterWithForceUse = new NoBitmapSelectorFilter( + "dim1", + "HELLO", + new FilterTuning(true, null, null) + ); + QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterWithForceUseAnalysis = + storageAdapter.analyzeFilter(noBitmapFilterWithForceUse, bitmapIndexSelector, null); + Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPreFilterBitmap() == null); + Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPostFilter() != null); + } } diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index b3302f84fafe..37e0638a23fb 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -48,6 +48,7 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.groupby.GroupByQuery; import org.apache.druid.query.groupby.GroupByQueryConfig; @@ -71,12 +72,14 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; /** @@ -693,12 +696,25 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return true; } + @Nullable + @Override + public FilterTuning getManualTuning() + { + return null; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { return true; } + @Override + public Set getRequiredColumns() + { + return null; + } + private class DictionaryRaceTestFilterDruidPredicateFactory implements DruidPredicateFactory { @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 9da1ac44f778..bf59aa9930a9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -2266,7 +2266,7 @@ public void testExplainCountStarOnView() throws Exception + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + "\"descending\":false," + "\"virtualColumns\":[]," - + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]}," + + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null,\"filterTuning\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1},\"filterTuning\":null}}]}," + "\"granularity\":{\"type\":\"all\"}," + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," + "\"postAggregations\":[]," @@ -4888,7 +4888,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception final String explanation = "DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"skipEmptyBuckets\":true,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\"}}], signature=[{a0:LONG}])\n" + " DruidSemiJoin(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\"},\"descending\":false}], leftExpressions=[[SUBSTRING($3, 1, 1)]], rightKeys=[[0]])\n" - + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n"; + + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null,\"filterTuning\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n"; testQuery( "EXPLAIN PLAN FOR SELECT COUNT(*)\n" From d4d6629e8dd881cd044e26428b916f76f3519d7a Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 31 Jul 2019 16:45:07 -0700 Subject: [PATCH 02/24] oops --- .../src/main/java/org/apache/druid/segment/FilteredOffset.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java b/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java index d387c65c6a10..7a6acb2267d1 100644 --- a/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java +++ b/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java @@ -56,7 +56,7 @@ public final class FilteredOffset extends Offset rowOffsetMatcherFactory ); } else { - if (postFilter.supportsBitmapIndex(bitmapIndexSelector)) { + if (postFilter.shouldUseIndex(bitmapIndexSelector)) { filterMatcher = rowOffsetMatcherFactory.makeRowOffsetMatcher( postFilter.getBitmapIndex(bitmapIndexSelector) ); From 383eeb4e525e0d45c96876792658662a8994f453 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 31 Jul 2019 16:48:44 -0700 Subject: [PATCH 03/24] adjust --- .../main/java/org/apache/druid/query/filter/FilterTuning.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java index 6ce30b174e62..5383aeadc6b8 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java @@ -51,7 +51,7 @@ public FilterTuning( @JsonProperty public Boolean getUseIndex() { - return useIndex; + return useIndex != null ? useIndex : true; } @JsonProperty From cc9c9b010e0a8b69934b712e41d9ddbf9aba850c Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 31 Jul 2019 17:30:12 -0700 Subject: [PATCH 04/24] woo --- .../org/apache/druid/segment/filter/FilterPartitionTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java index 5bbd728deb4b..966b418656bf 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/FilterPartitionTest.java @@ -74,6 +74,7 @@ public NoBitmapSelectorFilter( { super(dimension, value); } + public NoBitmapSelectorFilter( String dimension, String value, From f7b8a4972d6147cc6b039870906785954c4971f3 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 2 Aug 2019 15:41:43 -0700 Subject: [PATCH 05/24] javadoc --- .../druid/query/filter/FilterTuning.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java index 5383aeadc6b8..81d565d71329 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java @@ -25,6 +25,26 @@ import javax.annotation.Nullable; import java.util.Objects; +/** + * This class provides a mechansim to influence whether or not indexes are used for a {@link Filter} during processing + * by {@link org.apache.druid.segment.QueryableIndexStorageAdapter#analyzeFilter} (i.e. will a {@link Filter} be a "pre" + * filter in which we union indexes for all values that match the filter to create a + * {@link org.apache.druid.segment.BitmapOffset}/{@link org.apache.druid.segment.vector.BitmapVectorOffset}, or will it + * be used as a "post" filter and evaluated while scanning row values from the + * {@link org.apache.druid.segment.FilteredOffset}/{@link org.apache.druid.segment.vector.FilteredVectorOffset}. + * + * This is currently only manually supplied by the user by adding to a {@link DimFilter} which will pass through to the + * {@link Filter} implementation for it to provide as {@link Filter#getManualTuning()}. The main purpose at this time is + * to facilitate experimentation so that someday we can have {@link Filter} implementations intelligently, automatically + * use sensible defaults based on things like cardinality and who yet knows what additional information. + * + * It can also be used for advanced users to manually control which filters will be "pre" and "post" filters as + * described above to allow skipping indexes in known cases where filters are expensive (mostly high cardinality columns + * with expensive filters). + * + * As such, it is currently undocumented in user facing documentation on purpose, but whatever this turns into once more + * automatic usage of this is in place, should be documented in a future release. + */ public class FilterTuning { public static FilterTuning createDefault(boolean useIndex) From ef28052a37fe26fa34f29829b619128c02e03cfb Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 2 Aug 2019 17:15:05 -0700 Subject: [PATCH 06/24] review comments --- .../druid/query/filter/BloomDimFilter.java | 8 +++++ .../java/org/apache/druid/query/Druids.java | 2 +- .../druid/query/filter/BooleanFilter.java | 6 ---- .../druid/query/filter/BoundDimFilter.java | 2 ++ .../filter/ColumnComparisonDimFilter.java | 2 ++ .../query/filter/ExpressionDimFilter.java | 2 ++ .../org/apache/druid/query/filter/Filter.java | 36 +------------------ .../druid/query/filter/FilterTuning.java | 33 ++++++++--------- .../druid/query/filter/InDimFilter.java | 2 ++ .../druid/query/filter/IntervalDimFilter.java | 2 ++ .../query/filter/JavaScriptDimFilter.java | 2 ++ .../druid/query/filter/LikeDimFilter.java | 2 ++ .../druid/query/filter/RegexDimFilter.java | 8 +++++ .../query/filter/SearchQueryDimFilter.java | 2 ++ .../druid/query/filter/SelectorDimFilter.java | 2 ++ .../druid/query/filter/SpatialDimFilter.java | 8 +++++ .../druid/query/topn/TopNQueryBuilder.java | 2 +- .../druid/segment/filter/AndFilter.java | 11 ++++++ .../druid/segment/filter/BoundFilter.java | 10 +++--- .../filter/ColumnComparisonFilter.java | 11 +++--- .../filter/DimensionPredicateFilter.java | 10 +++--- .../segment/filter/ExpressionFilter.java | 10 +++--- .../apache/druid/segment/filter/Filters.java | 13 +++++++ .../apache/druid/segment/filter/InFilter.java | 10 +++--- .../segment/filter/JavaScriptFilter.java | 10 +++--- .../druid/segment/filter/LikeFilter.java | 10 +++--- .../druid/segment/filter/NotFilter.java | 7 ++-- .../apache/druid/segment/filter/OrFilter.java | 11 ++++++ .../druid/segment/filter/SelectorFilter.java | 9 +++-- .../druid/segment/filter/SpatialFilter.java | 25 +++++-------- .../druid/segment/filter/TrueFilter.java | 5 ++- .../druid/segment/filter/BaseFilterTest.java | 16 --------- .../IncrementalIndexStorageAdapterTest.java | 9 ----- .../druid/sql/calcite/filtration/Bounds.java | 27 +++++++++----- .../filtration/ConvertSelectorsToIns.java | 2 +- 35 files changed, 160 insertions(+), 167 deletions(-) diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java index 4c5270825804..c939f4874fbb 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -190,6 +191,13 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public String toString() { diff --git a/processing/src/main/java/org/apache/druid/query/Druids.java b/processing/src/main/java/org/apache/druid/query/Druids.java index e0599663a7d5..dcd6429aaa36 100644 --- a/processing/src/main/java/org/apache/druid/query/Druids.java +++ b/processing/src/main/java/org/apache/druid/query/Druids.java @@ -361,7 +361,7 @@ public SearchQueryBuilder dataSource(DataSource d) public SearchQueryBuilder filters(String dimensionName, String value) { - dimFilter = new SelectorDimFilter(dimensionName, value, null); + dimFilter = new SelectorDimFilter(dimensionName, value, null, null); return this; } diff --git a/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java index 2d3fe2ecc071..c22843120933 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java @@ -51,12 +51,6 @@ ValueMatcher makeMatcher( RowOffsetMatcherFactory rowOffsetMatcherFactory ); - @Override - default FilterTuning getManualTuning() - { - return null; - } - @Override default Set getRequiredColumns() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java index ef5d798a2d0c..54e73de25ce7 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -171,6 +172,7 @@ public StringComparator getOrdering() return ordering; } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java index 8fe5dfbced47..0560b2aeced7 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; @@ -90,6 +91,7 @@ public List getDimensions() return dimensions; } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java index a70a25213a71..3b735e6144ea 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Supplier; @@ -69,6 +70,7 @@ public String getExpression() return expression; } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java index 9b518fb1b029..fe09977cb25b 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java @@ -28,7 +28,6 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; -import javax.annotation.Nullable; import java.util.Set; public interface Filter @@ -137,27 +136,6 @@ default boolean canVectorizeMatcher() */ Set getRequiredColumns(); - /** - * "Manual" {@link FilterTuning} to allow explicit control of filter optimization behavior. This will likely be - * supplied by the {@link DimFilter} that is creating this {@link Filter} - */ - @Nullable - FilterTuning getManualTuning(); - - /** - * This method allows a filter to automatically compute a {@link FilterTuning} based on information it can gather - * from a {@link BitmapIndexSelector}. This method makes sense to override if the default implementation of - * {@link #shouldUseIndex(BitmapIndexSelector)} is sufficient for filter optimization. By default, creates a - * {@link FilterTuning} with no limits that will always use a bitmap index if - * {@link #supportsBitmapIndex(BitmapIndexSelector)} is true, unless a {@link FilterTuning} is provided by - * {@link #getManualTuning()} - */ - default FilterTuning computeTuning(BitmapIndexSelector selector) - { - final FilterTuning manual = getManualTuning(); - return manual != null ? manual : FilterTuning.createDefault(supportsBitmapIndex(selector)); - } - /** * Determine if a filter *should* use a bitmap index based on information collected from the supplied * {@link BitmapIndexSelector}. This method differs from {@link #supportsBitmapIndex(BitmapIndexSelector)} in that @@ -166,17 +144,5 @@ default FilterTuning computeTuning(BitmapIndexSelector selector) * available index. Override this method in a {@link Filter} implementation when {@link FilterTuning} alone is not * adequate for making this decision. */ - default boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) - { - if (supportsBitmapIndex(bitmapIndexSelector)) { - final FilterTuning tuning = computeTuning(bitmapIndexSelector); - return tuning.getUseIndex() - && (getRequiredColumns().size() == 0 || getRequiredColumns().stream().allMatch(column -> { - final int cardinality = bitmapIndexSelector.getBitmapIndex(column).getCardinality(); - return cardinality >= tuning.getUseIndexMinCardinalityThreshold() - && cardinality <= tuning.getUseIndexMaxCardinalityThreshold(); - })); - } - return false; - } + boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java index 81d565d71329..a0969cffbf6d 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java @@ -34,9 +34,9 @@ * {@link org.apache.druid.segment.FilteredOffset}/{@link org.apache.druid.segment.vector.FilteredVectorOffset}. * * This is currently only manually supplied by the user by adding to a {@link DimFilter} which will pass through to the - * {@link Filter} implementation for it to provide as {@link Filter#getManualTuning()}. The main purpose at this time is - * to facilitate experimentation so that someday we can have {@link Filter} implementations intelligently, automatically - * use sensible defaults based on things like cardinality and who yet knows what additional information. + * {@link Filter} implementation. The main purpose at this time is to facilitate experimentation so that someday we can + * have {@link Filter} implementations intelligently, automatically use sensible defaults based on things like + * cardinality and who yet knows what additional information. * * It can also be used for advanced users to manually control which filters will be "pre" and "post" filters as * described above to allow skipping indexes in known cases where filters are expensive (mostly high cardinality columns @@ -47,14 +47,9 @@ */ public class FilterTuning { - public static FilterTuning createDefault(boolean useIndex) - { - return new FilterTuning(useIndex, 0, Integer.MAX_VALUE); - } - - private final Boolean useIndex; - private final Integer useIndexMinCardinalityThreshold; - private final Integer useIndexMaxCardinalityThreshold; + private final boolean useIndex; + private final int useIndexMinCardinalityThreshold; + private final int useIndexMaxCardinalityThreshold; @JsonCreator public FilterTuning( @@ -63,27 +58,29 @@ public FilterTuning( @Nullable @JsonProperty("useIndexMaximumCardinalityThreshold") Integer useIndexMaxCardinalityThreshold ) { - this.useIndex = useIndex; - this.useIndexMinCardinalityThreshold = useIndexMinCardinalityThreshold; - this.useIndexMaxCardinalityThreshold = useIndexMaxCardinalityThreshold; + this.useIndex = useIndex != null ? useIndex : true; + this.useIndexMinCardinalityThreshold = + useIndexMinCardinalityThreshold != null ? useIndexMinCardinalityThreshold : 0; + this.useIndexMaxCardinalityThreshold = + useIndexMaxCardinalityThreshold != null ? useIndexMaxCardinalityThreshold : Integer.MAX_VALUE; } @JsonProperty - public Boolean getUseIndex() + public boolean getUseIndex() { - return useIndex != null ? useIndex : true; + return useIndex; } @JsonProperty public Integer getUseIndexMinCardinalityThreshold() { - return useIndexMinCardinalityThreshold != null ? useIndexMinCardinalityThreshold : 0; + return useIndexMinCardinalityThreshold; } @JsonProperty public Integer getUseIndexMaxCardinalityThreshold() { - return useIndexMaxCardinalityThreshold != null ? useIndexMaxCardinalityThreshold : Integer.MAX_VALUE; + return useIndexMaxCardinalityThreshold; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java index d1ba394cf50a..5ca756d6233c 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; @@ -122,6 +123,7 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java index 9e4510360b5f..7660a95ec68d 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -96,6 +97,7 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java index c322b8aa3027..5e443a080084 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -107,6 +108,7 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java index cc2c536f6ec2..e2ab1138b533 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -112,6 +113,7 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java index 95312553e938..71fb80baf09d 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -90,6 +91,13 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public byte[] getCacheKey() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java index 417f56ff2dbc..051fc420ef60 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java @@ -19,6 +19,7 @@ package org.apache.druid.query.filter; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -86,6 +87,7 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java index d917cb345cd8..387938f88d62 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -165,6 +166,7 @@ public ExtractionFn getExtractionFn() return extractionFn; } + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() { diff --git a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java index ae9b36cdb83f..be591070f2c5 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java @@ -20,6 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -97,6 +98,13 @@ public Bound getBound() return bound; } + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty + public FilterTuning getFilterTuning() + { + return filterTuning; + } + @Override public Filter toFilter() { diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java index 45227b93b739..d50c5657686a 100644 --- a/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java +++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQueryBuilder.java @@ -230,7 +230,7 @@ public TopNQueryBuilder filters(String dimensionName, String value) public TopNQueryBuilder filters(String dimensionName, String value, String... values) { - dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null); + dimFilter = new InDimFilter(dimensionName, Lists.asList(value, values), null, null); return this; } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java index 76fd626c9d45..1cbdc8fb5d37 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java @@ -125,6 +125,17 @@ public boolean canVectorizeMatcher() return filters.stream().allMatch(Filter::canVectorizeMatcher); } + @Override + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + { + for (Filter filter : filters) { + if (!filter.shouldUseIndex(bitmapIndexSelector)) { + return false; + } + } + return true; + } + @Override public ValueMatcher makeMatcher( BitmapIndexSelector selector, diff --git a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java index 87194ef80689..1b666f1fc8e1 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java @@ -46,7 +46,6 @@ import org.apache.druid.segment.column.BitmapIndex; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; -import javax.annotation.Nullable; import java.util.Comparator; import java.util.Set; @@ -55,7 +54,7 @@ public class BoundFilter implements Filter private final BoundDimFilter boundDimFilter; private final Comparator comparator; private final ExtractionFn extractionFn; - private final FilterTuning manualFilterTuning; + private final FilterTuning filterTuning; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; @@ -69,7 +68,7 @@ public BoundFilter(final BoundDimFilter boundDimFilter) this.longPredicateSupplier = boundDimFilter.getLongPredicateSupplier(); this.floatPredicateSupplier = boundDimFilter.getFloatPredicateSupplier(); this.doublePredicateSupplier = boundDimFilter.getDoublePredicateSupplier(); - this.manualFilterTuning = boundDimFilter.getFilterTuning(); + this.filterTuning = boundDimFilter.getFilterTuning(); } @Override @@ -161,11 +160,10 @@ public Set getRequiredColumns() return boundDimFilter.getRequiredColumns(); } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return manualFilterTuning; + return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java index c0359dc816e6..135f517ba199 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java @@ -46,7 +46,7 @@ public class ColumnComparisonFilter implements Filter { private final List dimensions; - private final FilterTuning manualFilterTuning; + private final FilterTuning filterTuning; public ColumnComparisonFilter( final List dimensions, @@ -54,7 +54,7 @@ public ColumnComparisonFilter( ) { this.dimensions = Preconditions.checkNotNull(dimensions, "dimensions"); - this.manualFilterTuning = filterTuning; + this.filterTuning = filterTuning; } @Override @@ -147,11 +147,10 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return false; } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return manualFilterTuning; + return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); } @Override @@ -163,7 +162,7 @@ public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, Bitm @Override public Set getRequiredColumns() { - return dimensions.stream().map(dim -> dim.getDimension()).collect(Collectors.toSet()); + return dimensions.stream().map(DimensionSpec::getDimension).collect(Collectors.toSet()); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java index 826418f61283..1544038a63b3 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java @@ -40,7 +40,6 @@ import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; -import javax.annotation.Nullable; import java.util.Set; /** @@ -51,7 +50,7 @@ public class DimensionPredicateFilter implements Filter private final DruidPredicateFactory predicateFactory; private final String basePredicateString; private final ExtractionFn extractionFn; - private final FilterTuning manualFilterTuning; + private final FilterTuning filterTuning; public DimensionPredicateFilter( final String dimension, @@ -73,7 +72,7 @@ public DimensionPredicateFilter( this.dimension = Preconditions.checkNotNull(dimension, "dimension"); this.basePredicateString = predicateFactory.toString(); this.extractionFn = extractionFn; - this.manualFilterTuning = filterTuning; + this.filterTuning = filterTuning; if (extractionFn == null) { this.predicateFactory = predicateFactory; @@ -149,11 +148,10 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return selector.getBitmapIndex(dimension) != null; } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return manualFilterTuning; + return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java index 4f43a2069228..b3e4a48ad110 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java @@ -38,7 +38,6 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.virtual.ExpressionSelectors; -import javax.annotation.Nullable; import java.util.Arrays; import java.util.Set; @@ -46,13 +45,13 @@ public class ExpressionFilter implements Filter { private final Supplier expr; private final Supplier> requiredBindings; - private final FilterTuning manualFilterTuning; + private final FilterTuning filterTuning; public ExpressionFilter(final Supplier expr, final FilterTuning filterTuning) { this.expr = expr; this.requiredBindings = Suppliers.memoize(() -> expr.get().analyzeInputs().getRequiredBindings()); - this.manualFilterTuning = filterTuning; + this.filterTuning = filterTuning; } @Override @@ -112,11 +111,10 @@ public boolean supportsBitmapIndex(final BitmapIndexSelector selector) } } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return manualFilterTuning; + return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java index a81eb60a8fea..c5336ce8c15e 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java @@ -39,6 +39,7 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.ValueMatcherColumnSelectorStrategy; import org.apache.druid.query.filter.ValueMatcherColumnSelectorStrategyFactory; @@ -651,4 +652,16 @@ private static void generateAllCombinations( generateAllCombinations(result, andList.subList(1, andList.size()), nonAndList); } } + + public static boolean shouldUseIndex(Filter filter, BitmapIndexSelector indexSelector, FilterTuning tuning) + { + if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseIndex()) { + return filter.getRequiredColumns().stream().allMatch(column -> { + final int cardinality = indexSelector.getBitmapIndex(column).getCardinality(); + return cardinality >= tuning.getUseIndexMinCardinalityThreshold() + && cardinality <= tuning.getUseIndexMaxCardinalityThreshold(); + }); + } + return false; + } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java index 41753cada3a5..ffd891878d73 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java @@ -44,7 +44,6 @@ import org.apache.druid.segment.column.BitmapIndex; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; -import javax.annotation.Nullable; import java.util.Iterator; import java.util.Set; @@ -55,7 +54,7 @@ public class InFilter implements Filter private final String dimension; private final Set values; private final ExtractionFn extractionFn; - private final FilterTuning manualFilterTuning; + private final FilterTuning filterTuning; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; private final Supplier doublePredicateSupplier; @@ -73,7 +72,7 @@ public InFilter( this.dimension = dimension; this.values = values; this.extractionFn = extractionFn; - this.manualFilterTuning = filterTuning; + this.filterTuning = filterTuning; this.longPredicateSupplier = longPredicateSupplier; this.floatPredicateSupplier = floatPredicateSupplier; this.doublePredicateSupplier = doublePredicateSupplier; @@ -180,11 +179,10 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return selector.getBitmapIndex(dimension) != null; } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return manualFilterTuning; + return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java index 29552676c3ba..170002c88abf 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java @@ -31,14 +31,13 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.mozilla.javascript.Context; -import javax.annotation.Nullable; import java.util.Set; public class JavaScriptFilter implements Filter { private final String dimension; private final JavaScriptDimFilter.JavaScriptPredicateFactory predicateFactory; - private final FilterTuning manualFilterTuning; + private final FilterTuning filterTuning; public JavaScriptFilter( String dimension, @@ -48,7 +47,7 @@ public JavaScriptFilter( { this.dimension = dimension; this.predicateFactory = predicate; - this.manualFilterTuning = filterTuning; + this.filterTuning = filterTuning; } @Override @@ -112,10 +111,9 @@ public Set getRequiredColumns() return ImmutableSet.of(dimension); } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return manualFilterTuning; + return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java index b750ee1a6395..ab8f9f082440 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java @@ -42,7 +42,6 @@ import org.apache.druid.segment.data.Indexed; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; -import javax.annotation.Nullable; import java.io.IOException; import java.io.UncheckedIOException; import java.util.NoSuchElementException; @@ -53,7 +52,7 @@ public class LikeFilter implements Filter private final String dimension; private final ExtractionFn extractionFn; private final LikeDimFilter.LikeMatcher likeMatcher; - private final FilterTuning manualFilterTuning; + private final FilterTuning filterTuning; public LikeFilter( final String dimension, @@ -65,7 +64,7 @@ public LikeFilter( this.dimension = dimension; this.extractionFn = extractionFn; this.likeMatcher = likeMatcher; - this.manualFilterTuning = filterTuning; + this.filterTuning = filterTuning; } @Override @@ -114,11 +113,10 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return selector.getBitmapIndex(dimension) != null; } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return manualFilterTuning; + return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java index 8b0af0e59c54..7991aa039c82 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java @@ -22,7 +22,6 @@ import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; -import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.BaseVectorValueMatcher; import org.apache.druid.query.filter.vector.ReadableVectorMatch; @@ -33,7 +32,6 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; -import javax.annotation.Nullable; import java.util.Set; /** @@ -119,11 +117,10 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return baseFilter.supportsBitmapIndex(selector); } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return baseFilter.getManualTuning(); + return baseFilter.shouldUseIndex(bitmapIndexSelector); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java index 01d82298e401..817f2e0cd00b 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java @@ -99,6 +99,17 @@ public boolean canVectorizeMatcher() return filters.stream().allMatch(Filter::canVectorizeMatcher); } + @Override + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + { + for (Filter f : filters) { + if (!f.shouldUseIndex(bitmapIndexSelector)) { + return false; + } + } + return true; + } + @Override public ValueMatcher makeMatcher( BitmapIndexSelector selector, diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java index ecf3e887c6a8..d02cc558493a 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java @@ -42,7 +42,7 @@ public class SelectorFilter implements Filter { private final String dimension; private final String value; - private final FilterTuning manualFilterTuning; + private final FilterTuning filterTuning; public SelectorFilter( String dimension, @@ -60,7 +60,7 @@ public SelectorFilter( { this.dimension = dimension; this.value = value; - this.manualFilterTuning = filterTuning; + this.filterTuning = filterTuning; } @Override @@ -91,11 +91,10 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return selector.getBitmapIndex(dimension) != null; } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return manualFilterTuning; + return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java index ee0fccc9f33b..a4ca13f052b3 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java @@ -37,7 +37,6 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.incremental.SpatialDimensionRowTransformer; -import javax.annotation.Nullable; import java.util.Set; /** @@ -46,7 +45,7 @@ public class SpatialFilter implements Filter { private final String dimension; private final Bound bound; - private final FilterTuning manualFilterTuning; + private final FilterTuning filterTuning; public SpatialFilter( String dimension, @@ -56,7 +55,7 @@ public SpatialFilter( { this.dimension = Preconditions.checkNotNull(dimension, "dimension"); this.bound = Preconditions.checkNotNull(bound, "bound"); - this.manualFilterTuning = filterTuning; + this.filterTuning = filterTuning; } @Override @@ -77,17 +76,12 @@ public ValueMatcher makeMatcher(ColumnSelectorFactory factory) @Override public Predicate makeStringPredicate() { - return new Predicate() - { - @Override - public boolean apply(String input) - { - if (input == null) { - return false; - } - final float[] coordinate = SpatialDimensionRowTransformer.decode(input); - return bound.contains(coordinate); + return input -> { + if (input == null) { + return false; } + final float[] coordinate = SpatialDimensionRowTransformer.decode(input); + return bound.contains(coordinate); }; } @@ -133,11 +127,10 @@ public Set getRequiredColumns() return ImmutableSet.of(dimension); } - @Nullable @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return manualFilterTuning; + return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java index aec150fb1093..ccd4c101d4be 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java @@ -22,7 +22,6 @@ import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; -import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; @@ -57,9 +56,9 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public FilterTuning getManualTuning() + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) { - return null; + return true; } @Override diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java index 6624287e6f91..4d0d1dfb240f 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java @@ -43,7 +43,6 @@ import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.filter.Filter; -import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.groupby.RowBasedColumnSelectorFactory; @@ -77,7 +76,6 @@ import org.junit.rules.TemporaryFolder; import org.junit.runners.Parameterized; -import javax.annotation.Nullable; import java.io.Closeable; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -399,13 +397,6 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return false; } - @Nullable - @Override - public FilterTuning getManualTuning() - { - return null; - } - @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { @@ -492,13 +483,6 @@ public Set getRequiredColumns() return null; } - @Nullable - @Override - public FilterTuning getManualTuning() - { - return null; - } - @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index 37e0638a23fb..19374455761a 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -48,7 +48,6 @@ import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.Filter; -import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.groupby.GroupByQuery; import org.apache.druid.query.groupby.GroupByQueryConfig; @@ -72,7 +71,6 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -696,13 +694,6 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return true; } - @Nullable - @Override - public FilterTuning getManualTuning() - { - return null; - } - @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java index 2af0baf33321..e06227b87d62 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java @@ -53,7 +53,8 @@ public static BoundDimFilter not(final BoundDimFilter bound) false, null, bound.getExtractionFn(), - bound.getOrdering() + bound.getOrdering(), + bound.getFilterTuning() ); } else { // bound.getLower() != null @@ -65,7 +66,8 @@ public static BoundDimFilter not(final BoundDimFilter bound) !bound.isLowerStrict(), null, bound.getExtractionFn(), - bound.getOrdering() + bound.getOrdering(), + bound.getFilterTuning() ); } } @@ -113,7 +115,8 @@ public static BoundDimFilter toFilter(final BoundRefKey boundRefKey, final Range range.hasUpperBound() && range.upperBoundType() == BoundType.OPEN, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -127,7 +130,8 @@ public static BoundDimFilter equalTo(final BoundRefKey boundRefKey, final String false, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -141,7 +145,8 @@ public static BoundDimFilter greaterThan(final BoundRefKey boundRefKey, final St false, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -155,7 +160,8 @@ public static BoundDimFilter greaterThanOrEqualTo(final BoundRefKey boundRefKey, false, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -169,7 +175,8 @@ public static BoundDimFilter lessThan(final BoundRefKey boundRefKey, final Strin true, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -183,7 +190,8 @@ public static BoundDimFilter lessThanOrEqualTo(final BoundRefKey boundRefKey, fi false, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } @@ -202,7 +210,8 @@ public static BoundDimFilter interval(final BoundRefKey boundRefKey, final Inter true, null, boundRefKey.getExtractionFn(), - boundRefKey.getComparator() + boundRefKey.getComparator(), + null ); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java index a594bbf0b6d4..04067db79047 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java @@ -89,7 +89,7 @@ public DimFilter process(DimFilter filter) } } - children.add(new InDimFilter(entry.getKey().getDimension(), values, entry.getKey().getExtractionFn())); + children.add(new InDimFilter(entry.getKey().getDimension(), values, entry.getKey().getExtractionFn(), null)); } } From 922c740d8e149337cbd2b8480abece9f0fc5df81 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 2 Aug 2019 17:18:01 -0700 Subject: [PATCH 07/24] fix --- .../main/java/org/apache/druid/query/filter/FilterTuning.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java index a0969cffbf6d..b769ebed8c10 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java @@ -72,13 +72,13 @@ public boolean getUseIndex() } @JsonProperty - public Integer getUseIndexMinCardinalityThreshold() + public int getUseIndexMinCardinalityThreshold() { return useIndexMinCardinalityThreshold; } @JsonProperty - public Integer getUseIndexMaxCardinalityThreshold() + public int getUseIndexMaxCardinalityThreshold() { return useIndexMaxCardinalityThreshold; } From 9587425f93fc56e5a8d51dbebf11b836083cc83b Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 2 Aug 2019 18:51:16 -0700 Subject: [PATCH 08/24] default --- .../java/org/apache/druid/query/filter/FilterTuning.java | 5 +++++ .../main/java/org/apache/druid/segment/filter/Filters.java | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java index b769ebed8c10..57632c6a1ef6 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java @@ -47,6 +47,11 @@ */ public class FilterTuning { + public static FilterTuning createDefault(Filter filter, BitmapIndexSelector selector) + { + return new FilterTuning(filter.supportsBitmapIndex(selector), null, null); + } + private final boolean useIndex; private final int useIndexMinCardinalityThreshold; private final int useIndexMaxCardinalityThreshold; diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java index c5336ce8c15e..563d229d1d54 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java @@ -653,8 +653,9 @@ private static void generateAllCombinations( } } - public static boolean shouldUseIndex(Filter filter, BitmapIndexSelector indexSelector, FilterTuning tuning) + public static boolean shouldUseIndex(Filter filter, BitmapIndexSelector indexSelector, FilterTuning filterTuning) { + final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector); if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseIndex()) { return filter.getRequiredColumns().stream().allMatch(column -> { final int cardinality = indexSelector.getBitmapIndex(column).getCardinality(); From 1f65d05f200de35d727a5781aaf937b2042298f4 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 2 Aug 2019 18:53:02 -0700 Subject: [PATCH 09/24] oops --- .../apache/druid/segment/filter/BaseFilterTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java index 4d0d1dfb240f..ed57e9067f58 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java @@ -409,6 +409,12 @@ public Set getRequiredColumns() return null; } + @Override + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + { + return false; + } + @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { @@ -483,6 +489,12 @@ public Set getRequiredColumns() return null; } + @Override + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + { + return false; + } + @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { From 3fbf5e6b23a2964f7a59c141860ffd253a8d85a4 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 2 Aug 2019 18:55:02 -0700 Subject: [PATCH 10/24] oof --- .../incremental/IncrementalIndexStorageAdapterTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index 19374455761a..d55651ff3f31 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -706,6 +706,12 @@ public Set getRequiredColumns() return null; } + @Override + public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + { + return true; + } + private class DictionaryRaceTestFilterDruidPredicateFactory implements DruidPredicateFactory { @Override From 6b86b3214f2b74c82aabdcb77a53802b87afdf8c Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 2 Aug 2019 19:33:16 -0700 Subject: [PATCH 11/24] this will fix it --- .../druid/query/filter/InDimFilterSerDesrTest.java | 10 +++++----- .../org/apache/druid/sql/calcite/CalciteQueryTest.java | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java index 7f9f9e872ec2..63e293eac2a1 100644 --- a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java +++ b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java @@ -37,8 +37,8 @@ public class InDimFilterSerDesrTest { private static ObjectMapper mapper; - private final String actualInFilter = - "{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[\"bad\",\"good\"],\"extractionFn\":null,\"filterTuning\":null}"; + private final String expectedFilter = + "{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[\"bad\",\"good\"],\"extractionFn\":null}"; @Before public void setUp() @@ -50,7 +50,7 @@ public void setUp() @Test public void testDeserialization() throws IOException { - final InDimFilter actualInDimFilter = mapper.readerFor(DimFilter.class).readValue(actualInFilter); + final InDimFilter actualInDimFilter = mapper.readerFor(DimFilter.class).readValue(expectedFilter); final InDimFilter expectedInDimFilter = new InDimFilter("dimTest", Arrays.asList("good", "bad"), null); Assert.assertEquals(expectedInDimFilter, actualInDimFilter); } @@ -59,8 +59,8 @@ public void testDeserialization() throws IOException public void testSerialization() throws IOException { final InDimFilter dimInFilter = new InDimFilter("dimTest", Arrays.asList("good", "bad"), null); - final String expectedInFilter = mapper.writeValueAsString(dimInFilter); - Assert.assertEquals(expectedInFilter, actualInFilter); + final String actualFilter = mapper.writeValueAsString(dimInFilter); + Assert.assertEquals(expectedFilter, actualFilter); } @Test diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 55884fa48604..12448503a196 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -2266,7 +2266,7 @@ public void testExplainCountStarOnView() throws Exception + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + "\"descending\":false," + "\"virtualColumns\":[]," - + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null,\"filterTuning\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1},\"filterTuning\":null}}]}," + + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]}," + "\"granularity\":{\"type\":\"all\"}," + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," + "\"postAggregations\":[]," @@ -4888,7 +4888,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception final String explanation = "DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"skipEmptyBuckets\":true,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\"}}], signature=[{a0:LONG}])\n" + " DruidSemiJoin(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\"},\"descending\":false}], leftExpressions=[[SUBSTRING($3, 1, 1)]], rightKeys=[[0]])\n" - + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null,\"filterTuning\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n"; + + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n"; testQuery( "EXPLAIN PLAN FOR SELECT COUNT(*)\n" From bb48537fc1c987243bef44b4e12b041bd7f3d012 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 5 Aug 2019 15:05:41 -0700 Subject: [PATCH 12/24] more nullable, refactor DimFilter.getRequiredColumns to use Set, formatting --- .../druid/query/filter/BloomDimFilter.java | 43 ++++++++++--------- .../druid/query/filter/AndDimFilter.java | 3 +- .../druid/query/filter/BoundDimFilter.java | 42 ++++++++++-------- .../filter/ColumnComparisonDimFilter.java | 18 +++----- .../apache/druid/query/filter/DimFilter.java | 4 +- .../query/filter/ExpressionDimFilter.java | 17 ++++---- .../query/filter/ExtractionDimFilter.java | 8 ++-- .../druid/query/filter/InDimFilter.java | 22 +++++----- .../druid/query/filter/IntervalDimFilter.java | 25 +++++------ .../query/filter/JavaScriptDimFilter.java | 21 +++++---- .../druid/query/filter/LikeDimFilter.java | 26 ++++++----- .../druid/query/filter/NotDimFilter.java | 4 +- .../druid/query/filter/OrDimFilter.java | 3 +- .../druid/query/filter/RegexDimFilter.java | 23 +++++----- .../query/filter/SearchQueryDimFilter.java | 17 +++++--- .../druid/query/filter/SelectorDimFilter.java | 22 ++++------ .../druid/query/filter/SpatialDimFilter.java | 18 ++++---- .../druid/query/filter/TrueDimFilter.java | 9 ++-- 18 files changed, 170 insertions(+), 155 deletions(-) diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java index c939f4874fbb..78838781a1bf 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java @@ -25,16 +25,17 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.hash.HashCode; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.segment.filter.DimensionPredicateFilter; -import java.util.HashSet; +import javax.annotation.Nullable; import java.util.Objects; +import java.util.Set; /** */ @@ -44,15 +45,17 @@ public class BloomDimFilter implements DimFilter private final String dimension; private final BloomKFilter bloomKFilter; private final HashCode hash; + @Nullable private final ExtractionFn extractionFn; + @Nullable private final FilterTuning filterTuning; @JsonCreator public BloomDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("bloomKFilter") BloomKFilterHolder bloomKFilterHolder, - @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("filterTuning") FilterTuning filterTuning + @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -65,11 +68,7 @@ public BloomDimFilter( } @VisibleForTesting - public BloomDimFilter( - String dimension, - BloomKFilterHolder bloomKFilterHolder, - ExtractionFn extractionFn - ) + public BloomDimFilter(String dimension, BloomKFilterHolder bloomKFilterHolder, @Nullable ExtractionFn extractionFn) { this(dimension, bloomKFilterHolder, extractionFn, null); } @@ -185,12 +184,14 @@ public BloomKFilter getBloomKFilter() return bloomKFilter; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() @@ -198,6 +199,18 @@ public FilterTuning getFilterTuning() return filterTuning; } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + + @Override + public Set getRequiredColumns() + { + return ImmutableSet.of(dimension); + } + @Override public String toString() { @@ -229,16 +242,4 @@ public int hashCode() { return Objects.hash(dimension, hash, extractionFn, filterTuning); } - - @Override - public RangeSet getDimensionRangeSet(String dimension) - { - return null; - } - - @Override - public HashSet getRequiredColumns() - { - return Sets.newHashSet(dimension); - } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java index 9903633609b4..682c2470c30d 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/AndDimFilter.java @@ -32,6 +32,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; /** */ @@ -99,7 +100,7 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { HashSet requiredColumns = new HashSet<>(); fields.forEach(field -> requiredColumns.addAll(field.getRequiredColumns())); diff --git a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java index 54e73de25ce7..55aa1fd69f48 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java @@ -26,9 +26,9 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.collect.BoundType; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; @@ -43,34 +43,40 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.Objects; +import java.util.Set; public class BoundDimFilter implements DimFilter { private final String dimension; + @Nullable private final String upper; + @Nullable private final String lower; + @Nullable private final boolean lowerStrict; + @Nullable private final boolean upperStrict; + @Nullable private final ExtractionFn extractionFn; private final StringComparator ordering; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; private final Supplier doublePredicateSupplier; + @Nullable private final FilterTuning filterTuning; @JsonCreator public BoundDimFilter( @JsonProperty("dimension") String dimension, - @JsonProperty("lower") String lower, - @JsonProperty("upper") String upper, - @JsonProperty("lowerStrict") Boolean lowerStrict, - @JsonProperty("upperStrict") Boolean upperStrict, + @Nullable @JsonProperty("lower") String lower, + @Nullable @JsonProperty("upper") String upper, + @Nullable @JsonProperty("lowerStrict") Boolean lowerStrict, + @Nullable @JsonProperty("upperStrict") Boolean upperStrict, @Deprecated @JsonProperty("alphaNumeric") Boolean alphaNumeric, - @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("ordering") StringComparator ordering, - @JsonProperty("filterTuning") FilterTuning filterTuning + @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, + @Nullable @JsonProperty("ordering") StringComparator ordering, + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be null"); @@ -108,13 +114,13 @@ public BoundDimFilter( @VisibleForTesting public BoundDimFilter( String dimension, - String lower, - String upper, - Boolean lowerStrict, - Boolean upperStrict, - Boolean alphaNumeric, - ExtractionFn extractionFn, - StringComparator ordering + @Nullable String lower, + @Nullable String upper, + @Nullable Boolean lowerStrict, + @Nullable Boolean upperStrict, + @Nullable Boolean alphaNumeric, + @Nullable ExtractionFn extractionFn, + @Nullable StringComparator ordering ) { this(dimension, lower, upper, lowerStrict, upperStrict, alphaNumeric, extractionFn, ordering, null); @@ -276,9 +282,9 @@ && getExtractionFn() == null } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java index 0560b2aeced7..79746bb98e33 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java @@ -26,14 +26,14 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.filter.ColumnComparisonFilter; -import java.util.HashSet; +import javax.annotation.Nullable; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; /** @@ -43,12 +43,13 @@ public class ColumnComparisonDimFilter implements DimFilter private static final Joiner COMMA_JOINER = Joiner.on(", "); private final List dimensions; + @Nullable private final FilterTuning filterTuning; @JsonCreator public ColumnComparisonDimFilter( @JsonProperty("dimensions") List dimensions, - @JsonProperty("filterTuning") FilterTuning filterTuning + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning ) { this.dimensions = Preconditions.checkNotNull(dimensions, "dimensions"); @@ -57,9 +58,7 @@ public ColumnComparisonDimFilter( } @VisibleForTesting - public ColumnComparisonDimFilter( - List dimensions - ) + public ColumnComparisonDimFilter(List dimensions) { this(dimensions, null); } @@ -134,11 +133,8 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimensions.stream() - .map(DimensionSpec::getDimension) - .collect(Collectors.toSet()) - ); + return dimensions.stream().map(DimensionSpec::getDimension).collect(Collectors.toSet()); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java index 81a4c5a41cac..0c02ba437a4a 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java @@ -24,7 +24,7 @@ import com.google.common.collect.RangeSet; import org.apache.druid.java.util.common.Cacheable; -import java.util.HashSet; +import java.util.Set; /** */ @@ -81,5 +81,5 @@ public interface DimFilter extends Cacheable /** * @return a HashSet that represents all columns' name which the DimFilter required to do filter. */ - HashSet getRequiredColumns(); + Set getRequiredColumns(); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java index 3b735e6144ea..d44c3776cec9 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java @@ -27,26 +27,27 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.Parser; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.filter.ExpressionFilter; -import java.util.HashSet; +import javax.annotation.Nullable; import java.util.Objects; +import java.util.Set; public class ExpressionDimFilter implements DimFilter { private final String expression; private final Supplier parsed; + @Nullable private final FilterTuning filterTuning; @JsonCreator public ExpressionDimFilter( @JsonProperty("expression") final String expression, - @JsonProperty("filterTuning") final FilterTuning filterTuning, + @Nullable @JsonProperty("filterTuning") final FilterTuning filterTuning, @JacksonInject ExprMacroTable macroTable ) { @@ -56,10 +57,7 @@ public ExpressionDimFilter( } @VisibleForTesting - public ExpressionDimFilter( - final String expression, - ExprMacroTable macroTable - ) + public ExpressionDimFilter(final String expression, ExprMacroTable macroTable) { this(expression, null, macroTable); } @@ -70,6 +68,7 @@ public String getExpression() return expression; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() @@ -96,9 +95,9 @@ public RangeSet getDimensionRangeSet(final String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(parsed.get().analyzeInputs().getRequiredBindings()); + return parsed.get().analyzeInputs().getRequiredBindings(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/ExtractionDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ExtractionDimFilter.java index 4387ce50f393..fe84a9e340eb 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ExtractionDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ExtractionDimFilter.java @@ -22,13 +22,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.extraction.ExtractionFn; import java.nio.ByteBuffer; -import java.util.HashSet; +import java.util.Set; /** * This class is deprecated, use SelectorDimFilter instead: {@link SelectorDimFilter} @@ -113,9 +113,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java index 5ca756d6233c..9b690941a767 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java @@ -26,10 +26,10 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; @@ -47,10 +47,10 @@ import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.filter.InFilter; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -66,7 +66,9 @@ public class InDimFilter implements DimFilter // Values can contain `null` object private final SortedSet values; private final String dimension; + @Nullable private final ExtractionFn extractionFn; + @Nullable private final FilterTuning filterTuning; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; @@ -76,8 +78,8 @@ public class InDimFilter implements DimFilter public InDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("values") Collection values, - @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("filterTuning") FilterTuning filterTuning + @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkNotNull(dimension, "dimension can not be null"); @@ -96,11 +98,7 @@ public InDimFilter( } @VisibleForTesting - public InDimFilter( - String dimension, - Collection values, - ExtractionFn extractionFn - ) + public InDimFilter(String dimension, Collection values, @Nullable ExtractionFn extractionFn) { this(dimension, values, extractionFn, null); } @@ -117,12 +115,14 @@ public Set getValues() return values; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() @@ -233,9 +233,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java index 7660a95ec68d..d900fdc0e26c 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java @@ -24,8 +24,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.primitives.Longs; import org.apache.druid.java.util.common.JodaUtils; import org.apache.druid.java.util.common.Pair; @@ -34,28 +34,31 @@ import org.apache.druid.query.ordering.StringComparators; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; public class IntervalDimFilter implements DimFilter { private final List intervals; private final List> intervalLongs; private final String dimension; + @Nullable private final ExtractionFn extractionFn; - private final OrDimFilter convertedFilter; + @Nullable private final FilterTuning filterTuning; + private final OrDimFilter convertedFilter; @JsonCreator public IntervalDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("intervals") List intervals, - @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("filterTuning") FilterTuning filterTuning + @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkNotNull(dimension, "dimension can not be null"); @@ -70,11 +73,7 @@ public IntervalDimFilter( } @VisibleForTesting - public IntervalDimFilter( - String dimension, - List intervals, - ExtractionFn extractionFn - ) + public IntervalDimFilter(String dimension, List intervals, @Nullable ExtractionFn extractionFn) { this(dimension, intervals, extractionFn, null); } @@ -91,12 +90,14 @@ public List getIntervals() return intervals; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() @@ -148,9 +149,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java index 5e443a080084..f82d12f53f57 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java @@ -26,8 +26,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.js.JavaScriptConfig; import org.apache.druid.query.extraction.ExtractionFn; @@ -38,17 +38,20 @@ import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; +import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.Objects; +import java.util.Set; public class JavaScriptDimFilter implements DimFilter { private final String dimension; private final String function; + @Nullable private final ExtractionFn extractionFn; - private final JavaScriptConfig config; + @Nullable private final FilterTuning filterTuning; + private final JavaScriptConfig config; /** * The field is declared volatile in order to ensure safe publication of the object @@ -65,8 +68,8 @@ public class JavaScriptDimFilter implements DimFilter public JavaScriptDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("function") String function, - @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("filterTuning") FilterTuning filterTuning, + @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning, @JacksonInject JavaScriptConfig config ) { @@ -83,7 +86,7 @@ public JavaScriptDimFilter( public JavaScriptDimFilter( String dimension, String function, - ExtractionFn extractionFn, + @Nullable ExtractionFn extractionFn, JavaScriptConfig config ) { @@ -102,12 +105,14 @@ public String getFunction() return function; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() @@ -176,9 +181,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java index e2ab1138b533..760f7a2002e3 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java @@ -25,8 +25,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; import org.apache.druid.common.config.NullHandling; @@ -37,8 +37,8 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.Objects; +import java.util.Set; import java.util.regex.Pattern; public class LikeDimFilter implements DimFilter @@ -50,18 +50,21 @@ public class LikeDimFilter implements DimFilter private final String dimension; private final String pattern; + @Nullable private final Character escapeChar; + @Nullable private final ExtractionFn extractionFn; - private final LikeMatcher likeMatcher; + @Nullable private final FilterTuning filterTuning; + private final LikeMatcher likeMatcher; @JsonCreator public LikeDimFilter( @JsonProperty("dimension") final String dimension, @JsonProperty("pattern") final String pattern, - @JsonProperty("escape") final String escape, - @JsonProperty("extractionFn") final ExtractionFn extractionFn, - @JsonProperty("filterTuning") final FilterTuning filterTuning + @Nullable @JsonProperty("escape") final String escape, + @Nullable @JsonProperty("extractionFn") final ExtractionFn extractionFn, + @Nullable @JsonProperty("filterTuning") final FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension"); @@ -82,8 +85,8 @@ public LikeDimFilter( public LikeDimFilter( final String dimension, final String pattern, - final String escape, - final ExtractionFn extractionFn + @Nullable final String escape, + @Nullable final ExtractionFn extractionFn ) { this(dimension, pattern, escape, extractionFn, null); @@ -101,18 +104,21 @@ public String getPattern() return pattern; } + @Nullable @JsonProperty public String getEscape() { return escapeChar != null ? escapeChar.toString() : null; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() @@ -159,9 +165,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java index 23f968074c7b..039667eb7edb 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/NotDimFilter.java @@ -27,8 +27,8 @@ import org.apache.druid.segment.filter.NotFilter; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.List; +import java.util.Set; /** */ @@ -102,7 +102,7 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { return field.getRequiredColumns(); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java index d2ed4e77bbde..762ad72ea5fe 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/OrDimFilter.java @@ -33,6 +33,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; /** */ @@ -107,7 +108,7 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { HashSet requiredColumns = new HashSet<>(); fields.forEach(field -> requiredColumns.addAll(field.getRequiredColumns())); diff --git a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java index 71fb80baf09d..3fccf65b3673 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java @@ -24,15 +24,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.segment.filter.RegexFilter; +import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.Objects; +import java.util.Set; import java.util.regex.Pattern; /** @@ -41,7 +42,9 @@ public class RegexDimFilter implements DimFilter { private final String dimension; private final String pattern; + @Nullable private final ExtractionFn extractionFn; + @Nullable private final FilterTuning filterTuning; private final Pattern compiledPattern; @@ -50,8 +53,8 @@ public class RegexDimFilter implements DimFilter public RegexDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("pattern") String pattern, - @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("filterTuning") FilterTuning filterTuning + @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -64,11 +67,7 @@ public RegexDimFilter( } @VisibleForTesting - public RegexDimFilter( - String dimension, - String pattern, - ExtractionFn extractionFn - ) + public RegexDimFilter(String dimension, String pattern, @Nullable ExtractionFn extractionFn) { this(dimension, pattern, extractionFn, null); } @@ -85,12 +84,14 @@ public String getPattern() return pattern; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() @@ -134,9 +135,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java index 051fc420ef60..b93a58e1897d 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java @@ -23,16 +23,17 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.search.SearchQuerySpec; import org.apache.druid.segment.filter.SearchQueryFilter; +import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.Objects; +import java.util.Set; /** */ @@ -40,14 +41,16 @@ public class SearchQueryDimFilter implements DimFilter { private final String dimension; private final SearchQuerySpec query; + @Nullable private final ExtractionFn extractionFn; + @Nullable private final FilterTuning filterTuning; public SearchQueryDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("query") SearchQuerySpec query, - @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("filterTuning") FilterTuning filterTuning + @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -63,7 +66,7 @@ public SearchQueryDimFilter( public SearchQueryDimFilter( String dimension, SearchQuerySpec query, - ExtractionFn extractionFn + @Nullable ExtractionFn extractionFn ) { this(dimension, query, extractionFn, null); @@ -130,9 +133,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java index 387938f88d62..c413dc6052e1 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java @@ -22,13 +22,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; @@ -42,8 +41,8 @@ import javax.annotation.Nullable; import java.util.Collections; -import java.util.HashSet; import java.util.Objects; +import java.util.Set; /** */ @@ -53,7 +52,9 @@ public class SelectorDimFilter implements DimFilter @Nullable private final String value; + @Nullable private final ExtractionFn extractionFn; + @Nullable private final FilterTuning filterTuning; private final Object initLock = new Object(); @@ -66,8 +67,8 @@ public class SelectorDimFilter implements DimFilter public SelectorDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("value") String value, - @JsonProperty("extractionFn") ExtractionFn extractionFn, - @JsonProperty("filterTuning") FilterTuning filterTuning + @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -78,12 +79,7 @@ public SelectorDimFilter( this.filterTuning = filterTuning; } - @VisibleForTesting - public SelectorDimFilter( - String dimension, - String value, - ExtractionFn extractionFn - ) + public SelectorDimFilter(String dimension, String value, @Nullable ExtractionFn extractionFn) { this(dimension, value, extractionFn, null); } @@ -224,9 +220,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java index be591070f2c5..844c1f9b2caa 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java @@ -24,15 +24,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; -import com.google.common.collect.Sets; import org.apache.druid.collections.spatial.search.Bound; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.segment.filter.SpatialFilter; +import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.Objects; +import java.util.Set; /** */ @@ -40,13 +41,14 @@ public class SpatialDimFilter implements DimFilter { private final String dimension; private final Bound bound; + @Nullable private final FilterTuning filterTuning; @JsonCreator public SpatialDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("bound") Bound bound, - @JsonProperty("filterTuning") FilterTuning filterTuning + @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); @@ -58,10 +60,7 @@ public SpatialDimFilter( } @VisibleForTesting - public SpatialDimFilter( - String dimension, - Bound bound - ) + public SpatialDimFilter(String dimension, Bound bound) { this(dimension, bound, null); } @@ -98,6 +97,7 @@ public Bound getBound() return bound; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() @@ -118,9 +118,9 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return Sets.newHashSet(dimension); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java index 405bece232bd..d10e6d9b8b5b 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/TrueDimFilter.java @@ -23,14 +23,13 @@ import org.apache.druid.segment.filter.TrueFilter; import java.nio.ByteBuffer; -import java.util.HashSet; +import java.util.Collections; +import java.util.Set; /** */ public class TrueDimFilter implements DimFilter { - private static HashSet REQUIRED_COLUMNS = new HashSet<>(); - @Override public byte[] getCacheKey() { @@ -56,8 +55,8 @@ public RangeSet getDimensionRangeSet(String dimension) } @Override - public HashSet getRequiredColumns() + public Set getRequiredColumns() { - return REQUIRED_COLUMNS; + return Collections.emptySet(); } } From d0d7f05e883b212459863f835955b2570f37ee47 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 5 Aug 2019 16:27:09 -0700 Subject: [PATCH 13/24] extract class DimFilterToStringBuilder with common code from custom DimFilter toString implementations --- .../druid/query/filter/BloomDimFilter.java | 10 ++- .../druid/query/filter/BoundDimFilter.java | 10 +-- .../apache/druid/query/filter/DimFilter.java | 66 +++++++++++++++++++ .../druid/query/filter/InDimFilter.java | 27 ++------ .../druid/query/filter/LikeDimFilter.java | 19 +----- .../druid/query/filter/SelectorDimFilter.java | 10 ++- 6 files changed, 87 insertions(+), 55 deletions(-) diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java index 78838781a1bf..2751e38726e4 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java @@ -28,7 +28,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.RangeSet; import com.google.common.hash.HashCode; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.segment.filter.DimensionPredicateFilter; @@ -214,11 +213,10 @@ public Set getRequiredColumns() @Override public String toString() { - if (extractionFn != null) { - return StringUtils.format("%s(%s) = %s", extractionFn, dimension, hash.toString()); - } else { - return StringUtils.format("%s = %s", dimension, hash.toString()); - } + return new DimFilterToStringBuilder().appendDimension(dimension, extractionFn) + .appendEquals(hash.toString()) + .appendFilterTuning(filterTuning) + .build(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java index 55aa1fd69f48..ed0374df476b 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java @@ -325,7 +325,7 @@ public int hashCode() @Override public String toString() { - final StringBuilder builder = new StringBuilder(); + final DimFilterToStringBuilder builder = new DimFilterToStringBuilder(); if (lower != null) { builder.append(lower); @@ -336,11 +336,7 @@ public String toString() } } - if (extractionFn != null) { - builder.append(StringUtils.format("%s(%s)", extractionFn, dimension)); - } else { - builder.append(dimension); - } + builder.appendDimension(dimension, extractionFn); if (!ordering.equals(StringComparators.LEXICOGRAPHIC)) { builder.append(StringUtils.format(" as %s", ordering.toString())); @@ -355,7 +351,7 @@ public String toString() builder.append(upper); } - return builder.toString(); + return builder.appendFilterTuning(filterTuning).build(); } private Supplier makeLongPredicateSupplier() diff --git a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java index 0c02ba437a4a..66d02270d2fa 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.collect.RangeSet; import org.apache.druid.java.util.common.Cacheable; +import org.apache.druid.query.extraction.ExtractionFn; import java.util.Set; @@ -82,4 +83,69 @@ public interface DimFilter extends Cacheable * @return a HashSet that represents all columns' name which the DimFilter required to do filter. */ Set getRequiredColumns(); + + /** + * Wrapper for {@link StringBuilder} to re-use common patterns in custom {@link DimFilter#toString()} implementations + */ + class DimFilterToStringBuilder + { + private final StringBuilder builder; + + public DimFilterToStringBuilder() + { + this.builder = new StringBuilder(); + } + + /** + * Append dimension name OR {@link ExtractionFn#toString()} with dimension wrapped in parenthesis + */ + DimFilterToStringBuilder appendDimension(String dimension, ExtractionFn extractionFn) + { + if (extractionFn != null) { + builder.append(extractionFn).append("("); + } + + builder.append(dimension); + + if (extractionFn != null) { + builder.append(")"); + } + return this; + } + + /** + * Add "=" expression + */ + DimFilterToStringBuilder appendEquals(String value) + { + builder.append(" = ").append(value); + return this; + } + + /** + * Add filter tuning to {@link #builder} if tuning exists + */ + DimFilterToStringBuilder appendFilterTuning(FilterTuning tuning) + { + if (tuning != null) { + builder.append(" (filterTuning=").append(tuning).append(")"); + } + + return this; + } + + /** + * Generic passthrough to {@link StringBuilder#append} + */ + DimFilterToStringBuilder append(T s) + { + builder.append(s); + return this; + } + + public String build() + { + return builder.toString(); + } + } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java index 9b690941a767..f189291642cf 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java @@ -241,26 +241,13 @@ public Set getRequiredColumns() @Override public String toString() { - final StringBuilder builder = new StringBuilder(); - - if (extractionFn != null) { - builder.append(extractionFn).append("("); - } - - builder.append(dimension); - - if (extractionFn != null) { - builder.append(")"); - } - - builder.append(" IN (") - .append( - Joiner.on(", ").join( - Iterables.transform(values, input -> StringUtils.nullToEmptyNonDruidDataString(input)) - ) - ) - .append(")"); - return builder.toString(); + final DimFilterToStringBuilder builder = new DimFilterToStringBuilder(); + return builder.appendDimension(dimension, extractionFn) + .append(" IN (") + .append(Joiner.on(", ").join(Iterables.transform(values, StringUtils::nullToEmptyNonDruidDataString))) + .append(")") + .appendFilterTuning(filterTuning) + .build(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java index 760f7a2002e3..92265a980c1c 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java @@ -196,25 +196,12 @@ public int hashCode() @Override public String toString() { - final StringBuilder builder = new StringBuilder(); - - if (extractionFn != null) { - builder.append(extractionFn).append("("); - } - - builder.append(dimension); - - if (extractionFn != null) { - builder.append(")"); - } - - builder.append(" LIKE '").append(pattern).append("'"); - + final DimFilterToStringBuilder builder = new DimFilterToStringBuilder(); + builder.appendDimension(dimension, extractionFn).append(" LIKE '").append(pattern).append("'"); if (escapeChar != null) { builder.append(" ESCAPE '").append(escapeChar).append("'"); } - - return builder.toString(); + return builder.appendFilterTuning(filterTuning).build(); } public static class LikeMatcher diff --git a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java index c413dc6052e1..d39a7d7beb13 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java @@ -33,7 +33,6 @@ import com.google.common.primitives.Floats; import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.guava.GuavaUtils; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.segment.filter.DimensionPredicateFilter; @@ -172,11 +171,10 @@ public FilterTuning getFilterTuning() @Override public String toString() { - if (extractionFn != null) { - return StringUtils.format("%s(%s) = %s", extractionFn, dimension, value); - } else { - return StringUtils.format("%s = %s", dimension, value); - } + return new DimFilterToStringBuilder().appendDimension(dimension, extractionFn) + .appendEquals(value) + .appendFilterTuning(filterTuning) + .build(); } @Override From 1aa5484b1bc05f8939e16fbe7a5a33c25cb61d2d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 5 Aug 2019 16:42:59 -0700 Subject: [PATCH 14/24] adjust variable naming --- .../apache/druid/query/filter/InDimFilterSerDesrTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java index 63e293eac2a1..691769aa7ba4 100644 --- a/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java +++ b/processing/src/test/java/org/apache/druid/query/filter/InDimFilterSerDesrTest.java @@ -37,7 +37,7 @@ public class InDimFilterSerDesrTest { private static ObjectMapper mapper; - private final String expectedFilter = + private final String serializedFilter = "{\"type\":\"in\",\"dimension\":\"dimTest\",\"values\":[\"bad\",\"good\"],\"extractionFn\":null}"; @Before @@ -50,7 +50,7 @@ public void setUp() @Test public void testDeserialization() throws IOException { - final InDimFilter actualInDimFilter = mapper.readerFor(DimFilter.class).readValue(expectedFilter); + final InDimFilter actualInDimFilter = mapper.readerFor(DimFilter.class).readValue(serializedFilter); final InDimFilter expectedInDimFilter = new InDimFilter("dimTest", Arrays.asList("good", "bad"), null); Assert.assertEquals(expectedInDimFilter, actualInDimFilter); } @@ -59,8 +59,8 @@ public void testDeserialization() throws IOException public void testSerialization() throws IOException { final InDimFilter dimInFilter = new InDimFilter("dimTest", Arrays.asList("good", "bad"), null); - final String actualFilter = mapper.writeValueAsString(dimInFilter); - Assert.assertEquals(expectedFilter, actualFilter); + final String actualSerializedFilter = mapper.writeValueAsString(dimInFilter); + Assert.assertEquals(serializedFilter, actualSerializedFilter); } @Test From 359cf4f811b0a86442173b34f60968e13dd205fd Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 5 Aug 2019 18:13:13 -0700 Subject: [PATCH 15/24] missing nullable --- .../java/org/apache/druid/query/filter/BoundDimFilter.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java index ed0374df476b..18ea21fe8958 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java @@ -53,9 +53,7 @@ public class BoundDimFilter implements DimFilter private final String upper; @Nullable private final String lower; - @Nullable private final boolean lowerStrict; - @Nullable private final boolean upperStrict; @Nullable private final ExtractionFn extractionFn; @@ -132,12 +130,14 @@ public String getDimension() return dimension; } + @Nullable @JsonProperty public String getUpper() { return upper; } + @Nullable @JsonProperty public String getLower() { @@ -166,6 +166,7 @@ public boolean hasUpperBound() return upper != null; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { @@ -178,6 +179,7 @@ public StringComparator getOrdering() return ordering; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() From 2be81b045f5314db87d4b117ea8e00ca527e1236 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 5 Aug 2019 18:18:30 -0700 Subject: [PATCH 16/24] more nullable --- .../org/apache/druid/query/filter/SearchQueryDimFilter.java | 2 ++ .../java/org/apache/druid/query/filter/SelectorDimFilter.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java index b93a58e1897d..ae0986c7269e 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java @@ -84,12 +84,14 @@ public SearchQuerySpec getQuery() return query; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() diff --git a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java index d39a7d7beb13..ba02ff39a09f 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java @@ -149,18 +149,21 @@ public String getDimension() return dimension; } + @Nullable @JsonProperty public String getValue() { return value; } + @Nullable @JsonProperty public ExtractionFn getExtractionFn() { return extractionFn; } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty public FilterTuning getFilterTuning() From a6e6a315b78829ad661dc328cf7420cf2f992076 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 6 Aug 2019 01:07:25 -0700 Subject: [PATCH 17/24] fix javadocs --- .../java/org/apache/druid/query/filter/Filter.java | 6 +++--- .../java/org/apache/druid/segment/filter/Filters.java | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java index fe09977cb25b..cbb032a87fa7 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java @@ -140,9 +140,9 @@ default boolean canVectorizeMatcher() * Determine if a filter *should* use a bitmap index based on information collected from the supplied * {@link BitmapIndexSelector}. This method differs from {@link #supportsBitmapIndex(BitmapIndexSelector)} in that * the former only indicates if a bitmap index is available and {@link #getBitmapIndex(BitmapIndexSelector)} may be - * used. This method, by default, will consider a {@link FilterTuning} to make decisions about when to use an - * available index. Override this method in a {@link Filter} implementation when {@link FilterTuning} alone is not - * adequate for making this decision. + * used. Implementations of this methods typically consider a {@link FilterTuning} to make decisions about when to + * use an available index. A "standard" implementation of this is available to all {@link Filter} implementations in + * {@link org.apache.druid.segment.filter.Filters#shouldUseIndex(Filter, BitmapIndexSelector, FilterTuning)}. */ boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector); } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java index 563d229d1d54..7ef93aa3447d 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java @@ -653,6 +653,16 @@ private static void generateAllCombinations( } } + /** + * This method provides a "standard" implementation of {@link Filter#shouldUseIndex(BitmapIndexSelector)} which takes + * a {@link Filter}, a {@link BitmapIndexSelector}, and {@link FilterTuning} to determine if: + * a) the filter supports bitmap indexes + * b) the filter tuning specifies that it should use the index + * c) the cardinality of the column is above the minimum threshold and below the maximum threshold to use the index + * + * If all these things are true, {@link org.apache.druid.segment.QueryableIndexStorageAdapter} will utilize the + * indexes. + */ public static boolean shouldUseIndex(Filter filter, BitmapIndexSelector indexSelector, FilterTuning filterTuning) { final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector); From 346e8fd44896f6b28c3e2c680ecd773146136f86 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 6 Aug 2019 12:46:48 -0700 Subject: [PATCH 18/24] nullable --- .../main/java/org/apache/druid/segment/filter/Filters.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java index 7ef93aa3447d..ce5927c3892c 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java @@ -663,7 +663,11 @@ private static void generateAllCombinations( * If all these things are true, {@link org.apache.druid.segment.QueryableIndexStorageAdapter} will utilize the * indexes. */ - public static boolean shouldUseIndex(Filter filter, BitmapIndexSelector indexSelector, FilterTuning filterTuning) + public static boolean shouldUseIndex( + Filter filter, + BitmapIndexSelector indexSelector, + @Nullable FilterTuning filterTuning + ) { final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector); if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseIndex()) { From b8a3cae77b4042ad8f8a7b7291bde7bc6e676ab3 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 7 Aug 2019 14:58:29 -0700 Subject: [PATCH 19/24] address review comments --- .../MomentSketchAggregatorFactory.java | 4 +- .../TDigestSketchAggregatorFactory.java | 2 +- .../theta/SketchMergeAggregatorFactory.java | 8 ++-- .../bloom/BloomFilterAggregatorFactory.java | 2 +- .../druid/query/filter/BloomDimFilter.java | 4 +- ...ApproximateHistogramAggregatorFactory.java | 2 +- ...mateHistogramFoldingAggregatorFactory.java | 2 +- ...ixedBucketsHistogramAggregatorFactory.java | 4 +- .../namespace/JdbcExtractionNamespace.java | 6 +-- .../namespace/UriExtractionNamespace.java | 2 +- .../input/orc/OrcHadoopInputRowParser.java | 2 +- .../indexing/common/task/CompactionTask.java | 18 +++---- .../IngestSegmentFirehoseFactory.java | 4 +- .../overlord/ImmutableWorkerInfo.java | 2 +- .../FilteredAggregatorFactory.java | 2 +- .../query/extraction/BucketExtractionFn.java | 4 +- .../druid/query/filter/BooleanFilter.java | 34 +++++++++++++ .../druid/query/filter/BoundDimFilter.java | 16 +++---- .../filter/ColumnComparisonDimFilter.java | 2 +- .../apache/druid/query/filter/DimFilter.java | 5 +- .../query/filter/ExpressionDimFilter.java | 2 +- .../org/apache/druid/query/filter/Filter.java | 37 +++++++++----- .../druid/query/filter/FilterTuning.java | 48 +++++++++---------- .../druid/query/filter/InDimFilter.java | 4 +- .../druid/query/filter/IntervalDimFilter.java | 4 +- .../query/filter/JavaScriptDimFilter.java | 4 +- .../druid/query/filter/LikeDimFilter.java | 6 +-- .../druid/query/filter/RegexDimFilter.java | 4 +- .../query/filter/SearchQueryDimFilter.java | 4 +- .../druid/query/filter/SelectorDimFilter.java | 4 +- .../druid/query/filter/SpatialDimFilter.java | 2 +- .../query/lookup/LookupExtractionFn.java | 6 +-- .../lookup/RegisteredLookupExtractionFn.java | 4 +- .../druid/query/scan/ScanResultValue.java | 2 +- .../apache/druid/segment/FilteredOffset.java | 2 +- .../segment/QueryableIndexStorageAdapter.java | 16 +++---- .../druid/segment/filter/AndFilter.java | 37 -------------- .../druid/segment/filter/BoundFilter.java | 13 +++-- .../filter/ColumnComparisonFilter.java | 4 +- .../filter/DimensionPredicateFilter.java | 4 +- .../segment/filter/ExpressionFilter.java | 4 +- .../apache/druid/segment/filter/Filters.java | 8 ++-- .../apache/druid/segment/filter/InFilter.java | 4 +- .../segment/filter/JavaScriptFilter.java | 12 ++--- .../druid/segment/filter/LikeFilter.java | 4 +- .../druid/segment/filter/NotFilter.java | 4 +- .../apache/druid/segment/filter/OrFilter.java | 34 ------------- .../druid/segment/filter/SelectorFilter.java | 5 +- .../druid/segment/filter/SpatialFilter.java | 12 ++--- .../druid/segment/filter/TrueFilter.java | 2 +- .../DictionaryEncodedColumnPartSerde.java | 2 +- .../serde/DoubleNumericColumnPartSerdeV2.java | 2 +- .../serde/FloatNumericColumnPartSerdeV2.java | 2 +- .../serde/LongNumericColumnPartSerdeV2.java | 2 +- .../druid/segment/filter/BaseFilterTest.java | 23 ++++----- .../IncrementalIndexStorageAdapterTest.java | 10 ++-- .../client/indexing/ClientCompactQuery.java | 4 +- .../client/indexing/IndexingWorkerInfo.java | 2 +- .../supervisor/NoopSupervisorSpec.java | 6 +-- .../druid/sql/calcite/filtration/Bounds.java | 2 +- 60 files changed, 227 insertions(+), 249 deletions(-) diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java index 1abe6a1a9ae6..75f70ae3d5bf 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java @@ -71,8 +71,8 @@ public class MomentSketchAggregatorFactory extends AggregatorFactory public MomentSketchAggregatorFactory( @JsonProperty("name") final String name, @JsonProperty("fieldName") final String fieldName, - @Nullable @JsonProperty("k") final Integer k, - @Nullable @JsonProperty("compress") final Boolean compress + @JsonProperty("k") @Nullable final Integer k, + @JsonProperty("compress") @Nullable final Boolean compress ) { this(name, fieldName, k, compress, AggregatorUtil.MOMENTS_SKETCH_BUILD_CACHE_TYPE_ID); diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java index b9d34f377d7e..74caf612e92c 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java @@ -76,7 +76,7 @@ public class TDigestSketchAggregatorFactory extends AggregatorFactory public TDigestSketchAggregatorFactory( @JsonProperty("name") final String name, @JsonProperty("fieldName") final String fieldName, - @Nullable @JsonProperty("compression") final Integer compression + @JsonProperty("compression") @Nullable final Integer compression ) { this(name, fieldName, compression, AggregatorUtil.TDIGEST_BUILD_SKETCH_CACHE_TYPE_ID); diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java index 2cc5388a5382..4d9af8c7cbdf 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java @@ -39,10 +39,10 @@ public class SketchMergeAggregatorFactory extends SketchAggregatorFactory public SketchMergeAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("fieldName") String fieldName, - @Nullable @JsonProperty("size") Integer size, - @Nullable @JsonProperty("shouldFinalize") Boolean shouldFinalize, - @Nullable @JsonProperty("isInputThetaSketch") Boolean isInputThetaSketch, - @Nullable @JsonProperty("errorBoundsStdDev") Integer errorBoundsStdDev + @JsonProperty("size") @Nullable Integer size, + @JsonProperty("shouldFinalize") @Nullable Boolean shouldFinalize, + @JsonProperty("isInputThetaSketch") @Nullable Boolean isInputThetaSketch, + @JsonProperty("errorBoundsStdDev") @Nullable Integer errorBoundsStdDev ) { super(name, fieldName, size, AggregatorUtil.SKETCH_MERGE_CACHE_TYPE_ID); diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java index 53839e82f7ad..d158be45a207 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java @@ -71,7 +71,7 @@ public class BloomFilterAggregatorFactory extends AggregatorFactory public BloomFilterAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("field") final DimensionSpec field, - @Nullable @JsonProperty("maxNumEntries") Integer maxNumEntries + @JsonProperty("maxNumEntries") @Nullable Integer maxNumEntries ) { this.name = name; diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java index 2751e38726e4..691826931459 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java @@ -53,8 +53,8 @@ public class BloomDimFilter implements DimFilter public BloomDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("bloomKFilter") BloomKFilterHolder bloomKFilterHolder, - @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java index 793a7fd2073a..bc1295466764 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java @@ -65,7 +65,7 @@ public ApproximateHistogramAggregatorFactory( @JsonProperty("numBuckets") Integer numBuckets, @JsonProperty("lowerLimit") Float lowerLimit, @JsonProperty("upperLimit") Float upperLimit, - @Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary + @JsonProperty("finalizeAsBase64Binary") @Nullable Boolean finalizeAsBase64Binary ) { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java index 7f29cdddb064..becbfe1ae64f 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java @@ -46,7 +46,7 @@ public ApproximateHistogramFoldingAggregatorFactory( @JsonProperty("numBuckets") Integer numBuckets, @JsonProperty("lowerLimit") Float lowerLimit, @JsonProperty("upperLimit") Float upperLimit, - @Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary + @JsonProperty("finalizeAsBase64Binary") @Nullable Boolean finalizeAsBase64Binary ) { super(name, fieldName, resolution, numBuckets, lowerLimit, upperLimit, finalizeAsBase64Binary); diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java index 5c90678db793..b8df5b62e0cb 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java @@ -59,11 +59,11 @@ public class FixedBucketsHistogramAggregatorFactory extends AggregatorFactory public FixedBucketsHistogramAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("fieldName") String fieldName, - @Nullable @JsonProperty("numBuckets") Integer numBuckets, + @JsonProperty("numBuckets") @Nullable Integer numBuckets, @JsonProperty("lowerLimit") double lowerLimit, @JsonProperty("upperLimit") double upperLimit, @JsonProperty("outlierHandlingMode") FixedBucketsHistogram.OutlierHandlingMode outlierHandlingMode, - @Nullable @JsonProperty("finalizeAsBase64Binary") Boolean finalizeAsBase64Binary + @JsonProperty("finalizeAsBase64Binary") @Nullable Boolean finalizeAsBase64Binary ) { this.name = name; diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespace.java index 314cd72c1257..ce4eddf526b5 100644 --- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespace.java @@ -59,9 +59,9 @@ public JdbcExtractionNamespace( @NotNull @JsonProperty(value = "table", required = true) final String table, @NotNull @JsonProperty(value = "keyColumn", required = true) final String keyColumn, @NotNull @JsonProperty(value = "valueColumn", required = true) final String valueColumn, - @Nullable @JsonProperty(value = "tsColumn", required = false) final String tsColumn, - @Nullable @JsonProperty(value = "filter", required = false) final String filter, - @Min(0) @Nullable @JsonProperty(value = "pollPeriod", required = false) final Period pollPeriod + @JsonProperty(value = "tsColumn", required = false) @Nullable final String tsColumn, + @JsonProperty(value = "filter", required = false) @Nullable final String filter, + @Min(0) @JsonProperty(value = "pollPeriod", required = false) @Nullable final Period pollPeriod ) { this.connectorConfig = Preconditions.checkNotNull(connectorConfig, "connectorConfig"); diff --git a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/UriExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/UriExtractionNamespace.java index d828ca7c3cd1..75de73d6c200 100644 --- a/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/UriExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/namespace/UriExtractionNamespace.java @@ -85,7 +85,7 @@ public UriExtractionNamespace( String fileRegex, @JsonProperty(value = "namespaceParseSpec", required = true) FlatDataParser namespaceParseSpec, - @Min(0) @Nullable @JsonProperty(value = "pollPeriod", required = false) + @Min(0) @JsonProperty(value = "pollPeriod", required = false) @Nullable Period pollPeriod, @Deprecated @JsonProperty(value = "versionRegex", required = false) diff --git a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcHadoopInputRowParser.java b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcHadoopInputRowParser.java index 38b65d1cee2a..9fcefcba2a60 100644 --- a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcHadoopInputRowParser.java +++ b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcHadoopInputRowParser.java @@ -44,7 +44,7 @@ public class OrcHadoopInputRowParser implements InputRowParser @JsonCreator public OrcHadoopInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec, - @Nullable @JsonProperty("binaryAsString") Boolean binaryAsString + @JsonProperty("binaryAsString") @Nullable Boolean binaryAsString ) { this.parseSpec = parseSpec; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java index 1f16f01ca56a..8aee62f4fe33 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java @@ -156,15 +156,15 @@ public CompactionTask( @JsonProperty("id") final String id, @JsonProperty("resource") final TaskResource taskResource, @JsonProperty("dataSource") final String dataSource, - @Nullable @JsonProperty("interval") final Interval interval, - @Nullable @JsonProperty("segments") final List segments, - @Nullable @JsonProperty("dimensions") final DimensionsSpec dimensions, - @Nullable @JsonProperty("dimensionsSpec") final DimensionsSpec dimensionsSpec, - @Nullable @JsonProperty("metricsSpec") final AggregatorFactory[] metricsSpec, - @Nullable @JsonProperty("segmentGranularity") final Granularity segmentGranularity, - @Nullable @JsonProperty("targetCompactionSizeBytes") final Long targetCompactionSizeBytes, - @Nullable @JsonProperty("tuningConfig") final IndexTuningConfig tuningConfig, - @Nullable @JsonProperty("context") final Map context, + @JsonProperty("interval") @Nullable final Interval interval, + @JsonProperty("segments") @Nullable final List segments, + @JsonProperty("dimensions") @Nullable final DimensionsSpec dimensions, + @JsonProperty("dimensionsSpec") @Nullable final DimensionsSpec dimensionsSpec, + @JsonProperty("metricsSpec") @Nullable final AggregatorFactory[] metricsSpec, + @JsonProperty("segmentGranularity") @Nullable final Granularity segmentGranularity, + @JsonProperty("targetCompactionSizeBytes") @Nullable final Long targetCompactionSizeBytes, + @JsonProperty("tuningConfig") @Nullable final IndexTuningConfig tuningConfig, + @JsonProperty("context") @Nullable final Map context, @JacksonInject ObjectMapper jsonMapper, @JacksonInject AuthorizerMapper authorizerMapper, @JacksonInject ChatHandlerProvider chatHandlerProvider, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java b/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java index ccbcb50874bb..53969484128c 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactory.java @@ -101,10 +101,10 @@ public class IngestSegmentFirehoseFactory implements FiniteFirehoseFactory segmentIds, + @JsonProperty("segments") @Nullable List segmentIds, @JsonProperty("filter") DimFilter dimFilter, @JsonProperty("dimensions") List dimensions, @JsonProperty("metrics") List metrics, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfo.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfo.java index 11da1f6733ee..2c077581b04e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfo.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfo.java @@ -51,7 +51,7 @@ public ImmutableWorkerInfo( @JsonProperty("availabilityGroups") Set availabilityGroups, @JsonProperty("runningTasks") Collection runningTasks, @JsonProperty("lastCompletedTaskTime") DateTime lastCompletedTaskTime, - @Nullable @JsonProperty("blacklistedUntil") DateTime blacklistedUntil + @JsonProperty("blacklistedUntil") @Nullable DateTime blacklistedUntil ) { this.worker = worker; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java index e5a3ef7e28c3..bc35dc2c4d0a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java @@ -63,7 +63,7 @@ public FilteredAggregatorFactory( public FilteredAggregatorFactory( @JsonProperty("aggregator") AggregatorFactory delegate, @JsonProperty("filter") DimFilter dimFilter, - @Nullable @JsonProperty("name") String name + @JsonProperty("name") @Nullable String name ) { Preconditions.checkNotNull(delegate, "aggregator"); diff --git a/processing/src/main/java/org/apache/druid/query/extraction/BucketExtractionFn.java b/processing/src/main/java/org/apache/druid/query/extraction/BucketExtractionFn.java index 96180f04481c..970116e62169 100644 --- a/processing/src/main/java/org/apache/druid/query/extraction/BucketExtractionFn.java +++ b/processing/src/main/java/org/apache/druid/query/extraction/BucketExtractionFn.java @@ -34,8 +34,8 @@ public class BucketExtractionFn implements ExtractionFn @JsonCreator public BucketExtractionFn( - @Nullable @JsonProperty("size") Double size, - @Nullable @JsonProperty("offset") Double offset + @JsonProperty("size") @Nullable Double size, + @JsonProperty("offset") @Nullable Double offset ) { this.size = size == null ? 1 : size; diff --git a/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java index c22843120933..9bbbdb643eb4 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BooleanFilter.java @@ -19,6 +19,7 @@ package org.apache.druid.query.filter; +import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; import java.util.HashSet; @@ -60,4 +61,37 @@ default Set getRequiredColumns() } return allColumns; } + + @Override + default boolean supportsBitmapIndex(BitmapIndexSelector selector) + { + for (Filter filter : getFilters()) { + if (!filter.supportsBitmapIndex(selector)) { + return false; + } + } + return true; + } + + @Override + default boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + for (Filter f : getFilters()) { + if (!f.shouldUseBitmapIndex(selector)) { + return false; + } + } + return true; + } + + @Override + default boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) + { + for (Filter filter : getFilters()) { + if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) { + return false; + } + } + return true; + } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java index 18ea21fe8958..8b7491a54975 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BoundDimFilter.java @@ -67,14 +67,14 @@ public class BoundDimFilter implements DimFilter @JsonCreator public BoundDimFilter( @JsonProperty("dimension") String dimension, - @Nullable @JsonProperty("lower") String lower, - @Nullable @JsonProperty("upper") String upper, - @Nullable @JsonProperty("lowerStrict") Boolean lowerStrict, - @Nullable @JsonProperty("upperStrict") Boolean upperStrict, - @Deprecated @JsonProperty("alphaNumeric") Boolean alphaNumeric, - @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, - @Nullable @JsonProperty("ordering") StringComparator ordering, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning + @JsonProperty("lower") @Nullable String lower, + @JsonProperty("upper") @Nullable String upper, + @JsonProperty("lowerStrict") @Nullable Boolean lowerStrict, + @JsonProperty("upperStrict") @Nullable Boolean upperStrict, + @Deprecated @JsonProperty("alphaNumeric") @Nullable Boolean alphaNumeric, + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("ordering") @Nullable StringComparator ordering, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be null"); diff --git a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java index 79746bb98e33..8d7edec15ed7 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java @@ -49,7 +49,7 @@ public class ColumnComparisonDimFilter implements DimFilter @JsonCreator public ColumnComparisonDimFilter( @JsonProperty("dimensions") List dimensions, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { this.dimensions = Preconditions.checkNotNull(dimensions, "dimensions"); diff --git a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java index 66d02270d2fa..7688d362b3e8 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/DimFilter.java @@ -25,6 +25,7 @@ import org.apache.druid.java.util.common.Cacheable; import org.apache.druid.query.extraction.ExtractionFn; +import javax.annotation.Nullable; import java.util.Set; /** @@ -99,7 +100,7 @@ public DimFilterToStringBuilder() /** * Append dimension name OR {@link ExtractionFn#toString()} with dimension wrapped in parenthesis */ - DimFilterToStringBuilder appendDimension(String dimension, ExtractionFn extractionFn) + DimFilterToStringBuilder appendDimension(String dimension, @Nullable ExtractionFn extractionFn) { if (extractionFn != null) { builder.append(extractionFn).append("("); @@ -125,7 +126,7 @@ DimFilterToStringBuilder appendEquals(String value) /** * Add filter tuning to {@link #builder} if tuning exists */ - DimFilterToStringBuilder appendFilterTuning(FilterTuning tuning) + DimFilterToStringBuilder appendFilterTuning(@Nullable FilterTuning tuning) { if (tuning != null) { builder.append(" (filterTuning=").append(tuning).append(")"); diff --git a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java index d44c3776cec9..d902d598192a 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ExpressionDimFilter.java @@ -47,7 +47,7 @@ public class ExpressionDimFilter implements DimFilter @JsonCreator public ExpressionDimFilter( @JsonProperty("expression") final String expression, - @Nullable @JsonProperty("filterTuning") final FilterTuning filterTuning, + @JsonProperty("filterTuning") @Nullable final FilterTuning filterTuning, @JacksonInject ExprMacroTable macroTable ) { diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java index cbb032a87fa7..ce10dcd43c23 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java @@ -102,8 +102,11 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory } /** - * Indicates whether this filter can return a bitmap index for filtering, based on - * the information provided by the input BitmapIndexSelector. + * Indicates whether this filter can return a bitmap index for filtering, based on the information provided by the + * input BitmapIndexSelector. + * + * Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)} will return a non-null + * {@link BitmapIndexSelector}. * * @param selector Object used to retrieve bitmap indexes * @@ -111,6 +114,26 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory */ boolean supportsBitmapIndex(BitmapIndexSelector selector); + /** + * Determine if a filter *should* use a bitmap index based on information collected from the supplied + * {@link BitmapIndexSelector}. This method differs from {@link #supportsBitmapIndex(BitmapIndexSelector)} in that + * the former only indicates if a bitmap index is available and {@link #getBitmapIndex(BitmapIndexSelector)} may be + * used. + * + * If shouldUseFilter(selector) returns true, {@link #supportsBitmapIndex} must also return true when called with the + * same selector object. Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)} + * will return a non-null {@link BitmapIndexSelector}. + * + * Implementations of this methods typically consider a {@link FilterTuning} to make decisions about when to + * use an available index. A "standard" implementation of this is available to all {@link Filter} implementations in + * {@link org.apache.druid.segment.filter.Filters#shouldUseIndex(Filter, BitmapIndexSelector, FilterTuning)}. + * + * @param selector Object used to retrieve bitmap indexes and provide information about the column + * + * @return true if this Filter should provide a bitmap index using the selector, false otherwise. + */ + boolean shouldUseBitmapIndex(BitmapIndexSelector selector); + /** * Indicates whether this filter supports selectivity estimation. * A filter supports selectivity estimation if it supports bitmap index and @@ -135,14 +158,4 @@ default boolean canVectorizeMatcher() * Set of columns used by a filter */ Set getRequiredColumns(); - - /** - * Determine if a filter *should* use a bitmap index based on information collected from the supplied - * {@link BitmapIndexSelector}. This method differs from {@link #supportsBitmapIndex(BitmapIndexSelector)} in that - * the former only indicates if a bitmap index is available and {@link #getBitmapIndex(BitmapIndexSelector)} may be - * used. Implementations of this methods typically consider a {@link FilterTuning} to make decisions about when to - * use an available index. A "standard" implementation of this is available to all {@link Filter} implementations in - * {@link org.apache.druid.segment.filter.Filters#shouldUseIndex(Filter, BitmapIndexSelector, FilterTuning)}. - */ - boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java index 57632c6a1ef6..586c5f1f7fd3 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java @@ -52,40 +52,40 @@ public static FilterTuning createDefault(Filter filter, BitmapIndexSelector sele return new FilterTuning(filter.supportsBitmapIndex(selector), null, null); } - private final boolean useIndex; - private final int useIndexMinCardinalityThreshold; - private final int useIndexMaxCardinalityThreshold; + private final boolean useBitmapIndex; + private final int minCardinalityToUseBitmapIndex; + private final int maxCardinalityToUseBitmapIndex; @JsonCreator public FilterTuning( - @Nullable @JsonProperty("useIndex") Boolean useIndex, - @Nullable @JsonProperty("useIndexMinCardinalityThreshold") Integer useIndexMinCardinalityThreshold, - @Nullable @JsonProperty("useIndexMaximumCardinalityThreshold") Integer useIndexMaxCardinalityThreshold + @JsonProperty("useBitmapIndex") @Nullable Boolean useBitmapIndex, + @JsonProperty("minCardinalityToUseBitmapIndex") @Nullable Integer minCardinalityToUseBitmapIndex, + @JsonProperty("useIndexMaximumCardinalityThreshold") @Nullable Integer maxCardinalityToUseBitmapIndex ) { - this.useIndex = useIndex != null ? useIndex : true; - this.useIndexMinCardinalityThreshold = - useIndexMinCardinalityThreshold != null ? useIndexMinCardinalityThreshold : 0; - this.useIndexMaxCardinalityThreshold = - useIndexMaxCardinalityThreshold != null ? useIndexMaxCardinalityThreshold : Integer.MAX_VALUE; + this.useBitmapIndex = useBitmapIndex != null ? useBitmapIndex : true; + this.minCardinalityToUseBitmapIndex = + minCardinalityToUseBitmapIndex != null ? minCardinalityToUseBitmapIndex : 0; + this.maxCardinalityToUseBitmapIndex = + maxCardinalityToUseBitmapIndex != null ? maxCardinalityToUseBitmapIndex : Integer.MAX_VALUE; } @JsonProperty - public boolean getUseIndex() + public boolean getUseBitmapIndex() { - return useIndex; + return useBitmapIndex; } @JsonProperty - public int getUseIndexMinCardinalityThreshold() + public int getMinCardinalityToUseBitmapIndex() { - return useIndexMinCardinalityThreshold; + return minCardinalityToUseBitmapIndex; } @JsonProperty - public int getUseIndexMaxCardinalityThreshold() + public int getMaxCardinalityToUseBitmapIndex() { - return useIndexMaxCardinalityThreshold; + return maxCardinalityToUseBitmapIndex; } @Override @@ -98,24 +98,24 @@ public boolean equals(Object o) return false; } FilterTuning that = (FilterTuning) o; - return Objects.equals(useIndex, that.useIndex) && - Objects.equals(useIndexMinCardinalityThreshold, that.useIndexMinCardinalityThreshold) && - Objects.equals(useIndexMaxCardinalityThreshold, that.useIndexMaxCardinalityThreshold); + return Objects.equals(useBitmapIndex, that.useBitmapIndex) && + Objects.equals(minCardinalityToUseBitmapIndex, that.minCardinalityToUseBitmapIndex) && + Objects.equals(maxCardinalityToUseBitmapIndex, that.maxCardinalityToUseBitmapIndex); } @Override public int hashCode() { - return Objects.hash(useIndex, useIndexMinCardinalityThreshold, useIndexMaxCardinalityThreshold); + return Objects.hash(useBitmapIndex, minCardinalityToUseBitmapIndex, maxCardinalityToUseBitmapIndex); } @Override public String toString() { return "FilterTuning{" + - "useIndex=" + useIndex + - ", useIndexMinCardinalityThreshold=" + useIndexMinCardinalityThreshold + - ", useIndexMaxCardinalityThreshold=" + useIndexMaxCardinalityThreshold + + "useBitmapIndex=" + useBitmapIndex + + ", minCardinalityToUseBitmapIndex=" + minCardinalityToUseBitmapIndex + + ", maxCardinalityToUseBitmapIndex=" + maxCardinalityToUseBitmapIndex + '}'; } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java index f189291642cf..3eba4e2601a3 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/InDimFilter.java @@ -78,8 +78,8 @@ public class InDimFilter implements DimFilter public InDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("values") Collection values, - @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkNotNull(dimension, "dimension can not be null"); diff --git a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java index d900fdc0e26c..c1e7ec0d7584 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/IntervalDimFilter.java @@ -57,8 +57,8 @@ public class IntervalDimFilter implements DimFilter public IntervalDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("intervals") List intervals, - @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkNotNull(dimension, "dimension can not be null"); diff --git a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java index f82d12f53f57..ea16c28c63f6 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/JavaScriptDimFilter.java @@ -68,8 +68,8 @@ public class JavaScriptDimFilter implements DimFilter public JavaScriptDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("function") String function, - @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning, + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning, @JacksonInject JavaScriptConfig config ) { diff --git a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java index 92265a980c1c..34749b2d2cfa 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/LikeDimFilter.java @@ -62,9 +62,9 @@ public class LikeDimFilter implements DimFilter public LikeDimFilter( @JsonProperty("dimension") final String dimension, @JsonProperty("pattern") final String pattern, - @Nullable @JsonProperty("escape") final String escape, - @Nullable @JsonProperty("extractionFn") final ExtractionFn extractionFn, - @Nullable @JsonProperty("filterTuning") final FilterTuning filterTuning + @JsonProperty("escape") @Nullable final String escape, + @JsonProperty("extractionFn") @Nullable final ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable final FilterTuning filterTuning ) { this.dimension = Preconditions.checkNotNull(dimension, "dimension"); diff --git a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java index 3fccf65b3673..b696ad9ff9dd 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/RegexDimFilter.java @@ -53,8 +53,8 @@ public class RegexDimFilter implements DimFilter public RegexDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("pattern") String pattern, - @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); diff --git a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java index ae0986c7269e..757cc7955875 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SearchQueryDimFilter.java @@ -49,8 +49,8 @@ public class SearchQueryDimFilter implements DimFilter public SearchQueryDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("query") SearchQuerySpec query, - @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); diff --git a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java index ba02ff39a09f..ef351a5c41e0 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SelectorDimFilter.java @@ -66,8 +66,8 @@ public class SelectorDimFilter implements DimFilter public SelectorDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("value") String value, - @Nullable @JsonProperty("extractionFn") ExtractionFn extractionFn, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning + @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); diff --git a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java index 844c1f9b2caa..c7a124820f11 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/SpatialDimFilter.java @@ -48,7 +48,7 @@ public class SpatialDimFilter implements DimFilter public SpatialDimFilter( @JsonProperty("dimension") String dimension, @JsonProperty("bound") Bound bound, - @Nullable @JsonProperty("filterTuning") FilterTuning filterTuning + @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); diff --git a/processing/src/main/java/org/apache/druid/query/lookup/LookupExtractionFn.java b/processing/src/main/java/org/apache/druid/query/lookup/LookupExtractionFn.java index 9266cf2da621..fa918af1c755 100644 --- a/processing/src/main/java/org/apache/druid/query/lookup/LookupExtractionFn.java +++ b/processing/src/main/java/org/apache/druid/query/lookup/LookupExtractionFn.java @@ -40,9 +40,9 @@ public class LookupExtractionFn extends FunctionalExtraction public LookupExtractionFn( @JsonProperty("lookup") final LookupExtractor lookup, @JsonProperty("retainMissingValue") final boolean retainMissingValue, - @Nullable @JsonProperty("replaceMissingValueWith") final String replaceMissingValueWith, - @Nullable @JsonProperty("injective") final Boolean injective, - @Nullable @JsonProperty("optimize") final Boolean optimize + @JsonProperty("replaceMissingValueWith") @Nullable final String replaceMissingValueWith, + @JsonProperty("injective") @Nullable final Boolean injective, + @JsonProperty("optimize") @Nullable final Boolean optimize ) { super( diff --git a/processing/src/main/java/org/apache/druid/query/lookup/RegisteredLookupExtractionFn.java b/processing/src/main/java/org/apache/druid/query/lookup/RegisteredLookupExtractionFn.java index 9ffe0fbc602a..d9b0b676e79b 100644 --- a/processing/src/main/java/org/apache/druid/query/lookup/RegisteredLookupExtractionFn.java +++ b/processing/src/main/java/org/apache/druid/query/lookup/RegisteredLookupExtractionFn.java @@ -47,8 +47,8 @@ public RegisteredLookupExtractionFn( @JacksonInject LookupExtractorFactoryContainerProvider manager, @JsonProperty("lookup") String lookup, @JsonProperty("retainMissingValue") final boolean retainMissingValue, - @Nullable @JsonProperty("replaceMissingValueWith") final String replaceMissingValueWith, - @Nullable @JsonProperty("injective") final Boolean injective, + @JsonProperty("replaceMissingValueWith") @Nullable final String replaceMissingValueWith, + @JsonProperty("injective") @Nullable final Boolean injective, @JsonProperty("optimize") Boolean optimize ) { diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanResultValue.java b/processing/src/main/java/org/apache/druid/query/scan/ScanResultValue.java index 8673b3479b03..aa368102dd4f 100644 --- a/processing/src/main/java/org/apache/druid/query/scan/ScanResultValue.java +++ b/processing/src/main/java/org/apache/druid/query/scan/ScanResultValue.java @@ -48,7 +48,7 @@ public class ScanResultValue implements Comparable @JsonCreator public ScanResultValue( - @Nullable @JsonProperty("segmentId") String segmentId, + @JsonProperty("segmentId") @Nullable String segmentId, @JsonProperty("columns") List columns, @JsonProperty("events") Object events ) diff --git a/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java b/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java index 7a6acb2267d1..a6c50d5f6396 100644 --- a/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java +++ b/processing/src/main/java/org/apache/druid/segment/FilteredOffset.java @@ -56,7 +56,7 @@ public final class FilteredOffset extends Offset rowOffsetMatcherFactory ); } else { - if (postFilter.shouldUseIndex(bitmapIndexSelector)) { + if (postFilter.shouldUseBitmapIndex(bitmapIndexSelector)) { filterMatcher = rowOffsetMatcherFactory.makeRowOffsetMatcher( postFilter.getBitmapIndex(bitmapIndexSelector) ); diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java index 58f215550785..87faef84d554 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java @@ -213,7 +213,7 @@ public boolean canVectorize( { if (filter != null) { final boolean filterCanVectorize = - filter.supportsBitmapIndex(makeBitmapIndexSelector(virtualColumns)) + filter.shouldUseBitmapIndex(makeBitmapIndexSelector(virtualColumns)) || filter.canVectorizeMatcher(); if (!filterCanVectorize) { @@ -361,7 +361,7 @@ public ColumnSelectorBitmapIndexSelector makeBitmapIndexSelector(final VirtualCo @VisibleForTesting public FilterAnalysis analyzeFilter( @Nullable final Filter filter, - ColumnSelectorBitmapIndexSelector bitmapIndexSelector, + ColumnSelectorBitmapIndexSelector indexSelector, @Nullable QueryMetrics queryMetrics ) { @@ -393,7 +393,7 @@ public FilterAnalysis analyzeFilter( // If we get an AndFilter, we can split the subfilters across both filtering stages for (Filter subfilter : ((AndFilter) filter).getFilters()) { - if (subfilter.shouldUseIndex(bitmapIndexSelector)) { + if (subfilter.supportsBitmapIndex(indexSelector) && subfilter.shouldUseBitmapIndex(indexSelector)) { preFilters.add(subfilter); } else { @@ -402,7 +402,7 @@ public FilterAnalysis analyzeFilter( } } else { // If we get an OrFilter or a single filter, handle the filter in one stage - if (filter.shouldUseIndex(bitmapIndexSelector)) { + if (filter.supportsBitmapIndex(indexSelector) && filter.shouldUseBitmapIndex(indexSelector)) { preFilters.add(filter); } else { postFilters.add(filter); @@ -416,15 +416,15 @@ public FilterAnalysis analyzeFilter( } else { if (queryMetrics != null) { BitmapResultFactory bitmapResultFactory = - queryMetrics.makeBitmapResultFactory(bitmapIndexSelector.getBitmapFactory()); + queryMetrics.makeBitmapResultFactory(indexSelector.getBitmapFactory()); long bitmapConstructionStartNs = System.nanoTime(); // Use AndFilter.getBitmapResult to intersect the preFilters to get its short-circuiting behavior. - preFilterBitmap = AndFilter.getBitmapIndex(bitmapIndexSelector, bitmapResultFactory, preFilters); + preFilterBitmap = AndFilter.getBitmapIndex(indexSelector, bitmapResultFactory, preFilters); preFilteredRows = preFilterBitmap.size(); queryMetrics.reportBitmapConstructionTime(System.nanoTime() - bitmapConstructionStartNs); } else { - BitmapResultFactory bitmapResultFactory = new DefaultBitmapResultFactory(bitmapIndexSelector.getBitmapFactory()); - preFilterBitmap = AndFilter.getBitmapIndex(bitmapIndexSelector, bitmapResultFactory, preFilters); + BitmapResultFactory bitmapResultFactory = new DefaultBitmapResultFactory(indexSelector.getBitmapFactory()); + preFilterBitmap = AndFilter.getBitmapIndex(indexSelector, bitmapResultFactory, preFilters); } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java index 1cbdc8fb5d37..2192340f7fc3 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java @@ -34,7 +34,6 @@ import org.apache.druid.query.filter.vector.ReadableVectorMatch; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; @@ -125,17 +124,6 @@ public boolean canVectorizeMatcher() return filters.stream().allMatch(Filter::canVectorizeMatcher); } - @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) - { - for (Filter filter : filters) { - if (!filter.shouldUseIndex(bitmapIndexSelector)) { - return false; - } - } - return true; - } - @Override public ValueMatcher makeMatcher( BitmapIndexSelector selector, @@ -170,31 +158,6 @@ public List getFilters() return filters; } - @Override - public boolean supportsBitmapIndex(BitmapIndexSelector selector) - { - for (Filter filter : filters) { - if (!filter.supportsBitmapIndex(selector)) { - return false; - } - } - return true; - } - - @Override - public boolean supportsSelectivityEstimation( - final ColumnSelector columnSelector, - final BitmapIndexSelector indexSelector - ) - { - for (Filter filter : filters) { - if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) { - return false; - } - } - return true; - } - @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java index 1b666f1fc8e1..5728f2b194a9 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java @@ -153,23 +153,22 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) { return selector.getBitmapIndex(boundDimFilter.getDimension()) != null; } - @Override - public Set getRequiredColumns() + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return boundDimFilter.getRequiredColumns(); + return Filters.shouldUseIndex(this, selector, filterTuning); } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { - return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); + return Filters.supportsSelectivityEstimation(this, boundDimFilter.getDimension(), columnSelector, indexSelector); } @Override - public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) + public Set getRequiredColumns() { - return Filters.supportsSelectivityEstimation(this, boundDimFilter.getDimension(), columnSelector, indexSelector); + return boundDimFilter.getRequiredColumns(); } private static Pair getStartEndIndexes( diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java index 135f517ba199..6ce997a07f1b 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java @@ -148,9 +148,9 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); + return Filters.shouldUseIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java index 1544038a63b3..ac2bb4b0c2d9 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java @@ -149,9 +149,9 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); + return Filters.shouldUseIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java index b3e4a48ad110..d135df419002 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java @@ -112,9 +112,9 @@ public boolean supportsBitmapIndex(final BitmapIndexSelector selector) } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); + return Filters.shouldUseIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java index ce5927c3892c..b1c355809b94 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java @@ -654,7 +654,7 @@ private static void generateAllCombinations( } /** - * This method provides a "standard" implementation of {@link Filter#shouldUseIndex(BitmapIndexSelector)} which takes + * This method provides a "standard" implementation of {@link Filter#shouldUseBitmapIndex(BitmapIndexSelector)} which takes * a {@link Filter}, a {@link BitmapIndexSelector}, and {@link FilterTuning} to determine if: * a) the filter supports bitmap indexes * b) the filter tuning specifies that it should use the index @@ -670,11 +670,11 @@ public static boolean shouldUseIndex( ) { final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector); - if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseIndex()) { + if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseBitmapIndex()) { return filter.getRequiredColumns().stream().allMatch(column -> { final int cardinality = indexSelector.getBitmapIndex(column).getCardinality(); - return cardinality >= tuning.getUseIndexMinCardinalityThreshold() - && cardinality <= tuning.getUseIndexMaxCardinalityThreshold(); + return cardinality >= tuning.getMinCardinalityToUseBitmapIndex() + && cardinality <= tuning.getMaxCardinalityToUseBitmapIndex(); }); } return false; diff --git a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java index ffd891878d73..a38dcff23ef0 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java @@ -180,9 +180,9 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); + return Filters.shouldUseIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java index 170002c88abf..4b1721d7ca1f 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java @@ -100,20 +100,20 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector); + return Filters.shouldUseIndex(this, selector, filterTuning); } @Override - public Set getRequiredColumns() + public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { - return ImmutableSet.of(dimension); + return Filters.supportsSelectivityEstimation(this, dimension, columnSelector, indexSelector); } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public Set getRequiredColumns() { - return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); + return ImmutableSet.of(dimension); } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java index ab8f9f082440..49e43592d111 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java @@ -114,9 +114,9 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); + return Filters.shouldUseIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java index 7991aa039c82..3a39e2f1370c 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java @@ -118,9 +118,9 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return baseFilter.shouldUseIndex(bitmapIndexSelector); + return baseFilter.shouldUseBitmapIndex(selector); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java index 817f2e0cd00b..1c54de1b3471 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java @@ -34,7 +34,6 @@ import org.apache.druid.query.filter.vector.VectorMatch; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.ColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; @@ -99,17 +98,6 @@ public boolean canVectorizeMatcher() return filters.stream().allMatch(Filter::canVectorizeMatcher); } - @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) - { - for (Filter f : filters) { - if (!f.shouldUseIndex(bitmapIndexSelector)) { - return false; - } - } - return true; - } - @Override public ValueMatcher makeMatcher( BitmapIndexSelector selector, @@ -144,28 +132,6 @@ public List getFilters() return filters; } - @Override - public boolean supportsBitmapIndex(BitmapIndexSelector selector) - { - for (Filter filter : filters) { - if (!filter.supportsBitmapIndex(selector)) { - return false; - } - } - return true; - } - - @Override - public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) - { - for (Filter filter : filters) { - if (!filter.supportsSelectivityEstimation(columnSelector, indexSelector)) { - return false; - } - } - return true; - } - @Override public double estimateSelectivity(BitmapIndexSelector indexSelector) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java index d02cc558493a..9ae417f5c781 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java @@ -42,6 +42,7 @@ public class SelectorFilter implements Filter { private final String dimension; private final String value; + @Nullable private final FilterTuning filterTuning; public SelectorFilter( @@ -92,9 +93,9 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); + return Filters.shouldUseIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java index a4ca13f052b3..f93024b61763 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java @@ -116,21 +116,21 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return false; + return Filters.shouldUseIndex(this, selector, filterTuning); } @Override - public Set getRequiredColumns() + public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { - return ImmutableSet.of(dimension); + return false; } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public Set getRequiredColumns() { - return Filters.shouldUseIndex(this, bitmapIndexSelector, filterTuning); + return ImmutableSet.of(dimension); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java index ccd4c101d4be..2ae0edfb19e6 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java @@ -56,7 +56,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { return true; } diff --git a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 7f0bc3da3bef..a0a733ec6f62 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -98,7 +98,7 @@ public byte asByte() @JsonCreator public static DictionaryEncodedColumnPartSerde createDeserializer( - @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory, + @JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory, @NotNull @JsonProperty("byteOrder") ByteOrder byteOrder ) { diff --git a/processing/src/main/java/org/apache/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java b/processing/src/main/java/org/apache/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java index 2262184d9f51..e2a5cd213528 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java @@ -42,7 +42,7 @@ public class DoubleNumericColumnPartSerdeV2 implements ColumnPartSerde @JsonCreator public static DoubleNumericColumnPartSerdeV2 getDoubleGenericColumnPartSerde( @JsonProperty("byteOrder") ByteOrder byteOrder, - @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + @JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory ) { return new DoubleNumericColumnPartSerdeV2( diff --git a/processing/src/main/java/org/apache/druid/segment/serde/FloatNumericColumnPartSerdeV2.java b/processing/src/main/java/org/apache/druid/segment/serde/FloatNumericColumnPartSerdeV2.java index b3c71db94a55..77d900dbe2ab 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/FloatNumericColumnPartSerdeV2.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/FloatNumericColumnPartSerdeV2.java @@ -40,7 +40,7 @@ public class FloatNumericColumnPartSerdeV2 implements ColumnPartSerde @JsonCreator public static FloatNumericColumnPartSerdeV2 createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder, - @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + @JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory ) { return new FloatNumericColumnPartSerdeV2( diff --git a/processing/src/main/java/org/apache/druid/segment/serde/LongNumericColumnPartSerdeV2.java b/processing/src/main/java/org/apache/druid/segment/serde/LongNumericColumnPartSerdeV2.java index 7a46c51eb394..75344ec9340c 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/LongNumericColumnPartSerdeV2.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/LongNumericColumnPartSerdeV2.java @@ -40,7 +40,7 @@ public class LongNumericColumnPartSerdeV2 implements ColumnPartSerde @JsonCreator public static LongNumericColumnPartSerdeV2 createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder, - @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + @JsonProperty("bitmapSerdeFactory") @Nullable BitmapSerdeFactory bitmapSerdeFactory ) { return new LongNumericColumnPartSerdeV2( diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java index ed57e9067f58..db4863ad7952 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java @@ -81,6 +81,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -398,21 +399,21 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { return false; } @Override - public Set getRequiredColumns() + public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { - return null; + return false; } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public Set getRequiredColumns() { - return false; + return Collections.emptySet(); } @Override @@ -471,6 +472,12 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) return false; } + @Override + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) + { + return false; + } + @Override public VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory) { @@ -489,12 +496,6 @@ public Set getRequiredColumns() return null; } - @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) - { - return false; - } - @Override public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index d55651ff3f31..2015690dde87 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -695,21 +695,21 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) } @Override - public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) + public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { return true; } @Override - public Set getRequiredColumns() + public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) { - return null; + return true; } @Override - public boolean shouldUseIndex(BitmapIndexSelector bitmapIndexSelector) + public Set getRequiredColumns() { - return true; + return Collections.emptySet(); } private class DictionaryRaceTestFilterDruidPredicateFactory implements DruidPredicateFactory diff --git a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQuery.java b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQuery.java index 1a09c451e14d..e29cd60ed272 100644 --- a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQuery.java +++ b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQuery.java @@ -46,8 +46,8 @@ public class ClientCompactQuery implements ClientQuery @JsonCreator public ClientCompactQuery( @JsonProperty("dataSource") String dataSource, - @Nullable @JsonProperty("interval") final Interval interval, - @Nullable @JsonProperty("segments") final List segments, + @JsonProperty("interval") @Nullable final Interval interval, + @JsonProperty("segments") @Nullable final List segments, @JsonProperty("targetCompactionSizeBytes") @Nullable Long targetCompactionSizeBytes, @JsonProperty("tuningConfig") ClientCompactQueryTuningConfig tuningConfig, @JsonProperty("context") Map context diff --git a/server/src/main/java/org/apache/druid/client/indexing/IndexingWorkerInfo.java b/server/src/main/java/org/apache/druid/client/indexing/IndexingWorkerInfo.java index b60fbc4218a1..8aa43439dd15 100644 --- a/server/src/main/java/org/apache/druid/client/indexing/IndexingWorkerInfo.java +++ b/server/src/main/java/org/apache/druid/client/indexing/IndexingWorkerInfo.java @@ -46,7 +46,7 @@ public IndexingWorkerInfo( @JsonProperty("availabilityGroups") Set availabilityGroups, @JsonProperty("runningTasks") Collection runningTasks, @JsonProperty("lastCompletedTaskTime") DateTime lastCompletedTaskTime, - @Nullable @JsonProperty("blacklistedUntil") DateTime blacklistedUntil + @JsonProperty("blacklistedUntil") @Nullable DateTime blacklistedUntil ) { this.worker = worker; diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java index 3a904b9e0f74..e11b017e5dca 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java @@ -59,9 +59,9 @@ public NoopSupervisorSpec( @JsonCreator public NoopSupervisorSpec( - @Nullable @JsonProperty("id") String id, - @Nullable @JsonProperty("dataSources") List datasources, - @Nullable @JsonProperty("suspended") Boolean suspended + @JsonProperty("id") @Nullable String id, + @JsonProperty("dataSources") @Nullable List datasources, + @JsonProperty("suspended") @Nullable Boolean suspended ) { this.id = id; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java index e06227b87d62..b41833d7b332 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/Bounds.java @@ -131,7 +131,7 @@ public static BoundDimFilter equalTo(final BoundRefKey boundRefKey, final String null, boundRefKey.getExtractionFn(), boundRefKey.getComparator(), - null + null ); } From cdb63a7eda6e6bfab6f89b52de5559bac54530cc Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 7 Aug 2019 15:32:55 -0700 Subject: [PATCH 20/24] javadocs, precondition --- .../java/org/apache/druid/query/filter/Filter.java | 10 +++++++--- .../java/org/apache/druid/segment/filter/Filters.java | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java index ce10dcd43c23..9bdfa81e190c 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java @@ -106,7 +106,8 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory * input BitmapIndexSelector. * * Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)} will return a non-null - * {@link BitmapIndexSelector}. + * {@link BitmapIndexSelector}, and also that all columns specified in {@link #getRequiredColumns()} have a bitmap + * index retrievable via {@link BitmapIndexSelector#getBitmapIndex(String)}. * * @param selector Object used to retrieve bitmap indexes * @@ -122,7 +123,9 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory * * If shouldUseFilter(selector) returns true, {@link #supportsBitmapIndex} must also return true when called with the * same selector object. Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)} - * will return a non-null {@link BitmapIndexSelector}. + * will return a non-null {@link BitmapIndexSelector}, and also that all columns specified in + * {@link #getRequiredColumns()} have a bitmap index retrievable via + * {@link BitmapIndexSelector#getBitmapIndex(String)}. * * Implementations of this methods typically consider a {@link FilterTuning} to make decisions about when to * use an available index. A "standard" implementation of this is available to all {@link Filter} implementations in @@ -155,7 +158,8 @@ default boolean canVectorizeMatcher() } /** - * Set of columns used by a filter + * Set of columns used by a filter. If {@link #supportsBitmapIndex} returns true, all columns returned by this method + * can be expected to have a bitmap index retrievable via {@link BitmapIndexSelector#getBitmapIndex(String)} */ Set getRequiredColumns(); } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java index b1c355809b94..8f48c117f5d2 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java @@ -656,7 +656,7 @@ private static void generateAllCombinations( /** * This method provides a "standard" implementation of {@link Filter#shouldUseBitmapIndex(BitmapIndexSelector)} which takes * a {@link Filter}, a {@link BitmapIndexSelector}, and {@link FilterTuning} to determine if: - * a) the filter supports bitmap indexes + * a) the filter supports bitmap indexes for all required columns * b) the filter tuning specifies that it should use the index * c) the cardinality of the column is above the minimum threshold and below the maximum threshold to use the index * @@ -672,7 +672,9 @@ public static boolean shouldUseIndex( final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector); if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseBitmapIndex()) { return filter.getRequiredColumns().stream().allMatch(column -> { - final int cardinality = indexSelector.getBitmapIndex(column).getCardinality(); + final BitmapIndex index = indexSelector.getBitmapIndex(column); + Preconditions.checkNotNull(index, "WTF?! column doesn't have a bitmap index"); + final int cardinality = index.getCardinality(); return cardinality >= tuning.getMinCardinalityToUseBitmapIndex() && cardinality <= tuning.getMaxCardinalityToUseBitmapIndex(); }); From 35098b02f9e377655ec8da7f5a99a9d49596a782 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 7 Aug 2019 15:33:55 -0700 Subject: [PATCH 21/24] nullable --- .../java/org/apache/druid/query/filter/BitmapIndexSelector.java | 1 + 1 file changed, 1 insertion(+) diff --git a/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java b/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java index b19327f9d17d..90307eb6380a 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java +++ b/processing/src/main/java/org/apache/druid/query/filter/BitmapIndexSelector.java @@ -38,6 +38,7 @@ public interface BitmapIndexSelector boolean hasMultipleValues(String dimension); int getNumRows(); BitmapFactory getBitmapFactory(); + @Nullable BitmapIndex getBitmapIndex(String dimension); @Nullable ImmutableBitmap getBitmapIndex(String dimension, String value); From 9e6a2e0c32a1bb8de4bae689588867f86c3fa763 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 7 Aug 2019 15:39:34 -0700 Subject: [PATCH 22/24] rename method to be consistent --- .../src/main/java/org/apache/druid/query/filter/Filter.java | 2 +- .../main/java/org/apache/druid/segment/filter/BoundFilter.java | 2 +- .../org/apache/druid/segment/filter/ColumnComparisonFilter.java | 2 +- .../apache/druid/segment/filter/DimensionPredicateFilter.java | 2 +- .../java/org/apache/druid/segment/filter/ExpressionFilter.java | 2 +- .../src/main/java/org/apache/druid/segment/filter/Filters.java | 2 +- .../src/main/java/org/apache/druid/segment/filter/InFilter.java | 2 +- .../java/org/apache/druid/segment/filter/JavaScriptFilter.java | 2 +- .../main/java/org/apache/druid/segment/filter/LikeFilter.java | 2 +- .../java/org/apache/druid/segment/filter/SelectorFilter.java | 2 +- .../java/org/apache/druid/segment/filter/SpatialFilter.java | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java index 9bdfa81e190c..27d64b86285a 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java @@ -129,7 +129,7 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory * * Implementations of this methods typically consider a {@link FilterTuning} to make decisions about when to * use an available index. A "standard" implementation of this is available to all {@link Filter} implementations in - * {@link org.apache.druid.segment.filter.Filters#shouldUseIndex(Filter, BitmapIndexSelector, FilterTuning)}. + * {@link org.apache.druid.segment.filter.Filters#shouldUseBitmapIndex(Filter, BitmapIndexSelector, FilterTuning)}. * * @param selector Object used to retrieve bitmap indexes and provide information about the column * diff --git a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java index 5728f2b194a9..f0f8a6b08ad8 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java @@ -156,7 +156,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, selector, filterTuning); + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java index 6ce997a07f1b..00051391f13c 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java @@ -150,7 +150,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, selector, filterTuning); + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java index ac2bb4b0c2d9..6029db54e545 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/DimensionPredicateFilter.java @@ -151,7 +151,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, selector, filterTuning); + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java index d135df419002..a8dd18fef434 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java @@ -114,7 +114,7 @@ public boolean supportsBitmapIndex(final BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, selector, filterTuning); + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java index 8f48c117f5d2..f3e73479dad6 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java @@ -663,7 +663,7 @@ private static void generateAllCombinations( * If all these things are true, {@link org.apache.druid.segment.QueryableIndexStorageAdapter} will utilize the * indexes. */ - public static boolean shouldUseIndex( + public static boolean shouldUseBitmapIndex( Filter filter, BitmapIndexSelector indexSelector, @Nullable FilterTuning filterTuning diff --git a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java index a38dcff23ef0..4594126b37a2 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/InFilter.java @@ -182,7 +182,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, selector, filterTuning); + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java index 4b1721d7ca1f..952f269f4724 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/JavaScriptFilter.java @@ -102,7 +102,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, selector, filterTuning); + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java index 49e43592d111..6d148e41c388 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/LikeFilter.java @@ -116,7 +116,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, selector, filterTuning); + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java index 9ae417f5c781..641e768faeff 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SelectorFilter.java @@ -95,7 +95,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, selector, filterTuning); + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java index f93024b61763..e07c56d20d66 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java @@ -118,7 +118,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseIndex(this, selector, filterTuning); + return Filters.shouldUseBitmapIndex(this, selector, filterTuning); } @Override From 8fc735ac8b1c50c26b8403c4ec21bdb536f265be Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 8 Aug 2019 12:52:17 -0700 Subject: [PATCH 23/24] review comments --- .../src/main/java/org/apache/druid/query/filter/Filter.java | 2 +- .../org/apache/druid/segment/filter/ColumnComparisonFilter.java | 2 +- .../src/main/java/org/apache/druid/segment/filter/Filters.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java index 27d64b86285a..27a504709dd2 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java @@ -103,7 +103,7 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory /** * Indicates whether this filter can return a bitmap index for filtering, based on the information provided by the - * input BitmapIndexSelector. + * input {@link BitmapIndexSelector}. * * Returning a value of true here guarantees that {@link #getBitmapIndex(BitmapIndexSelector)} will return a non-null * {@link BitmapIndexSelector}, and also that all columns specified in {@link #getRequiredColumns()} have a bitmap diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java index 00051391f13c..892b0bf6f670 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java @@ -150,7 +150,7 @@ public boolean supportsBitmapIndex(BitmapIndexSelector selector) @Override public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) { - return Filters.shouldUseBitmapIndex(this, selector, filterTuning); + return false; } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java index f3e73479dad6..4d3ebd952be5 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java @@ -673,7 +673,7 @@ public static boolean shouldUseBitmapIndex( if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseBitmapIndex()) { return filter.getRequiredColumns().stream().allMatch(column -> { final BitmapIndex index = indexSelector.getBitmapIndex(column); - Preconditions.checkNotNull(index, "WTF?! column doesn't have a bitmap index"); + Preconditions.checkNotNull(index, "Column does not have a bitmap index"); final int cardinality = index.getCardinality(); return cardinality >= tuning.getMinCardinalityToUseBitmapIndex() && cardinality <= tuning.getMaxCardinalityToUseBitmapIndex(); From 505b66865440bdad8bbef200af9352d8fe649d1d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 8 Aug 2019 15:22:06 -0700 Subject: [PATCH 24/24] remove tuning from ColumnComparisonFilter/ColumnComparisonDimFilter --- .../filter/ColumnComparisonDimFilter.java | 30 +++---------------- .../filter/ColumnComparisonFilter.java | 6 +--- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java index 8d7edec15ed7..dd97ce2808cf 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/ColumnComparisonDimFilter.java @@ -20,9 +20,7 @@ package org.apache.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.RangeSet; @@ -30,7 +28,6 @@ import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.filter.ColumnComparisonFilter; -import javax.annotation.Nullable; import java.util.List; import java.util.Objects; import java.util.Set; @@ -43,24 +40,14 @@ public class ColumnComparisonDimFilter implements DimFilter private static final Joiner COMMA_JOINER = Joiner.on(", "); private final List dimensions; - @Nullable - private final FilterTuning filterTuning; @JsonCreator public ColumnComparisonDimFilter( - @JsonProperty("dimensions") List dimensions, - @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning + @JsonProperty("dimensions") List dimensions ) { this.dimensions = Preconditions.checkNotNull(dimensions, "dimensions"); Preconditions.checkArgument(dimensions.size() >= 2, "dimensions must have a least 2 dimensions"); - this.filterTuning = filterTuning; - } - - @VisibleForTesting - public ColumnComparisonDimFilter(List dimensions) - { - this(dimensions, null); } @Override @@ -81,7 +68,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - return new ColumnComparisonFilter(dimensions, filterTuning); + return new ColumnComparisonFilter(dimensions); } @JsonProperty @@ -90,19 +77,11 @@ public List getDimensions() return dimensions; } - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonProperty - public FilterTuning getFilterTuning() - { - return filterTuning; - } - @Override public String toString() { return "ColumnComparisonDimFilter{" + "dimensions=[" + COMMA_JOINER.join(dimensions) + "]" + - ", filterTuning=" + filterTuning + "}"; } @@ -116,14 +95,13 @@ public boolean equals(Object o) return false; } ColumnComparisonDimFilter that = (ColumnComparisonDimFilter) o; - return dimensions.equals(that.dimensions) && - Objects.equals(filterTuning, that.filterTuning); + return dimensions.equals(that.dimensions); } @Override public int hashCode() { - return Objects.hash(dimensions, filterTuning); + return Objects.hash(dimensions); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java index 892b0bf6f670..5bcb350a9b59 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java @@ -25,7 +25,6 @@ import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.Filter; -import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.ValueGetter; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.ValueMatcherColumnSelectorStrategy; @@ -46,15 +45,12 @@ public class ColumnComparisonFilter implements Filter { private final List dimensions; - private final FilterTuning filterTuning; public ColumnComparisonFilter( - final List dimensions, - final FilterTuning filterTuning + final List dimensions ) { this.dimensions = Preconditions.checkNotNull(dimensions, "dimensions"); - this.filterTuning = filterTuning; } @Override