From f6989eb3e1ac71e37c5a3c7e5a298e760dfb26a1 Mon Sep 17 00:00:00 2001 From: "navis.ryu" Date: Sun, 17 Jul 2016 14:16:55 +0900 Subject: [PATCH] Support metric filters --- .../main/java/io/druid/math/expr/Evals.java | 67 +++++++++ .../query/aggregation/AggregatorUtil.java | 29 +++- .../FilteredAggregatorFactory.java | 16 +++ .../java/io/druid/query/filter/DimFilter.java | 5 +- .../io/druid/query/filter/DimFilterUtils.java | 2 + .../druid/query/filter/ExpressionFilter.java | 136 ++++++++++++++++++ .../query/filter/ValueMatcherFactory.java | 10 ++ .../groupby/RowBasedValueMatcherFactory.java | 38 +++++ .../epinephelinae/RowBasedGrouperHelper.java | 2 +- .../druid/query/search/SearchQueryRunner.java | 11 +- .../druid/segment/ColumnSelectorFactory.java | 2 +- .../io/druid/segment/DimensionIndexer.java | 3 + .../segment/QueryableIndexStorageAdapter.java | 18 ++- .../druid/segment/StringDimensionIndexer.java | 15 ++ .../io/druid/segment/column/ValueType.java | 38 ++++- .../segment/incremental/IncrementalIndex.java | 7 +- .../IncrementalIndexStorageAdapter.java | 77 +++++++++- .../incremental/OnheapIncrementalIndex.java | 4 +- .../aggregation/FilteredAggregatorTest.java | 2 +- .../aggregation/JavaScriptAggregatorTest.java | 2 +- .../query/groupby/GroupByQueryRunnerTest.java | 11 ++ .../TestColumnSelectorFactory.java | 2 +- .../query/search/SearchQueryRunnerTest.java | 12 ++ .../query/select/SelectQueryRunnerTest.java | 106 +++++++++++--- 24 files changed, 568 insertions(+), 47 deletions(-) create mode 100644 processing/src/main/java/io/druid/query/filter/ExpressionFilter.java diff --git a/common/src/main/java/io/druid/math/expr/Evals.java b/common/src/main/java/io/druid/math/expr/Evals.java index 702037309a91..1e1067468b47 100644 --- a/common/src/main/java/io/druid/math/expr/Evals.java +++ b/common/src/main/java/io/druid/math/expr/Evals.java @@ -20,7 +20,10 @@ package io.druid.math.expr; import com.google.common.base.Strings; +import com.google.common.base.Function; +import com.google.common.primitives.Longs; import io.druid.common.guava.GuavaUtils; +import io.druid.data.input.impl.DimensionSchema; import io.druid.java.util.common.logger.Logger; import java.util.Arrays; @@ -112,4 +115,68 @@ public static boolean asBoolean(String x) { return !Strings.isNullOrEmpty(x) && Boolean.valueOf(x); } + + public static boolean asBoolean(Number x) + { + if (x == null) { + return false; + } else if (x instanceof Integer) { + return x.intValue() > 0; + } else if (x instanceof Long) { + return x.longValue() > 0; + } else if (x instanceof Float) { + return x.floatValue() > 0; + } + return x.doubleValue() > 0; + } + + public static Function asNumberFunc(DimensionSchema.ValueType type) + { + switch (type) { + case FLOAT: + return new Function() + { + @Override + public Number apply(Comparable input) + { + return input == null ? 0F : (Float) input; + } + }; + case LONG: + return new Function() + { + @Override + public Number apply(Comparable input) + { + return input == null ? 0L : (Long) input; + } + }; + case STRING: + return new Function() + { + @Override + public Number apply(Comparable input) + { + return toNumeric(input); + } + }; + } + throw new UnsupportedOperationException("Unsupported type " + type); + } + + private static Number toNumeric(Object value) + { + if (value == null || value instanceof Number) { + return (Number) value; + } + final String stringVal = String.valueOf(value).trim(); + if (stringVal.isEmpty()) { + return null; + } + Long longValue = Longs.tryParse(stringVal); + if (longValue != null) { + return longValue; + } + return Double.valueOf(stringVal); + } } diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index e106ef828c59..686cc378268a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation; +import com.google.common.base.Supplier; import com.google.common.collect.Lists; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.FloatColumnSelector; @@ -100,7 +101,7 @@ public static FloatColumnSelector getFloatColumnSelector( return metricFactory.makeFloatColumnSelector(fieldName); } if (fieldName == null && fieldExpression != null) { - final NumericColumnSelector numeric = metricFactory.makeMathExpressionSelector(fieldExpression); + final NumericColumnSelector numeric = metricFactory.makeExpressionSelector(fieldExpression); return new FloatColumnSelector() { @Override @@ -125,7 +126,7 @@ public static LongColumnSelector getLongColumnSelector( return metricFactory.makeLongColumnSelector(fieldName); } if (fieldName == null && fieldExpression != null) { - final NumericColumnSelector numeric = metricFactory.makeMathExpressionSelector(fieldExpression); + final NumericColumnSelector numeric = metricFactory.makeExpressionSelector(fieldExpression); return new LongColumnSelector() { @Override @@ -138,4 +139,28 @@ public long get() } throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } + + public static Supplier asSupplier(final FloatColumnSelector selector) + { + return new Supplier() + { + @Override + public Number get() + { + return selector.get(); + } + }; + } + + public static Supplier asSupplier(final LongColumnSelector selector) + { + return new Supplier() + { + @Override + public Number get() + { + return selector.get(); + } + }; + } } diff --git a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java index 9e830f71df8d..5b84aa39fcff 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java @@ -23,6 +23,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Strings; +import io.druid.math.expr.Evals; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.query.filter.DruidLongPredicate; @@ -34,6 +35,7 @@ import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; +import io.druid.segment.NumericColumnSelector; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.BooleanValueMatcher; import io.druid.segment.filter.Filters; @@ -391,5 +393,19 @@ private ValueType getTypeForDimension(String dimension) ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(dimension); return capabilities == null ? ValueType.STRING : capabilities.getType(); } + + @Override + public ValueMatcher makeExpressionMatcher(String expression) + { + final NumericColumnSelector selector = columnSelectorFactory.makeExpressionSelector(expression); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return Evals.asBoolean(selector.get()); + } + }; + } } } diff --git a/processing/src/main/java/io/druid/query/filter/DimFilter.java b/processing/src/main/java/io/druid/query/filter/DimFilter.java index 4de0c061fccf..f613e594870a 100644 --- a/processing/src/main/java/io/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilter.java @@ -25,7 +25,7 @@ /** */ -@JsonTypeInfo(use=JsonTypeInfo.Id.NAME, property="type") +@JsonTypeInfo(use=JsonTypeInfo.Id.NAME, property="type",defaultImpl = ExpressionFilter.class) @JsonSubTypes(value={ @JsonSubTypes.Type(name="and", value=AndDimFilter.class), @JsonSubTypes.Type(name="or", value=OrDimFilter.class), @@ -39,7 +39,8 @@ @JsonSubTypes.Type(name="in", value=InDimFilter.class), @JsonSubTypes.Type(name="bound", value=BoundDimFilter.class), @JsonSubTypes.Type(name="interval", value=IntervalDimFilter.class), - @JsonSubTypes.Type(name="like", value=LikeDimFilter.class) + @JsonSubTypes.Type(name="like", value=LikeDimFilter.class), + @JsonSubTypes.Type(name="expression", value=ExpressionFilter.class) }) public interface DimFilter { diff --git a/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java b/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java index 2ca189f87023..99c674ba0fa2 100644 --- a/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java @@ -50,6 +50,8 @@ public class DimFilterUtils static final byte BOUND_CACHE_ID = 0xA; static final byte INTERVAL_CACHE_ID = 0xB; static final byte LIKE_CACHE_ID = 0xC; + static final byte EXPR_CACHE_ID = 0xD; + public static final byte STRING_SEPARATOR = (byte) 0xFF; static byte[] computeCacheKey(byte cacheIdKey, List filters) diff --git a/processing/src/main/java/io/druid/query/filter/ExpressionFilter.java b/processing/src/main/java/io/druid/query/filter/ExpressionFilter.java new file mode 100644 index 000000000000..90f8472ea1f3 --- /dev/null +++ b/processing/src/main/java/io/druid/query/filter/ExpressionFilter.java @@ -0,0 +1,136 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.collect.RangeSet; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.common.utils.StringUtils; + +import java.nio.ByteBuffer; + +/** + */ +public class ExpressionFilter implements DimFilter +{ + private final String expression; + + @JsonCreator + public ExpressionFilter( + @JsonProperty("expression") String expression + ) + { + this.expression = Preconditions.checkNotNull(expression, "expression can not be null"); + } + + @JsonProperty + public String getExpression() + { + return expression; + } + + @Override + public byte[] getCacheKey() + { + byte[] expressionBytes = StringUtils.toUtf8(expression); + return ByteBuffer.allocate(1 + expressionBytes.length) + .put(DimFilterUtils.EXPR_CACHE_ID) + .put(expressionBytes) + .array(); + } + + @Override + public DimFilter optimize() + { + return this; + } + + @Override + public Filter toFilter() + { + return new Filter() + { + @Override + public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector) + { + throw new IllegalStateException("should not be called"); + } + + @Override + public ValueMatcher makeMatcher(ValueMatcherFactory factory) + { + return factory.makeExpressionMatcher(expression); + } + + @Override + public boolean supportsBitmapIndex(BitmapIndexSelector selector) + { + return false; + } + + @Override + public String toString() + { + return ExpressionFilter.this.toString(); + } + }; + } + + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + + @Override + public String toString() + { + return "ExpressionFilter{" + + "expression='" + expression + '\'' + + '}'; + } + + @Override + public int hashCode() + { + return expression.hashCode(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ExpressionFilter that = (ExpressionFilter) o; + + if (!expression.equals(that.expression)) { + return false; + } + + return true; + } +} diff --git a/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java b/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java index 625951253607..9d202facb084 100644 --- a/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java +++ b/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java @@ -60,4 +60,14 @@ public interface ValueMatcherFactory * @return An object that applies a predicate to row values */ public ValueMatcher makeValueMatcher(String dimension, DruidPredicateFactory predicateFactory); + + /** + * Create a ValueMatcher that applies expression to row values. + * + * The matcher returned does not use any type of index to evaluate result. + * + * @param expression + * @return + */ + public ValueMatcher makeExpressionMatcher(String expression); } diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedValueMatcherFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedValueMatcherFactory.java index 7511998dc94e..4ae6d8d6dc2f 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedValueMatcherFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedValueMatcherFactory.java @@ -21,8 +21,13 @@ import com.google.common.base.Predicate; import com.google.common.base.Strings; +import com.google.common.base.Supplier; +import com.google.common.collect.Maps; import io.druid.common.guava.GuavaUtils; import io.druid.data.input.Row; +import io.druid.math.expr.Evals; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; import io.druid.query.filter.DruidLongPredicate; import io.druid.query.filter.DruidPredicateFactory; import io.druid.query.filter.ValueMatcher; @@ -31,6 +36,7 @@ import io.druid.segment.filter.BooleanValueMatcher; import java.util.List; +import java.util.Map; import java.util.Objects; public class RowBasedValueMatcherFactory implements ValueMatcherFactory @@ -110,6 +116,38 @@ public boolean matches() } } + @Override + public ValueMatcher makeExpressionMatcher(String expression) + { + final Expr parsed = Parser.parse(expression); + + final List required = Parser.findRequiredBindings(parsed); + final Map> values = Maps.newHashMapWithExpectedSize(required.size()); + + for (final String columnName : required) { + values.put( + columnName, new Supplier() + { + @Override + public Number get() + { + return Evals.toNumber((row.getRaw(columnName))); + } + } + ); + } + final Expr.ObjectBinding binding = Parser.withSuppliers(values); + + return new ValueMatcher() + { + @Override + public boolean matches() + { + return parsed.eval(binding).asBoolean(); + } + }; + } + // Precondition: value must be run through Strings.emptyToNull private boolean doesMatch(final Object raw, final String value) { diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index 33f014662fdb..9cb4c58f1578 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -777,7 +777,7 @@ public Object get() } @Override - public NumericColumnSelector makeMathExpressionSelector(String expression) + public NumericColumnSelector makeExpressionSelector(String expression) { final Expr parsed = Parser.parse(expression); diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 9e6b7e85920d..ff51839e5a6d 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -103,7 +103,13 @@ public Sequence> run( // Closing this will cause segfaults in unit tests. final QueryableIndex index = segment.asQueryableIndex(); + FALLBACK_TO_CURSOR: if (index != null) { + final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); + final ColumnSelectorBitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(bitmapFactory, index); + if (filter != null && !filter.supportsBitmapIndex(selector)) { + break FALLBACK_TO_CURSOR; + } final TreeMap retVal = Maps.newTreeMap(query.getSort().getComparator()); Iterable dimsToSearch; @@ -113,10 +119,7 @@ public Sequence> run( dimsToSearch = dimensions; } - final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); - - final ImmutableBitmap baseFilter = - filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); + final ImmutableBitmap baseFilter = filter == null ? null : filter.getBitmapIndex(selector); ImmutableBitmap timeFilteredBitmap; if (!interval.contains(segment.getDataInterval())) { diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java index afb38476c048..a800603d59ae 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java @@ -31,6 +31,6 @@ public interface ColumnSelectorFactory public FloatColumnSelector makeFloatColumnSelector(String columnName); public LongColumnSelector makeLongColumnSelector(String columnName); public ObjectColumnSelector makeObjectColumnSelector(String columnName); - public NumericColumnSelector makeMathExpressionSelector(String expression); + public NumericColumnSelector makeExpressionSelector(String expression); public ColumnCapabilities getColumnCapabilities(String columnName); } diff --git a/processing/src/main/java/io/druid/segment/DimensionIndexer.java b/processing/src/main/java/io/druid/segment/DimensionIndexer.java index ed41d7145245..4d773b9ac28c 100644 --- a/processing/src/main/java/io/druid/segment/DimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DimensionIndexer.java @@ -275,6 +275,9 @@ public Object makeColumnValueSelector( */ public Object convertUnsortedEncodedArrayToActualArrayOrList(EncodedTypeArray key, boolean asList); + public ActualType convertUnsortedEncodedArrayToActualValue(EncodedTypeArray key, int index); + + public int getLengthOfUnsortedEncodedArray(EncodedTypeArray key); /** * Given a row value array from a TimeAndDims key, as described in the documentation for compareUnsortedEncodedArrays(), diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index 9e3a04a017cf..9b2ebdae227a 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -29,11 +29,11 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Closer; - import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.granularity.QueryGranularity; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; +import io.druid.math.expr.Evals; import io.druid.math.expr.Expr; import io.druid.math.expr.Parser; import io.druid.query.QueryInterruptedException; @@ -807,7 +807,7 @@ public Object get() } @Override - public NumericColumnSelector makeMathExpressionSelector(String expression) + public NumericColumnSelector makeExpressionSelector(String expression) { final Expr parsed = Parser.parse(expression); final List required = Parser.findRequiredBindings(parsed); @@ -1141,6 +1141,20 @@ private ValueMatcher makeLongValueMatcher(String dimension, final DruidLongPredi ); } + @Override + public ValueMatcher makeExpressionMatcher(String expression) + { + final NumericColumnSelector selector = cursor.makeExpressionSelector(expression); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return Evals.asBoolean(selector.get()); + } + }; + } + private ValueType getTypeForDimension(String dimension) { ColumnCapabilities capabilities = getColumnCapabilites(index, dimension); diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index c869c98dadc7..646cf714b630 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -510,6 +510,21 @@ public Object convertUnsortedEncodedArrayToActualArrayOrList(int[] key, boolean } } + @Override + public String convertUnsortedEncodedArrayToActualValue(int[] key, int index) + { + if (key == null || key.length <= index) { + return null; + } + return getActualValue(key[index], false); + } + + @Override + public int getLengthOfUnsortedEncodedArray(int[] key) + { + return key.length; + } + @Override public int[] convertUnsortedEncodedArrayToSortedEncodedArray(int[] key) { diff --git a/processing/src/main/java/io/druid/segment/column/ValueType.java b/processing/src/main/java/io/druid/segment/column/ValueType.java index 1760fea8c725..0efb03218c75 100644 --- a/processing/src/main/java/io/druid/segment/column/ValueType.java +++ b/processing/src/main/java/io/druid/segment/column/ValueType.java @@ -19,14 +19,42 @@ package io.druid.segment.column; +import io.druid.data.input.impl.DimensionSchema; + /** -*/ + */ public enum ValueType { - FLOAT, - LONG, - STRING, - COMPLEX; + FLOAT { + @Override + public DimensionSchema.ValueType asDimensionType() + { + return DimensionSchema.ValueType.STRING; + } + }, + LONG { + @Override + public DimensionSchema.ValueType asDimensionType() + { + return DimensionSchema.ValueType.LONG; + } + }, + STRING { + @Override + public DimensionSchema.ValueType asDimensionType() + { + return DimensionSchema.ValueType.STRING; + } + }, + COMPLEX { + @Override + public DimensionSchema.ValueType asDimensionType() + { + return DimensionSchema.ValueType.COMPLEX; + } + }; + + public abstract DimensionSchema.ValueType asDimensionType(); public static ValueType typeFor(Class clazz) { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 6ec6db6d5f9e..3a5821ba0b3f 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -295,7 +295,7 @@ public int lookupId(String name) } @Override - public NumericColumnSelector makeMathExpressionSelector(String expression) + public NumericColumnSelector makeExpressionSelector(String expression) { final Expr parsed = Parser.parse(expression); @@ -673,6 +673,11 @@ public DimensionDesc getDimension(String dimension) } } + public MetricDesc getMetric(String metric) + { + return metricDescs.get(metric); + } + public String getMetricType(String metric) { final MetricDesc metricDesc = metricDescs.get(metric); diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 443abed638ae..5e62dc7a3210 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -22,18 +22,22 @@ import com.google.common.base.Function; import com.google.common.base.Strings; import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import io.druid.data.input.impl.DimensionSchema; import io.druid.granularity.QueryGranularity; +import io.druid.math.expr.Evals; import io.druid.math.expr.Expr; import io.druid.math.expr.Parser; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.query.QueryInterruptedException; import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.DruidLongPredicate; @@ -562,7 +566,7 @@ public ColumnCapabilities getColumnCapabilities(String columnName) } @Override - public NumericColumnSelector makeMathExpressionSelector(String expression) + public NumericColumnSelector makeExpressionSelector(String expression) { final Expr parsed = Parser.parse(expression); @@ -723,6 +727,77 @@ private ValueType getTypeForDimension(String dimension) ColumnCapabilities capabilities = index.getCapabilities(dimension); return capabilities == null ? ValueType.STRING : capabilities.getType(); } + + @Override + public ValueMatcher makeExpressionMatcher(String expression) + { + final Expr parsed = Parser.parse(expression); + + final Map> values = Maps.newHashMap(); + for (String column : Parser.findRequiredBindings(parsed)) { + IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(column); + if (dimensionDesc != null) { + if (dimensionDesc.getCapabilities().hasMultipleValues()) { + throw new IllegalArgumentException("multi-valued dimension"); + } + final int dimIndex = dimensionDesc.getIndex(); + final DimensionIndexer indexer = dimensionDesc.getIndexer(); + final Supplier supplier = new Supplier() + { + @Override + public Comparable get() + { + final Object[] dims = holder.getKey().getDims(); + if (dimIndex < dims.length && dims[dimIndex] != null && + indexer.getLengthOfUnsortedEncodedArray(dims[dimIndex]) == 1) { + return indexer.convertUnsortedEncodedArrayToActualValue(dims[dimIndex], 0); + } + return null; + } + }; + DimensionSchema.ValueType type = dimensionDesc.getCapabilities().getType().asDimensionType(); + values.put(column, Suppliers.compose(Evals.asNumberFunc(type), supplier)); + continue; + } + IncrementalIndex.MetricDesc metricDesc = index.getMetric(column); + if (metricDesc != null) { + final int metricIndex = metricDesc.getIndex(); + final ValueType type = ValueType.valueOf(metricDesc.getType().toUpperCase()); + if (type == ValueType.FLOAT) { + final FloatColumnSelector selector = new FloatColumnSelector() + { + @Override + public float get() + { + return index.getMetricFloatValue(holder.getValue(), metricIndex); + } + }; + values.put(column, AggregatorUtil.asSupplier(selector)); + } else if (type == ValueType.LONG) { + final LongColumnSelector selector = new LongColumnSelector() + { + @Override + public long get() + { + return index.getMetricLongValue(holder.getValue(), metricIndex); + } + }; + values.put(column, AggregatorUtil.asSupplier(selector)); + } else { + throw new UnsupportedOperationException("Unsupported type " + type); + } + } + } + + final Expr.ObjectBinding binding = Parser.withSuppliers(values); + return new ValueMatcher() { + @Override + public boolean matches() + { + return parsed.eval(binding).asBoolean(); + } + }; + } } @Override diff --git a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java index 4604b9ce8e04..44e399c39aa0 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java @@ -410,9 +410,9 @@ public ColumnCapabilities getColumnCapabilities(String columnName) } @Override - public NumericColumnSelector makeMathExpressionSelector(String expression) + public NumericColumnSelector makeExpressionSelector(String expression) { - return delegate.makeMathExpressionSelector(expression); + return delegate.makeExpressionSelector(expression); } } diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 042c5ed34a68..e4fe8ef0d61e 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -185,7 +185,7 @@ public ColumnCapabilities getColumnCapabilities(String columnName) } @Override - public NumericColumnSelector makeMathExpressionSelector(String expression) + public NumericColumnSelector makeExpressionSelector(String expression) { throw new UnsupportedOperationException(); } diff --git a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java index 3bb6d26fe65a..a53beb5666fb 100644 --- a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java @@ -79,7 +79,7 @@ public ColumnCapabilities getColumnCapabilities(String columnName) } @Override - public NumericColumnSelector makeMathExpressionSelector(String expression) + public NumericColumnSelector makeExpressionSelector(String expression) { return null; } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 0c36dea67a3d..cfbae49d751b 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -82,6 +82,7 @@ import io.druid.query.filter.AndDimFilter; import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; +import io.druid.query.filter.ExpressionFilter; import io.druid.query.filter.ExtractionDimFilter; import io.druid.query.filter.InDimFilter; import io.druid.query.filter.JavaScriptDimFilter; @@ -3946,6 +3947,16 @@ public void testSubqueryWithExtractionFnInOuterQuery() // Subqueries are handled by the ToolChest Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); + + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "a", "rows", 6L, "idx", 771L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "a", "rows", 6L, "idx", 778L) + ); + + query = query.withDimFilter(new ExpressionFilter("idx > 100 && idx < 200")); + TestHelper.assertExpectedObjects( + expectedResults, GroupByQueryRunnerTestHelper.runQuery(factory, runner, query), "" + ); } @Test diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java index 8373698976c0..cb8e7291d798 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java @@ -90,7 +90,7 @@ public Object get() } @Override - public NumericColumnSelector makeMathExpressionSelector(String expression) + public NumericColumnSelector makeExpressionSelector(String expression) { throw new UnsupportedOperationException("expression is not supported in current context"); } diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 10586137931c..6bbd9e915d4c 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -33,6 +33,7 @@ import io.druid.query.extraction.MapLookupExtractor; import io.druid.query.filter.AndDimFilter; import io.druid.query.filter.DimFilter; +import io.druid.query.filter.ExpressionFilter; import io.druid.query.filter.ExtractionDimFilter; import io.druid.query.filter.SelectorDimFilter; import io.druid.query.lookup.LookupExtractionFn; @@ -130,6 +131,17 @@ public void testSearch() expectedHits.add(new SearchHit(QueryRunnerTestHelper.partialNullDimension, "value", 186)); checkSearchQuery(searchQuery, expectedHits); + + searchQuery = searchQuery.withDimFilter(new ExpressionFilter("index < 100")); + + expectedHits = Lists.newLinkedList(); + expectedHits.add(new SearchHit(QueryRunnerTestHelper.placementishDimension, "a", 14)); + expectedHits.add(new SearchHit(QueryRunnerTestHelper.qualityDimension, "automotive", 14)); + expectedHits.add(new SearchHit(QueryRunnerTestHelper.qualityDimension, "entertainment", 12)); + expectedHits.add(new SearchHit(QueryRunnerTestHelper.qualityDimension, "health", 24)); + expectedHits.add(new SearchHit(QueryRunnerTestHelper.qualityDimension, "mezzanine", 23)); + + checkSearchQuery(searchQuery, expectedHits); } @Test diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index 258bb57a4f32..1504b282e171 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -40,6 +40,7 @@ import io.druid.query.extraction.MapLookupExtractor; import io.druid.query.filter.AndDimFilter; import io.druid.query.filter.DimFilter; +import io.druid.query.filter.ExpressionFilter; import io.druid.query.filter.SelectorDimFilter; import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.spec.LegacySegmentSpec; @@ -96,6 +97,9 @@ public class SelectQueryRunnerTest "2011-01-13T00:00:00.000Z upfront premium preferred ppreferred 1564.617729 value" }; + public static final QuerySegmentSpec I_0112_0113 = new LegacySegmentSpec( + new Interval("2011-01-12/2011-01-13") + ); public static final QuerySegmentSpec I_0112_0114 = new LegacySegmentSpec( new Interval("2011-01-12/2011-01-14") ); @@ -165,6 +169,52 @@ public void testFullOnSelect() verify(expectedResults, populateNullColumnAtLastForQueryableIndexCase(results, "null_column")); } + @Test + public void testFilterOnMetric() + { + SelectQuery query = newTestQuery() + .filters(new ExpressionFilter("index > 100 && index < 1000")) + .intervals(I_0112_0113) + .build(); + + List> results = Sequences.toList( + runner.run(query, Maps.newHashMap()), + Lists.>newArrayList() + ); + Assert.assertEquals(1, results.size()); + + List result = results.get(0).getValue().getEvents(); + List expected; + if (!descending) { + expected = Arrays.asList( + new EventHolder( + "testSegment", 0, ImmutableMap.of( + "timestamp", "2011-01-12", "market", "upfront", "quality", "mezzanine", "index", 800F + ) + ), + new EventHolder( + "testSegment", 1, ImmutableMap.of( + "timestamp", "2011-01-12", "market", "upfront", "quality", "premium", "index", 800F + ) + ) + ); + } else { + expected = Arrays.asList( + new EventHolder( + "testSegment", -1, ImmutableMap.of( + "timestamp", "2011-01-12", "market", "upfront", "quality", "premium", "index", 800F + ) + ), + new EventHolder( + "testSegment", -2, ImmutableMap.of( + "timestamp", "2011-01-12", "market", "upfront", "quality", "mezzanine", "index", 800F + ) + ) + ); + } + verify(expected, result); + } + @Test public void testSequentialPaging() { @@ -714,41 +764,51 @@ private static void verify( Assert.assertEquals(expected.getTimestamp(), actual.getTimestamp()); - for (Map.Entry entry : expected.getValue().getPagingIdentifiers().entrySet()) { - Assert.assertEquals(entry.getValue(), actual.getValue().getPagingIdentifiers().get(entry.getKey())); + SelectResultValue e = expected.getValue(); + SelectResultValue r = actual.getValue(); + for (Map.Entry entry : e.getPagingIdentifiers().entrySet()) { + Assert.assertEquals(entry.getValue(), r.getPagingIdentifiers().get(entry.getKey())); } - Assert.assertEquals(expected.getValue().getDimensions(), actual.getValue().getDimensions()); - Assert.assertEquals(expected.getValue().getMetrics(), actual.getValue().getMetrics()); + Assert.assertEquals(e.getDimensions(), r.getDimensions()); + Assert.assertEquals(e.getMetrics(), r.getMetrics()); - Iterator expectedEvts = expected.getValue().getEvents().iterator(); - Iterator actualEvts = actual.getValue().getEvents().iterator(); + verify(e.getEvents(), r.getEvents()); + } + + if (actualIter.hasNext()) { + throw new ISE("This iterator should be exhausted!"); + } + } - while (expectedEvts.hasNext()) { - EventHolder exHolder = expectedEvts.next(); - EventHolder acHolder = actualEvts.next(); + private static void verify(List expected, List actual) + { + Iterator expectedEvts = expected.iterator(); + Iterator actualEvts = actual.iterator(); - Assert.assertEquals(exHolder.getTimestamp(), acHolder.getTimestamp()); - Assert.assertEquals(exHolder.getOffset(), acHolder.getOffset()); + while (expectedEvts.hasNext()) { + EventHolder exHolder = expectedEvts.next(); + EventHolder acHolder = actualEvts.next(); - for (Map.Entry ex : exHolder.getEvent().entrySet()) { - Object actVal = acHolder.getEvent().get(ex.getKey()); + Assert.assertEquals(exHolder.getTimestamp(), acHolder.getTimestamp()); + Assert.assertEquals(exHolder.getOffset(), acHolder.getOffset()); - // work around for current II limitations - if (acHolder.getEvent().get(ex.getKey()) instanceof Double) { - actVal = ((Double) actVal).floatValue(); - } - Assert.assertEquals("invalid value for " + ex.getKey(), ex.getValue(), actVal); + for (Map.Entry ex : exHolder.getEvent().entrySet()) { + if (ex.getKey().equals("timestamp")) { + continue; // already done above } - } + Object actVal = acHolder.getEvent().get(ex.getKey()); - if (actualEvts.hasNext()) { - throw new ISE("This event iterator should be exhausted!"); + // work around for current II limitations + if (acHolder.getEvent().get(ex.getKey()) instanceof Double) { + actVal = ((Double) actVal).floatValue(); + } + Assert.assertEquals("invalid value for " + ex.getKey(), ex.getValue(), actVal); } } - if (actualIter.hasNext()) { - throw new ISE("This iterator should be exhausted!"); + if (actualEvts.hasNext()) { + throw new ISE("This event iterator should be exhausted!"); } }