diff --git a/extensions-core/filters/pom.xml b/extensions-core/filters/pom.xml new file mode 100644 index 000000000000..ad9847b5b9c0 --- /dev/null +++ b/extensions-core/filters/pom.xml @@ -0,0 +1,115 @@ + + + + + 4.0.0 + + io.druid.extensions + druid-filters + druid-filters + Druid filter extensions + + + io.druid + druid + 0.9.0-SNAPSHOT + ../../pom.xml + + + + + io.druid + druid-api + ${druid.api.version} + provided + + + io.druid + druid-processing + ${project.parent.version} + provided + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + provided + + + com.fasterxml.jackson.datatype + jackson-datatype-guava + ${jackson.version} + provided + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + ${jackson.version} + provided + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + ${jackson.version} + provided + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + ${jackson.version} + provided + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-smile-provider + ${jackson.version} + provided + + + + + junit + junit + test + + + io.druid + druid-processing + ${project.parent.version} + test-jar + test + + + + diff --git a/extensions-core/filters/src/main/java/io/druid/query/filter/BinaryOperator.java b/extensions-core/filters/src/main/java/io/druid/query/filter/BinaryOperator.java new file mode 100644 index 000000000000..d1f204158382 --- /dev/null +++ b/extensions-core/filters/src/main/java/io/druid/query/filter/BinaryOperator.java @@ -0,0 +1,154 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.google.common.base.Predicate; +import com.google.common.base.Strings; + +import java.util.Comparator; + +/** + */ +public enum BinaryOperator +{ + GT { + @Override + public Predicate toPredicate(final Comparator comparator, final String value) + { + return new Predicate() + { + @Override + public boolean apply(String input) + { + return comparator.compare(input, value) > 0; + } + }; + } + }, + GTE { + @Override + public Predicate toPredicate(final Comparator comparator, final String value) + { + return new Predicate() + { + @Override + public boolean apply(String input) + { + return comparator.compare(input, value) >= 0; + } + }; + } + }, + LT { + @Override + public Predicate toPredicate(final Comparator comparator, final String value) + { + return new Predicate() + { + @Override + public boolean apply(String input) + { + return comparator.compare(input, value) < 0; + } + }; + } + }, + LTE { + @Override + public Predicate toPredicate(final Comparator comparator, final String value) + { + return new Predicate() + { + @Override + public boolean apply(String input) + { + return comparator.compare(input, value) <= 0; + } + }; + } + }, + EQ { + @Override + public Predicate toPredicate(final Comparator comparator, final String value) + { + return new Predicate() + { + @Override + public boolean apply(String input) + { + return comparator.compare(input, value) == 0; + } + }; + } + }, + NE { + @Override + public Predicate toPredicate(final Comparator comparator, final String value) + { + return new Predicate() + { + @Override + public boolean apply(String input) + { + return comparator.compare(input, value) != 0; + } + }; + } + }; + + public abstract Predicate toPredicate(Comparator comparator, final String value); + + public static BinaryOperator get(String value) + { + if (Strings.isNullOrEmpty(value)) { + return EQ; + } + value = value.toLowerCase(); + switch (value) { + case "gt": + case "greaterthan": + case ">": + return GT; + case "gte": + case "greaterthanorequalto": + case ">=": + return GTE; + case "lt": + case "lessthan": + case "<": + return LT; + case "lte": + case "lessthanorequalto": + case "<=": + return LTE; + case "eq": + case "equals": + case "=": + case "==": + return EQ; + case "ne": + case "notequals": + case "!=": + case "<>": + return NE; + } + throw new IllegalArgumentException("Invalid operator " + value); + } +} diff --git a/extensions-core/filters/src/main/java/io/druid/query/filter/FilterExtensionsModule.java b/extensions-core/filters/src/main/java/io/druid/query/filter/FilterExtensionsModule.java new file mode 100644 index 000000000000..193d344e98b5 --- /dev/null +++ b/extensions-core/filters/src/main/java/io/druid/query/filter/FilterExtensionsModule.java @@ -0,0 +1,56 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.query.filter; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import io.druid.initialization.DruidModule; + +import java.util.Arrays; +import java.util.List; + +public class FilterExtensionsModule implements DruidModule +{ + public FilterExtensionsModule() {} + + @Override + public List getJacksonModules() + { + return Arrays.asList( + new SimpleModule("FilterExtensionsModule") + .setMixInAnnotation(DimFilter.class, DimFilterMixIn.class) + ); + } + + @Override + public void configure(Binder binder) + { + } +} + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "selectorEx", value = SelectorDimFilterExtension.class), +}) +abstract class DimFilterMixIn +{ +} diff --git a/extensions-core/filters/src/main/java/io/druid/query/filter/SelectorDimFilterExtension.java b/extensions-core/filters/src/main/java/io/druid/query/filter/SelectorDimFilterExtension.java new file mode 100644 index 000000000000..4a842ece01ae --- /dev/null +++ b/extensions-core/filters/src/main/java/io/druid/query/filter/SelectorDimFilterExtension.java @@ -0,0 +1,129 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import io.druid.query.ordering.StringComparators; + +import java.nio.ByteBuffer; + +/** + */ +public class SelectorDimFilterExtension extends SelectorDimFilter implements DimFilterExtension +{ + private final String compareType; + private final BinaryOperator operator; + + @JsonCreator + public SelectorDimFilterExtension( + @JsonProperty("dimension") String dimension, + @JsonProperty("value") String value, + @JsonProperty("operator") String operator, + @JsonProperty("compareType") String compareType + ) + { + super(dimension, value); + this.operator = BinaryOperator.get(operator); + this.compareType = compareType == null ? StringComparators.LEXICOGRAPHIC_NAME : compareType; + + // don't allow null comparison, for now + Preconditions.checkArgument( + !(this.operator != BinaryOperator.EQ && this.operator != BinaryOperator.NE && Strings.isNullOrEmpty(value)), + "null comparison is not allowed, except equals/not-equals" + ); + Preconditions.checkArgument(StringComparators.validate(this.compareType), "Invalid compare type " + compareType); + } + + public SelectorDimFilterExtension(String dimension, String value, String operator) + { + this(dimension, value, operator, null); + } + + public SelectorDimFilterExtension(String dimension, String value) + { + this(dimension, value, null, null); + } + + @Override + public byte[] getCacheKey() + { + byte[] cacheKey = super.getCacheKey(); + byte[] dimensionBytes = com.metamx.common.StringUtils.toUtf8(compareType); + + return ByteBuffer.allocate(cacheKey.length + dimensionBytes.length + 1) + .put(cacheKey) + .put(dimensionBytes) + .put((byte) operator.ordinal()) + .array(); + } + + @JsonProperty + public String getOperator() + { + return operator.name(); + } + + @JsonProperty + public String getCompareType() + { + return compareType; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (super.equals(o)) { + SelectorDimFilterExtension that = (SelectorDimFilterExtension) o; + return operator.equals(that.operator) && compareType.equals(that.compareType); + } + + return false; + } + + @Override + public int hashCode() + { + int result = super.hashCode(); + result = 31 * result + operator.ordinal(); + result = 31 * result + compareType.hashCode(); + return result; + } + + @Override + public String toString() + { + return String.format("%s %s %s %s", dimension, operator.name(), value, compareType); + } + + @Override + public Filter toFilter() + { + return new SelectorFilterExtension(dimension, value, compareType, operator); + } +} diff --git a/extensions-core/filters/src/main/java/io/druid/query/filter/SelectorFilterExtension.java b/extensions-core/filters/src/main/java/io/druid/query/filter/SelectorFilterExtension.java new file mode 100644 index 000000000000..372e179bc5bc --- /dev/null +++ b/extensions-core/filters/src/main/java/io/druid/query/filter/SelectorFilterExtension.java @@ -0,0 +1,188 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.google.common.base.Predicate; +import com.google.common.base.Strings; +import com.metamx.collections.bitmap.BitmapFactory; +import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.ordering.StringComparators; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.data.ArrayIndexed; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.filter.BooleanValueMatcher; + +import java.util.Arrays; + +/** + */ +public class SelectorFilterExtension implements Filter +{ + private final String dimension; + private final String value; + private final BinaryOperator operator; + private final StringComparators.StringComparator comparator; + + public SelectorFilterExtension( + String dimension, + String value, + String compareType, + BinaryOperator operator + ) + { + this.dimension = dimension; + this.value = value; + this.comparator = StringComparators.makeComparator(compareType); + this.operator = operator; + } + + public SelectorFilterExtension(String dimension, String value) + { + this(dimension, value, StringComparators.LEXICOGRAPHIC_NAME, null); + } + + @Override + public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector) + { + final BitmapFactory factory = selector.getBitmapFactory(); + final Indexed values = sortValues(selector.getDimensionValues(dimension)); + switch (operator) { + case GT: + case GTE: { + int index = values.indexOf(value); + int start = index < 0 ? -index - 1 : operator == BinaryOperator.GT ? index + 1 : index; + ImmutableBitmap bitmap = factory.makeEmptyImmutableBitmap(); + for (int cursor = start; cursor < values.size(); cursor++) { + bitmap = bitmap.union(selector.getBitmapIndex(dimension, values.get(cursor))); + } + return bitmap; + } + case LT: + case LTE: { + int index = values.indexOf(value); + int limit = index < 0 ? -index - 2 : operator == BinaryOperator.LT ? index - 1 : index; + ImmutableBitmap bitmap = factory.makeEmptyImmutableBitmap(); + for (int cursor = 0; cursor <= limit; cursor++) { + bitmap = bitmap.union(selector.getBitmapIndex(dimension, values.get(cursor))); + } + return bitmap; + } + case EQ: + return selector.getBitmapIndex(dimension, value); + case NE: + return factory.complement(selector.getBitmapIndex(dimension, value), selector.getNumRows()); + } + throw new IllegalArgumentException("Not supported operator " + operator); + } + + private Indexed sortValues(Indexed values) + { + if (comparator instanceof StringComparators.LexicographicComparator) { + return values; + } + String[] valueArray = new String[values.size()]; + for (int i = 0; i < valueArray.length; i++) { + valueArray[i] = values.get(i); + } + Arrays.sort(valueArray, comparator); + return new ArrayIndexed<>(valueArray, comparator, String.class); + } + + @Override + public ValueMatcher makeMatcher(ValueMatcherFactory factory) + { + if (operator == BinaryOperator.EQ || operator == BinaryOperator.NE) { + final ValueMatcher matcher = factory.makeValueMatcher(dimension, value); + return operator == BinaryOperator.EQ ? matcher : new RevertedMatcher(matcher); + } + return factory.makeValueMatcher(dimension, operator.toPredicate(comparator, value)); + } + + @Override + public ValueMatcher makeMatcher(ColumnSelectorFactory columnSelectorFactory) + { + final DimensionSelector dimensionSelector = columnSelectorFactory.makeDimensionSelector( + new DefaultDimensionSpec(dimension, dimension) + ); + + if (operator == BinaryOperator.EQ || operator == BinaryOperator.NE) { + final ValueMatcher matcher = toValueMatcher(dimensionSelector); + return operator == BinaryOperator.EQ ? matcher : new RevertedMatcher(matcher); + } + + final Predicate predicate = operator.toPredicate(comparator, value); + return new ValueMatcher() + { + @Override + public boolean matches() + { + final IndexedInts row = dimensionSelector.getRow(); + final int size = row.size(); + for (int i = 0; i < size; ++i) { + if (predicate.apply(dimensionSelector.lookupName(row.get(i)))) { + return true; + } + } + return false; + } + }; + } + + private ValueMatcher toValueMatcher(final DimensionSelector dimensionSelector) + { + if (dimensionSelector == null) { + // Missing columns match a null or empty string value and don't match anything else + return new BooleanValueMatcher(Strings.isNullOrEmpty(value)); + } + final int valueId = dimensionSelector.lookupId(value); + return new ValueMatcher() + { + @Override + public boolean matches() + { + final IndexedInts row = dimensionSelector.getRow(); + final int size = row.size(); + for (int i = 0; i < size; ++i) { + if (row.get(i) == valueId) { + return true; + } + } + return false; + } + }; + } + + // for NE + private static class RevertedMatcher implements ValueMatcher + { + private final ValueMatcher matcher; + + private RevertedMatcher(ValueMatcher matcher) {this.matcher = matcher;} + + @Override + public boolean matches() + { + return !matcher.matches(); + } + } +} diff --git a/extensions-core/filters/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/filters/src/main/resources/META-INF/services/io.druid.initialization.DruidModule new file mode 100644 index 000000000000..55765444ac7d --- /dev/null +++ b/extensions-core/filters/src/main/resources/META-INF/services/io.druid.initialization.DruidModule @@ -0,0 +1 @@ +io.druid.query.filter.FilterExtensionsModule diff --git a/extensions-core/filters/src/test/java/io/druid/query/filter/CompareTypeTest.java b/extensions-core/filters/src/test/java/io/druid/query/filter/CompareTypeTest.java new file mode 100644 index 000000000000..08fdf133af98 --- /dev/null +++ b/extensions-core/filters/src/test/java/io/druid/query/filter/CompareTypeTest.java @@ -0,0 +1,223 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.io.CharSource; +import com.metamx.common.guava.Sequences; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.Druids; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.Result; +import io.druid.query.select.EventHolder; +import io.druid.query.select.PagingSpec; +import io.druid.query.select.SelectQuery; +import io.druid.query.select.SelectQueryEngine; +import io.druid.query.select.SelectQueryQueryToolChest; +import io.druid.query.select.SelectQueryRunnerFactory; +import io.druid.query.select.SelectResultValue; +import io.druid.segment.IncrementalIndexSegment; +import io.druid.segment.QueryableIndex; +import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.TestIndex; +import io.druid.segment.incremental.IncrementalIndex; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import static io.druid.query.QueryRunnerTestHelper.makeQueryRunner; +import static io.druid.query.QueryRunnerTestHelper.marketDimension; +import static io.druid.query.ordering.StringComparators.ALPHANUMERIC_NAME; +import static io.druid.query.ordering.StringComparators.LEXICOGRAPHIC_NAME; +import static io.druid.query.ordering.StringComparators.NUMERIC_NAME; + +/** + */ +@RunWith(Parameterized.class) +public class CompareTypeTest +{ + @Parameterized.Parameters + public static Iterable constructorFeeder() throws IOException + { + SelectQueryRunnerFactory factory = new SelectQueryRunnerFactory( + new SelectQueryQueryToolChest( + new DefaultObjectMapper(), + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ), + new SelectQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + + CharSource input = CharSource.wrap( + "2011-01-12T00:00:00.000Z\t9.45 automotive preferred apreferred 100.000000\n" + + "2011-01-12T00:00:00.000Z\t-1.2 business preferred bpreferred 100.000000\n" + + "2011-01-12T00:00:00.000Z\t4.7 entertainment preferred epreferred 100.000000\n" + + "2011-01-12T00:00:00.000Z\t\thealth preferred hpreferred 100.000000\n" + + "2011-01-12T00:00:00.000Z\t13.45 mezzanine preferred mpreferred 100.000000\n" + ); + + IncrementalIndex index1 = TestIndex.makeRealtimeIndex(input); + IncrementalIndex index2 = TestIndex.makeRealtimeIndex(input); + + QueryableIndex index3 = TestIndex.persistRealtimeAndLoadMMapped(index1); + QueryableIndex index4 = TestIndex.persistRealtimeAndLoadMMapped(index2); + + return QueryRunnerTestHelper.transformToConstructionFeeder( + Arrays.asList( + makeQueryRunner(factory, "index1", new IncrementalIndexSegment(index1, "index1")), + makeQueryRunner(factory, "index2", new IncrementalIndexSegment(index2, "index2")), + makeQueryRunner(factory, "index3", new QueryableIndexSegment("index3", index3)), + makeQueryRunner(factory, "index4", new QueryableIndexSegment("index4", index4)) + ) + ); + } + + private final QueryRunner runner; + + public CompareTypeTest( + QueryRunner runner + ) + { + this.runner = runner; + } + + + @Test + public void testTimeSeriesWithFilteredAggWithSelectDimFilter() + { + Druids.SelectQueryBuilder builder = + Druids.newSelectQueryBuilder() + .dimensions(Arrays.asList(QueryRunnerTestHelper.marketDimension)) + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.allGran) + .intervals(QueryRunnerTestHelper.fullOnInterval) + .pagingSpec(new PagingSpec(null, 10)); + + // lexicographic : [null, -1.2, 13.45, 4.7, 9.45] + // alphaNumeric : [null, 4.7, 9.45, 13.45, -1.2] + // numeric : [null, -1.2, 4.7, 9.45, 13.45] + + // existing + validate(builder, ">", "4.7", "[9.45]", "[-1.2, 13.45, 9.45]", "[13.45, 9.45]"); + validate(builder, ">=", "4.7", "[4.7, 9.45]", "[-1.2, 13.45, 4.7, 9.45]", "[13.45, 4.7, 9.45]"); + validate(builder, "<", "4.7", "[null, -1.2, 13.45]", "[null]", "[null, -1.2]"); + validate(builder, "<=", "4.7", "[null, -1.2, 13.45, 4.7]", "[null, 4.7]", "[null, -1.2, 4.7]"); + validate(builder, "==", "4.7", "[4.7]", "[4.7]", "[4.7]"); + validate( + builder, "<>", "4.7", + "[null, -1.2, 13.45, 9.45]", "[null, -1.2, 13.45, 9.45]", "[null, -1.2, 13.45, 9.45]" + ); + + // non-existing (between) + validate(builder, ">", "3.1", "[4.7, 9.45]", "[-1.2, 13.45, 4.7, 9.45]", "[13.45, 4.7, 9.45]"); + validate(builder, ">=", "3.1", "[4.7, 9.45]", "[-1.2, 13.45, 4.7, 9.45]", "[13.45, 4.7, 9.45]"); + validate(builder, "<", "3.1", "[null, -1.2, 13.45]", "[null]", "[null, -1.2]"); + validate(builder, "<=", "3.1", "[null, -1.2, 13.45]", "[null]", "[null, -1.2]"); + validate(builder, "==", "3.1", "[]", "[]", "[]"); + validate( + builder, "<>", "3.1", + "[null, -1.2, 13.45, 4.7, 9.45]", "[null, -1.2, 13.45, 4.7, 9.45]", "[null, -1.2, 13.45, 4.7, 9.45]" + ); + + // non-existing (smaller than min) + validate(builder, ">", "-12.4", "[13.45, 4.7, 9.45]", "[]", "[-1.2, 13.45, 4.7, 9.45]"); + validate(builder, ">=", "-12.4", "[13.45, 4.7, 9.45]", "[]", "[-1.2, 13.45, 4.7, 9.45]"); + validate(builder, "<", "-12.4", "[null, -1.2]", "[null, -1.2, 13.45, 4.7, 9.45]", "[null]"); + validate(builder, "<=", "-12.4", "[null, -1.2]", "[null, -1.2, 13.45, 4.7, 9.45]", "[null]"); + validate(builder, "==", "-12.4", "[]", "[]", "[]"); + validate( + builder, "<>", "-12.4", + "[null, -1.2, 13.45, 4.7, 9.45]", "[null, -1.2, 13.45, 4.7, 9.45]", "[null, -1.2, 13.45, 4.7, 9.45]" + ); + + // non-existing (bigger than max) + validate(builder, ">", "42", "[9.45]", "[-1.2]", "[]"); + validate(builder, ">=", "42", "[9.45]", "[-1.2]", "[]"); + validate( + builder, "<", "42", + "[null, -1.2, 13.45, 4.7]", "[null, 13.45, 4.7, 9.45]", "[null, -1.2, 13.45, 4.7, 9.45]" + ); + validate( + builder, "<=", "42", + "[null, -1.2, 13.45, 4.7]", "[null, 13.45, 4.7, 9.45]", "[null, -1.2, 13.45, 4.7, 9.45]" + ); + validate(builder, "==", "42", "[]", "[]", "[]"); + validate( + builder, "<>", "42", + "[null, -1.2, 13.45, 4.7, 9.45]", "[null, -1.2, 13.45, 4.7, 9.45]", "[null, -1.2, 13.45, 4.7, 9.45]" + ); + + // null + validate(builder, "==", "", "[null]", "[null]", "[null]"); + validate(builder, "<>", "", "[-1.2, 13.45, 4.7, 9.45]", "[-1.2, 13.45, 4.7, 9.45]", "[-1.2, 13.45, 4.7, 9.45]"); + } + + private void validate( + Druids.SelectQueryBuilder builder, + String operation, + String value, + String lexicographicExpected, + String alphaNumericExpected, + String numericExpected + ) + { + builder.filters(new SelectorDimFilterExtension(marketDimension, value, operation, LEXICOGRAPHIC_NAME)); + validate(builder.build(), lexicographicExpected); + + builder.filters(new SelectorDimFilterExtension(marketDimension, value, operation, ALPHANUMERIC_NAME)); + validate(builder.build(), alphaNumericExpected); + + builder.filters(new SelectorDimFilterExtension(marketDimension, value, operation, NUMERIC_NAME)); + validate(builder.build(), numericExpected); + } + + private void validate(SelectQuery query, String expected) + { + Iterable> results = Sequences.toList( + runner.run(query, ImmutableMap.of()), + Lists.>newArrayList() + ); + Iterator> iterator = results.iterator(); + Assert.assertTrue(iterator.hasNext()); + List events = iterator.next().getValue().getEvents(); + List markets = Lists.transform( + events, new Function() + { + @Override + public String apply(EventHolder input) + { + return (String) input.getEvent().get(marketDimension); + } + } + ); + Assert.assertEquals(expected, markets.toString()); + Assert.assertFalse(iterator.hasNext()); + } + +} diff --git a/extensions-core/filters/src/test/java/io/druid/query/filter/SelectQueryTest.java b/extensions-core/filters/src/test/java/io/druid/query/filter/SelectQueryTest.java new file mode 100644 index 000000000000..5a5ce1c8d108 --- /dev/null +++ b/extensions-core/filters/src/test/java/io/druid/query/filter/SelectQueryTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.metamx.common.guava.Sequences; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.Druids; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.Result; +import io.druid.query.TableDataSource; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.select.PagingSpec; +import io.druid.query.select.SelectQueryEngine; +import io.druid.query.select.SelectQueryQueryToolChest; +import io.druid.query.select.SelectQueryRunnerFactory; +import io.druid.query.select.SelectQueryRunnerTest; +import io.druid.query.select.SelectResultValue; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.Arrays; + +/** + */ +@RunWith(Parameterized.class) +public class SelectQueryTest +{ + + @Parameterized.Parameters(name = "{0}:descending={1}") + public static Iterable constructorFeeder() throws IOException + { + return QueryRunnerTestHelper.cartesian( + QueryRunnerTestHelper.makeQueryRunners( + new SelectQueryRunnerFactory( + new SelectQueryQueryToolChest( + new DefaultObjectMapper(), + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ), + new SelectQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ) + ), + Arrays.asList(false, true) + ); + } + + private final QueryRunner runner; + private final boolean descending; + + public SelectQueryTest(QueryRunner runner, boolean descending) + { + this.runner = runner; + this.descending = descending; + } + + @Test + public void testSelectWithSelectDimFilter() + { + Druids.SelectQueryBuilder builder = new Druids.SelectQueryBuilder() + .dataSource(new TableDataSource(QueryRunnerTestHelper.dataSource)) + .intervals(SelectQueryRunnerTest.I_0112_0114) + .descending(descending) + .granularity(QueryRunnerTestHelper.allGran) + .dimensionSpecs(DefaultDimensionSpec.toSpec(Arrays.asList(QueryRunnerTestHelper.qualityDimension))) + .metrics(Arrays.asList(QueryRunnerTestHelper.indexMetric)) + .pagingSpec(new PagingSpec(null, 20)); + + // existing + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "entertainment", "GT"), + new JavaScriptDimFilter( + QueryRunnerTestHelper.qualityDimension, + "function(dim){ return dim > 'entertainment'; }" + ) + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "entertainment", "GTE"), + new JavaScriptDimFilter( + QueryRunnerTestHelper.qualityDimension, + "function(dim){ return dim >= 'entertainment'; }" + ) + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "entertainment", "LT"), + new JavaScriptDimFilter( + QueryRunnerTestHelper.qualityDimension, + "function(dim){ return dim < 'entertainment'; }" + ) + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "entertainment", "LTE"), + new JavaScriptDimFilter( + QueryRunnerTestHelper.qualityDimension, + "function(dim){ return dim <= 'entertainment'; }" + ) + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "entertainment", "EQ"), + new JavaScriptDimFilter( + QueryRunnerTestHelper.qualityDimension, + "function(dim){ return dim === 'entertainment'; }" + ) + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "entertainment", "NE"), + new JavaScriptDimFilter( + QueryRunnerTestHelper.qualityDimension, + "function(dim){ return dim != 'entertainment'; }" + ) + ); + + // non-existing (between) + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "financial", "GT"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim > 'financial'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "financial", "GTE"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim >= 'financial'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "financial", "LT"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim < 'financial'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "financial", "LTE"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim <= 'financial'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "financial", "EQ"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim === 'financial'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "financial", "NE"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim != 'financial'; }") + ); + + // non-existing (smaller than min) + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "abcb", "GT"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim > 'abcb'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "abcb", "GTE"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim >= 'abcb'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "abcb", "LT"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim < 'abcb'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "abcb", "LTE"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim <= 'abcb'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "abcb", "EQ"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim === 'abcb'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "abcb", "NE"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim != 'abcb'; }") + ); + + // non-existing (bigger than max) + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "zztop", "GT"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim > 'zztop'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "zztop", "GTE"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim >= 'zztop'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "zztop", "LT"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim < 'zztop'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "zztop", "LTE"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim <= 'zztop'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "zztop", "EQ"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim === 'zztop'; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension(QueryRunnerTestHelper.qualityDimension, "zztop", "NE"), + new JavaScriptDimFilter(QueryRunnerTestHelper.qualityDimension, "function(dim){ return dim != 'zztop'; }") + ); + + // null + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension("partial_null_column", "", "EQ"), + new JavaScriptDimFilter("partial_null_column", "function(dim){ return dim === null; }") + ); + validateSelectDimFilter( + builder, + new SelectorDimFilterExtension("partial_null_column", "", "NE"), + new JavaScriptDimFilter("partial_null_column", "function(dim){ return dim != null; }") + ); + } + + private void validateSelectDimFilter( + Druids.SelectQueryBuilder builder, + SelectorDimFilterExtension selectorFilter, + JavaScriptDimFilter javaScriptDimFilter + ) + { + + Iterable> expected = Sequences.toList( + runner.run(builder.filters(javaScriptDimFilter).build(), ImmutableMap.of()), + Lists.>newArrayList() + ); + Iterable> results = Sequences.toList( + runner.run(builder.filters(selectorFilter).build(), ImmutableMap.of()), + Lists.>newArrayList() + ); + SelectQueryRunnerTest.verify(expected, results); + } +} diff --git a/extensions-core/filters/src/test/java/io/druid/query/filter/TimeseriesQueryTest.java b/extensions-core/filters/src/test/java/io/druid/query/filter/TimeseriesQueryTest.java new file mode 100644 index 000000000000..7e290a295cd5 --- /dev/null +++ b/extensions-core/filters/src/test/java/io/druid/query/filter/TimeseriesQueryTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.metamx.common.guava.Sequences; +import io.druid.query.Druids; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.FilteredAggregatorFactory; +import io.druid.query.timeseries.TimeseriesQueryEngine; +import io.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; +import io.druid.query.timeseries.TimeseriesResultValue; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; + +/** + */ +@RunWith(Parameterized.class) +public class TimeseriesQueryTest +{ + public static final Map CONTEXT = ImmutableMap.of(); + + @Parameterized.Parameters(name = "{0}:descending={1}") + public static Iterable constructorFeeder() throws IOException + { + return QueryRunnerTestHelper.cartesian( + // runners + QueryRunnerTestHelper.makeQueryRunners( + new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest( + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ) + ), + // descending? + Arrays.asList(false, true) + ); + } + + private final QueryRunner runner; + private final boolean descending; + + public TimeseriesQueryTest( + QueryRunner runner, boolean descending + ) + { + this.runner = runner; + this.descending = descending; + } + + @Test + public void testTimeSeriesWithFilteredAggWithSelectDimFilter() + { + Druids.TimeseriesQueryBuilder builder = + Druids.newTimeseriesQueryBuilder() + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.allGran) + .intervals(QueryRunnerTestHelper.firstToThird) + .descending(descending); + + // existing + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, ">", "entertainment", 20); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, ">=", "entertainment", 22); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<", "entertainment", 4); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<=", "entertainment", 6); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "==", "entertainment", 2); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<>", "entertainment", 24); + + // non-existing (between) + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, ">", "financial", 20); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, ">=", "financial", 20); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<", "financial", 6); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<=", "financial", 6); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "==", "financial", 0); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<>", "financial", 26); + + // non-existing (smaller than min) + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, ">", "abcb", 26); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, ">=", "abcb", 26); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<", "abcb", 0); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<=", "abcb", 0); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "==", "abcb", 0); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<>", "abcb", 26); + + // non-existing (bigger than max) + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, ">", "zztop", 0); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, ">=", "zztop", 0); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<", "zztop", 26); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<=", "zztop", 26); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "==", "zztop", 0); + validateSelectDimFilter(builder, QueryRunnerTestHelper.qualityDimension, "<>", "zztop", 26); + + // null + validateSelectDimFilter(builder, "partial_null_column", "==", "", 22); + validateSelectDimFilter(builder, "partial_null_column", "<>", "", 4); + } + + private void validateSelectDimFilter( + Druids.TimeseriesQueryBuilder builder, + String dimension, + String operation, + String value, + long expected + ) + { + builder.aggregators( + Arrays.asList( + new FilteredAggregatorFactory( + new CountAggregatorFactory("filteredAgg"), + new SelectorDimFilterExtension(dimension, value, operation) + ) + ) + ); + Iterable> results = Sequences.toList( + runner.run(builder.build(), ImmutableMap.of()), + Lists.>newArrayList() + ); + Iterator> iterator = results.iterator(); + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(expected, iterator.next().getValue().getLongMetric("filteredAgg").longValue()); + Assert.assertFalse(iterator.hasNext()); + } +} diff --git a/pom.xml b/pom.xml index a70e710d8b58..5c1d1e394ab0 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ extensions-core/postgresql-metadata-storage extensions-core/namespace-lookup extensions-core/s3-extensions + extensions-core/filters extensions-contrib/azure-extensions extensions-contrib/cassandra-storage diff --git a/processing/src/main/java/io/druid/query/filter/DimFilterExtension.java b/processing/src/main/java/io/druid/query/filter/DimFilterExtension.java new file mode 100644 index 000000000000..70a8770d2dd5 --- /dev/null +++ b/processing/src/main/java/io/druid/query/filter/DimFilterExtension.java @@ -0,0 +1,27 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +/** + */ +public interface DimFilterExtension extends DimFilter +{ + Filter toFilter(); +} diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 93b0002016e1..3fdc30d8cf68 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -30,8 +30,8 @@ */ public class SelectorDimFilter implements DimFilter { - private final String dimension; - private final String value; + protected final String dimension; + protected final String value; @JsonCreator public SelectorDimFilter( @@ -83,7 +83,7 @@ public boolean equals(Object o) if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof SelectorDimFilter)) { return false; } diff --git a/processing/src/main/java/io/druid/query/ordering/StringComparators.java b/processing/src/main/java/io/druid/query/ordering/StringComparators.java index ecd31bc884f0..e2367ea3ec95 100644 --- a/processing/src/main/java/io/druid/query/ordering/StringComparators.java +++ b/processing/src/main/java/io/druid/query/ordering/StringComparators.java @@ -19,34 +19,47 @@ package io.druid.query.ordering; -import java.util.Comparator; - import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; +import com.google.common.base.Strings; +import com.google.common.primitives.Doubles; import com.google.common.primitives.UnsignedBytes; import com.metamx.common.IAE; import com.metamx.common.StringUtils; +import java.util.Comparator; + public class StringComparators { public static final String LEXICOGRAPHIC_NAME = "lexicographic"; public static final String ALPHANUMERIC_NAME = "alphanumeric"; - + public static final String NUMERIC_NAME = "numeric"; + public static final LexicographicComparator LEXICOGRAPHIC = new LexicographicComparator(); public static final AlphanumericComparator ALPHANUMERIC = new AlphanumericComparator(); - + public static final NumberComparator NUMERIC = new NumberComparator(); + + public static boolean validate(String compareType) + { + return compareType != null && ( + compareType.equals(LEXICOGRAPHIC_NAME) || + compareType.equals(ALPHANUMERIC_NAME) || + compareType.equals(NUMERIC_NAME)); + } + @JsonTypeInfo(use=Id.NAME, include=As.PROPERTY, property="type", defaultImpl = LexicographicComparator.class) @JsonSubTypes(value = { @JsonSubTypes.Type(name = StringComparators.LEXICOGRAPHIC_NAME, value = LexicographicComparator.class), - @JsonSubTypes.Type(name = StringComparators.ALPHANUMERIC_NAME, value = AlphanumericComparator.class) + @JsonSubTypes.Type(name = StringComparators.ALPHANUMERIC_NAME, value = AlphanumericComparator.class), + @JsonSubTypes.Type(name = StringComparators.NUMERIC_NAME, value = NumberComparator.class) }) public static interface StringComparator extends Comparator { } - + public static class LexicographicComparator implements StringComparator { @Override @@ -69,7 +82,7 @@ public int compare(String s, String s2) StringUtils.toUtf8(s2) ); } - + @Override public boolean equals(Object o) { @@ -79,17 +92,17 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - + return true; } - + @Override public String toString() { return StringComparators.LEXICOGRAPHIC_NAME; } } - + public static class AlphanumericComparator implements StringComparator { // This code is based on https://github.com/amjjd/java-alphanum, see @@ -272,7 +285,7 @@ private int compareNonNumeric(String str0, String str1, int[] pos) // compare the substrings return String.CASE_INSENSITIVE_ORDER.compare(str0.substring(start0, pos[0]), str1.substring(start1, pos[1])); } - + @Override public boolean equals(Object o) { @@ -282,10 +295,10 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - + return true; } - + @Override public String toString() { @@ -293,14 +306,64 @@ public String toString() } } + public static class NumberComparator implements StringComparator { + + @Override + public String toString() + { + return StringComparators.ALPHANUMERIC_NAME; + } + + @Override + public int compare(String o1, String o2) + { + boolean n1 = Strings.isNullOrEmpty(o1); + boolean n2 = Strings.isNullOrEmpty(o2); + if (n1 && n2) { + return 0; + } + if (!n1 && n2) { + return 1; + } + if (n1 && !n2) { + return -1; + } + boolean m1 = o1.charAt(0) == '-'; + boolean m2 = o2.charAt(0) == '-'; + if (!m1 && m2) { + return 1; + } + if (m1 && !m2) { + return -1; + } + return Doubles.compare(Double.valueOf(o1), Double.valueOf(o2)); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + return true; + } + } + public static StringComparator makeComparator(String type) { - if (type.equals(StringComparators.LEXICOGRAPHIC_NAME)) { - return LEXICOGRAPHIC; - } else if (type.equals(StringComparators.ALPHANUMERIC_NAME)) { - return ALPHANUMERIC; - } else { - throw new IAE("Unknown string comparator[%s]", type); + switch (type) { + case StringComparators.LEXICOGRAPHIC_NAME: + return LEXICOGRAPHIC; + case StringComparators.ALPHANUMERIC_NAME: + return ALPHANUMERIC; + case StringComparators.NUMERIC_NAME: + return NUMERIC; + default: + throw new IAE("Unknown string comparator[%s]", type); } } } diff --git a/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java b/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java index 81a594e2f562..623c67da818e 100644 --- a/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java @@ -20,6 +20,7 @@ package io.druid.segment.data; import java.util.Arrays; +import java.util.Comparator; import java.util.Iterator; /** @@ -27,6 +28,7 @@ public class ArrayIndexed implements Indexed { private final T[] baseArray; + private final Comparator comparator; private final Class clazz; public ArrayIndexed( @@ -35,6 +37,18 @@ public ArrayIndexed( ) { this.baseArray = baseArray; + this.comparator = null; + this.clazz = clazz; + } + + public ArrayIndexed( + T[] baseArray, + Comparator comparator, + Class clazz + ) + { + this.baseArray = baseArray; + this.comparator = comparator; this.clazz = clazz; } @@ -59,7 +73,9 @@ public T get(int index) @Override public int indexOf(T value) { - return Arrays.asList(baseArray).indexOf(value); + return comparator == null + ? Arrays.binarySearch(baseArray, value) + : Arrays.binarySearch(baseArray, value, comparator); } @Override diff --git a/processing/src/main/java/io/druid/segment/filter/Filters.java b/processing/src/main/java/io/druid/segment/filter/Filters.java index b9b4910deaf9..17f39165be95 100644 --- a/processing/src/main/java/io/druid/segment/filter/Filters.java +++ b/processing/src/main/java/io/druid/segment/filter/Filters.java @@ -24,6 +24,7 @@ import io.druid.query.filter.AndDimFilter; import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; +import io.druid.query.filter.DimFilterExtension; import io.druid.query.filter.ExtractionDimFilter; import io.druid.query.filter.Filter; import io.druid.query.filter.InDimFilter; @@ -63,6 +64,10 @@ public static Filter convertDimensionFilters(DimFilter dimFilter) return null; } + if (dimFilter instanceof DimFilterExtension) { + return ((DimFilterExtension) dimFilter).toFilter(); + } + Filter filter = null; if (dimFilter instanceof AndDimFilter) { filter = new AndFilter(convertDimensionFilters(((AndDimFilter) dimFilter).getFields())); diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index 3fc958063c86..e773e2cffd8a 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -583,7 +583,7 @@ private List> toExpected( return expected; } - private static void verify( + public static void verify( Iterable> expectedResults, Iterable> actualResults ) @@ -615,7 +615,7 @@ private static void verify( Object actVal = acHolder.getEvent().get(ex.getKey()); // work around for current II limitations - if (acHolder.getEvent().get(ex.getKey()) instanceof Double) { + if (ex.getValue() instanceof Float && actVal instanceof Double) { actVal = ((Double) actVal).floatValue(); } Assert.assertEquals(ex.getValue(), actVal);