diff --git a/NOTICE b/NOTICE index 0b11258a6038..e160033f74e3 100644 --- a/NOTICE +++ b/NOTICE @@ -10,3 +10,8 @@ This product contains a modified version of Andrew Duffy's java-alphanum library * https://github.com/amjjd/java-alphanum/blob/5c036e2e492cc7f3b7bcdebd46b8f9e2a87927e5/LICENSE.txt (Apache License, Version 2.0) * HOMEPAGE: * https://github.com/amjjd/java-alphanum + +------------------------------------------------------------------------------- + +This product contains codes from Apache hive under the Apache License, Version 2.0 + diff --git a/common/src/main/java/io/druid/cache/Cacheable.java b/common/src/main/java/io/druid/cache/Cacheable.java new file mode 100644 index 000000000000..26b8597007c7 --- /dev/null +++ b/common/src/main/java/io/druid/cache/Cacheable.java @@ -0,0 +1,27 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.cache; + +/** + */ +public interface Cacheable +{ + byte[] getCacheKey(); +} diff --git a/common/src/main/java/io/druid/collections/IterableUtils.java b/common/src/main/java/io/druid/collections/IterableUtils.java new file mode 100644 index 000000000000..291e8de33400 --- /dev/null +++ b/common/src/main/java/io/druid/collections/IterableUtils.java @@ -0,0 +1,100 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections; + +import java.lang.reflect.Array; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Iterator; + +/** + */ +public class IterableUtils +{ + // simple cartesian iterable + public static Iterable cartesian(final Iterable... iterables) + { + return cartesian(Object.class, iterables); + } + + @SafeVarargs + public static Iterable cartesian(final Class clazz, final Iterable... iterables) + { + return new Iterable() + { + @Override + public Iterator iterator() + { + return new Iterator() + { + private final Iterator[] iterators = new Iterator[iterables.length]; + + private final T[] cached = (T[]) Array.newInstance(clazz, iterables.length); + private final BitSet valid = new BitSet(iterables.length); + + @Override + public boolean hasNext() + { + return hasNext(0); + } + + private boolean hasNext(int index) + { + if (iterators[index] == null) { + iterators[index] = iterables[index].iterator(); + } + for (; hasMore(index); valid.clear(index)) { + if (index == iterables.length - 1 || hasNext(index + 1)) { + return true; + } + } + iterators[index] = null; + return false; + } + + private boolean hasMore(int index) + { + return valid.get(index) || iterators[index].hasNext(); + } + + @Override + public T[] next() + { + for (int index = 0; index < iterables.length; index++) { + if (!valid.get(index)) { + cached[index] = iterators[index].next(); + valid.set(index); + } + } + T[] result = Arrays.copyOf(cached, cached.length); + valid.clear(cached.length - 1); + return result; + } + + @Override + public void remove() + { + throw new UnsupportedOperationException("remove"); + } + }; + } + }; + } +} diff --git a/common/src/main/java/io/druid/common/utils/SerializerUtils.java b/common/src/main/java/io/druid/common/utils/SerializerUtils.java index 336cca7444ac..814a357f8273 100644 --- a/common/src/main/java/io/druid/common/utils/SerializerUtils.java +++ b/common/src/main/java/io/druid/common/utils/SerializerUtils.java @@ -22,7 +22,9 @@ import com.google.common.io.ByteStreams; import com.google.common.io.OutputSupplier; import com.google.common.primitives.Ints; +import com.metamx.common.Pair; import com.metamx.common.StringUtils; +import io.druid.cache.Cacheable; import io.druid.collections.IntList; import java.io.IOException; @@ -37,6 +39,8 @@ public class SerializerUtils { private static final Charset UTF8 = Charset.forName("UTF-8"); + private static final byte[] EMPTY_BYTES = new byte[0]; + private static final byte[][] EMPTY_BYTES_ARRAY = new byte[][] {EMPTY_BYTES}; public void writeString(T out, String name) throws IOException { @@ -72,7 +76,7 @@ public String readString(ByteBuffer in) throws IOException final int length = in.getInt(); return StringUtils.fromUtf8(readBytes(in, length)); } - + public byte[] readBytes(ByteBuffer in, int length) throws IOException { byte[] bytes = new byte[length]; @@ -80,6 +84,44 @@ public byte[] readBytes(ByteBuffer in, int length) throws IOException return bytes; } + public static Pair serializeUTFs(String... values) + { + if (values == null) { + return Pair.of(0, EMPTY_BYTES_ARRAY); + } + int totalLength = 0; + byte[][] bytes = new byte[values.length][]; + + for (int idx = 0; idx < values.length; idx++) { + bytes[idx] = StringUtils.toUtf8(values[idx]); + if (bytes[idx] == null) { + bytes[idx] = EMPTY_BYTES; + } else { + totalLength += bytes[idx].length; + } + } + return Pair.of(totalLength, bytes); + } + + public static Pair serializeUTFs(Cacheable... values) + { + if (values == null) { + return Pair.of(0, EMPTY_BYTES_ARRAY); + } + int totalLength = 0; + byte[][] bytes = new byte[values.length][]; + + for (int idx = 0; idx < values.length; idx++) { + bytes[idx] = values[idx].getCacheKey(); + if (bytes[idx] == null) { + bytes[idx] = EMPTY_BYTES; + } else { + totalLength += bytes[idx].length; + } + } + return Pair.of(totalLength, bytes); + } + public void writeStrings(OutputStream out, String[] names) throws IOException { writeStrings(out, Arrays.asList(names)); diff --git a/common/src/test/java/io/druid/collections/IterableUtilsTest.java b/common/src/test/java/io/druid/collections/IterableUtilsTest.java new file mode 100644 index 000000000000..240f4c2ac763 --- /dev/null +++ b/common/src/test/java/io/druid/collections/IterableUtilsTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + */ +public class IterableUtilsTest +{ + @Test + public void testBasic() throws Exception + { + Set[] sets = { + ImmutableSet.of("A", "B"), + ImmutableSet.of("1", "2", "3"), + ImmutableSet.of("x", "y") + }; + + Iterable> expected = Sets.cartesianProduct(Arrays.>asList(sets)); + Iterable result = IterableUtils.cartesian(String.class, sets); + for (String[] x : result) { + System.out.println("> " + Arrays.toString(x)); + } + + Iterator> e = expected.iterator(); + Iterator r = result.iterator(); + while (e.hasNext() && r.hasNext()) { + Assert.assertEquals(String.valueOf(e.next()), Arrays.toString(r.next())); + } + Assert.assertFalse(e.hasNext() || r.hasNext()); + } +} diff --git a/common/src/test/java/io/druid/common/utils/SerializerUtilsTest.java b/common/src/test/java/io/druid/common/utils/SerializerUtilsTest.java index 4215e0adeba7..9fe6c527b6ca 100644 --- a/common/src/test/java/io/druid/common/utils/SerializerUtilsTest.java +++ b/common/src/test/java/io/druid/common/utils/SerializerUtilsTest.java @@ -19,6 +19,7 @@ package io.druid.common.utils; +import com.metamx.common.Pair; import io.druid.collections.IntList; import org.junit.After; import org.junit.Assert; @@ -227,6 +228,16 @@ public void testByteBufferReadStrings() throws IOException Assert.assertArrayEquals(strings, actuals); } + @Test + public void testSerializeUTF() throws Exception { + Pair result = SerializerUtils.serializeUTFs(strings); + Assert.assertEquals(4, result.lhs.intValue()); + Assert.assertEquals(strings.length, result.rhs.length); + Assert.assertArrayEquals(strings[0].getBytes("UTF-8"), result.rhs[0]); + Assert.assertArrayEquals(strings[1].getBytes("UTF-8"), result.rhs[1]); + Assert.assertArrayEquals(strings[2].getBytes("UTF-8"), result.rhs[2]); + } + @After public void tearDown() throws IOException { diff --git a/docs/content/querying/filters.md b/docs/content/querying/filters.md index bebdd93c62ff..f03b8eceb4a4 100644 --- a/docs/content/querying/filters.md +++ b/docs/content/querying/filters.md @@ -63,18 +63,28 @@ The filter specified at field can be any other filter defined on this page. ### JavaScript filter -The JavaScript filter matches a dimension against the specified JavaScript function predicate. The filter matches values for which the function returns true. +The JavaScript filter matches dimensions against the specified JavaScript function predicate. The filter matches values for which the function returns true. -The function takes a single argument, the dimension value, and returns either true or false. +The function takes same number of arguments as the dimension values, and returns either true or false. ```json { "type" : "javascript", - "dimension" : , - "function" : "function(value) { <...> }" + "dimension" : , + "dimensions" : , + "function" : "function(value1, value2, ...) { <...> }" + "byRow" : "false" } ``` +|property|description|required?| +|--------|-----------|---------| +|type|This String should always be "javascript".|yes| +|dimension|The dimension to perform the search over.|either of dimension or dimensions| +|dimensions|The dimensions to perform the search over.|either of dimension or dimensions| +|function|A JSON function to be applied.|yes| +|byRow|If this is true, multi-valued column is handed over as single object array to function. In this case, bitmap index cannot be applied. Default: false.|no| + **Example** The following matches any dimension values for the dimension `name` between `'bar'` and `'foo'` @@ -86,6 +96,16 @@ The following matches any dimension values for the dimension `name` between `'ba } ``` +The following matches rows where the values for the given two dimensions dim1 and dim2 are the same. +```json +{ + "type" : "javascript", + "dimensions" : ["dim1","dim2"], + "function" : "function(x, y) { return x === y }", + "byRow" : "true" +} +``` + The JavaScript filter supports the use of extraction functions, see [Filtering with Extraction Functions](#filtering-with-extraction-functions) for details. diff --git a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java index a9679b7cdee9..5bd1647b5971 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java @@ -23,6 +23,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import io.druid.collections.IterableUtils; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.query.filter.ValueMatcher; @@ -32,8 +34,10 @@ import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.BooleanValueMatcher; import io.druid.segment.filter.Filters; +import io.druid.segment.incremental.IncrementalIndex; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.BitSet; import java.util.Comparator; import java.util.List; @@ -299,5 +303,42 @@ public boolean matches() } }; } + + @Override + public ValueMatcher makeValueMatcher(final String[] dimensions, final Predicate predicate) + { + final DimensionSelector[] dimValues = new DimensionSelector[dimensions.length]; + for (int i = 0; i < dimensions.length; i++) { + dimValues[i] = columnSelectorFactory.makeDimensionSelector( + new DefaultDimensionSpec(dimensions[i], dimensions[i]) + ); + } + + return new ValueMatcher() + { + @Override + public boolean matches() + { + List[] dimValuesList = new List[dimValues.length]; + for (int i = 0; i < dimValues.length; i++) { + final IndexedInts row = dimValues[i].getRow(); + if (row == null || row.size() == 0) { + dimValuesList[i] = Arrays.asList((Object) null); + } else { + dimValuesList[i] = Lists.newArrayListWithCapacity(row.size()); + for (int j = 0; j < row.size(); j++) { + dimValuesList[i].add(dimValues[i].lookupName(row.get(j))); + } + } + } + for (Object[] param : IterableUtils.cartesian(Object.class, dimValuesList)) { + if (predicate.apply(param)) { + return true; + } + } + return false; + } + }; + } } } diff --git a/processing/src/main/java/io/druid/query/extraction/ExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/ExtractionFn.java index 7a2a92f54ecc..3c66cfaa3751 100644 --- a/processing/src/main/java/io/druid/query/extraction/ExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/ExtractionFn.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.druid.cache.Cacheable; import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.RegisteredLookupExtractionFn; @@ -50,7 +51,7 @@ * regular expression with a capture group. When the regular expression matches the value of a dimension, * the value captured by the group is used for grouping operations instead of the dimension value. */ -public interface ExtractionFn +public interface ExtractionFn extends Cacheable { /** * Returns a byte[] unique to all concrete implementations of DimExtractionFn. This byte[] is used to diff --git a/processing/src/main/java/io/druid/query/extraction/ExtractionFns.java b/processing/src/main/java/io/druid/query/extraction/ExtractionFns.java new file mode 100644 index 000000000000..7f19a4f35fd4 --- /dev/null +++ b/processing/src/main/java/io/druid/query/extraction/ExtractionFns.java @@ -0,0 +1,83 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.extraction; + +import com.google.common.base.Function; +import com.google.common.base.Functions; + +import javax.annotation.Nullable; +import java.util.Arrays; + +/** + */ +public class ExtractionFns +{ + public static Function toFunction(final ExtractionFn extractionFn) + { + return new Function() + { + @Nullable + @Override + public String apply(String input) + { + return extractionFn.apply(input); + } + }; + } + + public static Function[] toFunctionsWithNull(ExtractionFn[] extractionFns, int length) + { + if (extractionFns == null) { + Function[] functions = new Function[length]; + Arrays.fill(functions, Functions.identity()); + return functions; + } + return toFunctions(extractionFns); + } + + public static Function[] toFunctions(ExtractionFn[] extractionFns) + { + final Function[] functions = new Function[extractionFns.length]; + for (int i = 0; i < functions.length; i++) { + functions[i] = extractionFns[i] != null ? toFunction(extractionFns[i]) : Functions.identity(); + } + return functions; + } + + public static Function toTransform(final ExtractionFn[] extractionFns) + { + if (extractionFns == null) { + return Functions.identity(); + } + final Function[] functions = toFunctions(extractionFns); + return new Function() + { + private final Object[] transformed = new Object[functions.length]; + @Override + public Object[] apply(Object[] input) + { + for (int i = 0; i < functions.length; i++) { + transformed[i] = functions[i].apply(input[i]); + } + return transformed; + } + }; + } +} diff --git a/processing/src/main/java/io/druid/query/filter/AndDimFilter.java b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java index de1b6025dde2..219b9517d36b 100644 --- a/processing/src/main/java/io/druid/query/filter/AndDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import io.druid.query.Druids; import io.druid.segment.filter.AndFilter; import io.druid.segment.filter.Filters; @@ -33,6 +34,11 @@ */ public class AndDimFilter implements DimFilter { + public static DimFilter of(DimFilter... filters) + { + return filters == null ? null : filters.length == 1 ? filters[0] : new AndDimFilter(Lists.newArrayList(filters)); + } + private static final Joiner AND_JOINER = Joiner.on(" && "); final private List fields; diff --git a/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java index d1dcd453704f..0ef59e1f8b1e 100644 --- a/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java @@ -165,6 +165,18 @@ public Filter toFilter() return new BoundFilter(this); } + public String toString() + { + return "BoundDimFilter{" + + "dimension='" + dimension + '\'' + + ", upper='" + upper + '\'' + + ", lower='" + lower + '\'' + + ", lowerStrict=" + lowerStrict + + ", upperStrict=" + upperStrict + + ", alphaNumeric=" + alphaNumeric + + '}'; + } + @Override public boolean equals(Object o) { diff --git a/processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java b/processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java index 706e5ce5471a..8b7c4227a334 100644 --- a/processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java @@ -37,8 +37,9 @@ public class DimFilterCacheHelper static final byte JAVASCRIPT_CACHE_ID = 0x7; static final byte SPATIAL_CACHE_ID = 0x8; static final byte IN_CACHE_ID = 0x9; + static final byte BOUND_CACHE_ID = 0xA; + public static final byte STRING_SEPARATOR = (byte) 0xFF; - public static byte BOUND_CACHE_ID = 0xA; static byte[] computeCacheKey(byte cacheIdKey, List filters) { diff --git a/processing/src/main/java/io/druid/query/filter/Filter.java b/processing/src/main/java/io/druid/query/filter/Filter.java index e63f9b22667a..3eb8c7383b8b 100644 --- a/processing/src/main/java/io/druid/query/filter/Filter.java +++ b/processing/src/main/java/io/druid/query/filter/Filter.java @@ -20,11 +20,50 @@ package io.druid.query.filter; import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.segment.ColumnSelectorFactory; + +import java.util.List; /** */ public interface Filter { - public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector); - public ValueMatcher makeMatcher(ValueMatcherFactory factory); + // this should be supported, at least + ValueMatcher makeMatcher(ValueMatcherFactory factory); + + // return true only when getBitmapIndex() is implemented + boolean supportsBitmap(); + + // optional. bitmap based filter will be applied first whenever it's possible + ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector); + + // optional. if bitmap filter cannot be supported, this should be implemented instead + ValueMatcher makeMatcher(ColumnSelectorFactory columnSelectorFactory); + + // extend this + abstract class AbstractFilter implements Filter + { + @Override + public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector) + { + throw new UnsupportedOperationException(); + } + + @Override + public ValueMatcher makeMatcher(ColumnSelectorFactory columnSelectorFactory) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean supportsBitmap() + { + return true; + } + } + + // marker for and/or/not + interface RelationalFilter { + List getChildren(); + } } diff --git a/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java index 9509b8f9445e..9cee2427fc83 100644 --- a/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java @@ -21,37 +21,69 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.metamx.common.Pair; import com.metamx.common.StringUtils; +import io.druid.common.utils.SerializerUtils; import io.druid.query.extraction.ExtractionFn; +import io.druid.segment.filter.ByRowJavaScriptFilter; import io.druid.segment.filter.JavaScriptFilter; import java.nio.ByteBuffer; +import java.util.Arrays; public class JavaScriptDimFilter implements DimFilter { - private final String dimension; + public static JavaScriptDimFilter of(String dimension, String function, ExtractionFn extractionFn) + { + return new JavaScriptDimFilter(dimension, extractionFn, function, null, null, false); + } + + public static JavaScriptDimFilter of(String[] dimensions, String function, ExtractionFn[] extractionFns) + { + return new JavaScriptDimFilter(null, null, function, dimensions, extractionFns, false); + } + + public static JavaScriptDimFilter byRow(String dimension, String function, ExtractionFn extractionFn) + { + return new JavaScriptDimFilter(dimension, extractionFn, function, null, null, true); + } + + public static JavaScriptDimFilter byRow(String[] dimensions, String function, ExtractionFn[] extractionFns) + { + return new JavaScriptDimFilter(null, null, function, dimensions, extractionFns, true); + } + + private final String[] dimensions; private final String function; - private final ExtractionFn extractionFn; + private final ExtractionFn[] extractionFns; + private final boolean byRow; @JsonCreator public JavaScriptDimFilter( + // for backwards compatibility @JsonProperty("dimension") String dimension, + @JsonProperty("extractionFn") ExtractionFn extractionFn, @JsonProperty("function") String function, - @JsonProperty("extractionFn") ExtractionFn extractionFn + @JsonProperty("dimensions") String[] dimensions, + @JsonProperty("extractionFns") ExtractionFn[] extractionFns, + @JsonProperty("byRow") boolean byRow ) { - Preconditions.checkArgument(dimension != null, "dimension must not be null"); + Preconditions.checkArgument(dimension != null ^ dimensions != null, "dimensions(xor dimension) must not be null"); Preconditions.checkArgument(function != null, "function must not be null"); - this.dimension = dimension; + this.dimensions = dimensions != null ? dimensions : new String[]{dimension}; this.function = function; - this.extractionFn = extractionFn; + this.extractionFns = dimension == null ? extractionFns + : extractionFn == null ? null : new ExtractionFn[]{extractionFn}; + this.byRow = byRow; } @JsonProperty - public String getDimension() + public String[] getDimensions() { - return dimension; + return dimensions; } @JsonProperty @@ -61,26 +93,40 @@ public String getFunction() } @JsonProperty - public ExtractionFn getExtractionFn() + public ExtractionFn[] getExtractionFn() + { + return extractionFns; + } + + public boolean isByRow() { - return extractionFn; + return byRow; } @Override public byte[] getCacheKey() { - final byte[] dimensionBytes = StringUtils.toUtf8(dimension); final byte[] functionBytes = StringUtils.toUtf8(function); - byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - - return ByteBuffer.allocate(3 + dimensionBytes.length + functionBytes.length + extractionFnBytes.length) - .put(DimFilterCacheHelper.JAVASCRIPT_CACHE_ID) - .put(dimensionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) - .put(functionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) - .put(extractionFnBytes) - .array(); + Pair extractionFnBytes = SerializerUtils.serializeUTFs(extractionFns); + Pair dimensionsBytes = SerializerUtils.serializeUTFs(dimensions); + + ByteBuffer byteBuffer = ByteBuffer.allocate( + 2 + + dimensionsBytes.rhs.length + dimensionsBytes.lhs + + functionBytes.length + 1 + + extractionFnBytes.rhs.length + extractionFnBytes.lhs + ); + byteBuffer.put(DimFilterCacheHelper.JAVASCRIPT_CACHE_ID).put(byRow ? (byte) 1 : 0); + + for (byte[] dimBytes : dimensionsBytes.rhs) { + byteBuffer.put(dimBytes).put(DimFilterCacheHelper.STRING_SEPARATOR); + } + byteBuffer.put(functionBytes).put(DimFilterCacheHelper.STRING_SEPARATOR); + + for (byte[] extractionFnByte : extractionFnBytes.rhs) { + byteBuffer.put(extractionFnByte).put(DimFilterCacheHelper.STRING_SEPARATOR); + } + return byteBuffer.array(); } @Override @@ -92,16 +138,21 @@ public DimFilter optimize() @Override public Filter toFilter() { - return new JavaScriptFilter(dimension, function, extractionFn); + if (byRow) { + return new ByRowJavaScriptFilter(dimensions, function, extractionFns); + } else { + return new JavaScriptFilter(dimensions, function, extractionFns); + } } @Override public String toString() { return "JavaScriptDimFilter{" + - "dimension='" + dimension + '\'' + + "dimensions=['" + Joiner.on("', '").join(dimensions) + "']" + ", function='" + function + '\'' + - ", extractionFn='" + extractionFn + '\'' + + (extractionFns == null ? "" : ", extractionFns='" + Joiner.on("', '").join(extractionFns) + '\'') + + ", byRow='" + byRow + '\'' + '}'; } @@ -117,22 +168,25 @@ public boolean equals(Object o) JavaScriptDimFilter that = (JavaScriptDimFilter) o; - if (!dimension.equals(that.dimension)) { + if (!Arrays.equals(dimensions, that.dimensions)) { + return false; + } + if (byRow ^ that.byRow) { return false; } if (!function.equals(that.function)) { return false; } - return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; - + return Arrays.equals(extractionFns, that.extractionFns); } @Override public int hashCode() { - int result = dimension.hashCode(); + int result = Arrays.hashCode(dimensions); + result = 31 * result + (byRow ? 0 : 1); result = 31 * result + function.hashCode(); - result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0); + result = 31 * result + Arrays.hashCode(extractionFns); return result; } } diff --git a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java index 32cbb658baec..2010a443375a 100644 --- a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java @@ -98,4 +98,8 @@ public String toString() { return "!" + field; } + + public static DimFilter of(DimFilter filter) { + return filter == null ? null : new NotDimFilter(filter); + } } diff --git a/processing/src/main/java/io/druid/query/filter/OrDimFilter.java b/processing/src/main/java/io/druid/query/filter/OrDimFilter.java index 49b7b77f7f18..168d797bca7e 100644 --- a/processing/src/main/java/io/druid/query/filter/OrDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/OrDimFilter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import io.druid.query.Druids; import io.druid.segment.filter.Filters; import io.druid.segment.filter.OrFilter; @@ -33,6 +34,11 @@ */ public class OrDimFilter implements DimFilter { + public static DimFilter of(DimFilter... filters) + { + return filters == null ? null : filters.length == 1 ? filters[0] : new OrDimFilter(Lists.newArrayList(filters)); + } + private static final Joiner OR_JOINER = Joiner.on(" || "); final private List fields; diff --git a/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java b/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java index 3da61be99b1f..972da0147a61 100644 --- a/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java +++ b/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java @@ -27,4 +27,5 @@ public interface ValueMatcherFactory { public ValueMatcher makeValueMatcher(String dimension, Comparable value); public ValueMatcher makeValueMatcher(String dimension, Predicate predicate); + public ValueMatcher makeValueMatcher(String[] dimension, Predicate value); } diff --git a/processing/src/main/java/io/druid/segment/BitmapOffset.java b/processing/src/main/java/io/druid/segment/BitmapOffset.java index 47ec7db72770..95be1fb53050 100644 --- a/processing/src/main/java/io/druid/segment/BitmapOffset.java +++ b/processing/src/main/java/io/druid/segment/BitmapOffset.java @@ -77,12 +77,14 @@ private BitmapOffset(BitmapOffset otherOffset) } @Override - public void increment() + public boolean increment() { if (itr.hasNext()) { val = itr.next(); + return true; } else { val = INVALID_VALUE; + return false; } } diff --git a/processing/src/main/java/io/druid/segment/IndexedIntsOffset.java b/processing/src/main/java/io/druid/segment/IndexedIntsOffset.java index af0cc824cce4..060c0a7ad961 100644 --- a/processing/src/main/java/io/druid/segment/IndexedIntsOffset.java +++ b/processing/src/main/java/io/druid/segment/IndexedIntsOffset.java @@ -28,23 +28,25 @@ class IndexedIntsOffset implements Offset { int currRow; private final IndexedInts invertedIndex; + private final int limit; public IndexedIntsOffset(IndexedInts invertedIndex) { this.invertedIndex = invertedIndex; + this.limit = invertedIndex.size(); currRow = 0; } @Override - public void increment() + public boolean increment() { - ++currRow; + return ++currRow < limit; } @Override public boolean withinBounds() { - return currRow < invertedIndex.size(); + return currRow < limit; } @Override diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index fbbaf27dc977..fccff3a0fb63 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -21,7 +21,6 @@ import com.google.common.base.Function; import com.google.common.base.Predicates; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; @@ -35,6 +34,7 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; @@ -45,6 +45,8 @@ import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.Offset; +import io.druid.segment.filter.BooleanValueMatcher; +import io.druid.segment.filter.Filters; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -213,8 +215,13 @@ public Sequence makeCursors(Filter filter, Interval interval, QueryGranu actualInterval = actualInterval.withEnd(dataInterval.getEnd()); } + final Filter[] filters = Filters.partitionWithBitmapSupport(filter); + + final Filter bitmapFilter = filters == null ? null : filters[0]; + final Filter valuesFilter = filters == null ? null : filters[1]; + final Offset offset; - if (filter == null) { + if (bitmapFilter == null) { offset = new NoFilterOffset(0, index.getNumRows(), descending); } else { final ColumnSelectorBitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( @@ -222,7 +229,7 @@ public Sequence makeCursors(Filter filter, Interval interval, QueryGranu index ); - offset = new BitmapOffset(selector.getBitmapFactory(), filter.getBitmapIndex(selector), descending); + offset = new BitmapOffset(selector.getBitmapFactory(), bitmapFilter.getBitmapIndex(selector), descending); } return Sequences.filter( @@ -231,6 +238,7 @@ public Sequence makeCursors(Filter filter, Interval interval, QueryGranu actualInterval, gran, offset, + valuesFilter, minDataTimestamp, maxDataTimestamp, descending @@ -249,11 +257,14 @@ private static class CursorSequenceBuilder private final long maxDataTimestamp; private final boolean descending; + private final Filter filter; + public CursorSequenceBuilder( ColumnSelector index, Interval interval, QueryGranularity gran, Offset offset, + Filter filter, long minDataTimestamp, long maxDataTimestamp, boolean descending @@ -263,6 +274,7 @@ public CursorSequenceBuilder( this.interval = interval; this.gran = gran; this.offset = offset; + this.filter = filter; this.minDataTimestamp = minDataTimestamp; this.maxDataTimestamp = maxDataTimestamp; this.descending = descending; @@ -327,8 +339,15 @@ public Cursor apply(final Long input) { private final Offset initOffset = offset.clone(); private final DateTime myBucket = gran.toDateTime(input); + private final ValueMatcher filterMatcher = + filter == null ? BooleanValueMatcher.TRUE : filter.makeMatcher(this); private Offset cursorOffset = offset; + { + while (!filterMatcher.matches() && cursorOffset.increment()) { + } + } + @Override public DateTime getTime() { @@ -341,7 +360,9 @@ public void advance() if (Thread.interrupted()) { throw new QueryInterruptedException(new InterruptedException()); } - cursorOffset.increment(); + + while (cursorOffset.increment() && !filterMatcher.matches()) { + } } @Override @@ -802,9 +823,9 @@ public boolean withinBounds() protected abstract boolean timeInRange(long current); @Override - public void increment() + public boolean increment() { - baseOffset.increment(); + return baseOffset.increment(); } @Override @@ -892,9 +913,9 @@ private static class NoFilterOffset implements Offset } @Override - public void increment() + public boolean increment() { - currentOffset++; + return ++currentOffset < rowCount; } @Override diff --git a/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java b/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java index ffcd333cb9c1..8e7ccfdf63c7 100644 --- a/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java @@ -49,9 +49,9 @@ public int getOffset() } @Override - public void increment() + public boolean increment() { - ++currIndex; + return ++currIndex < ints.length; } @Override diff --git a/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java b/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java index 3db91fe22b77..3d5c08b91f85 100644 --- a/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java +++ b/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java @@ -21,7 +21,8 @@ /** */ -public class IntersectingOffset implements Offset { +public class IntersectingOffset implements Offset +{ private final Offset lhs; private final Offset rhs; @@ -33,26 +34,28 @@ public IntersectingOffset( this.lhs = lhs; this.rhs = rhs; - findIntersection(); + findIntersection(lhs.withinBounds(), rhs.withinBounds()); } @Override - public int getOffset() { + public int getOffset() + { return lhs.getOffset(); } @Override - public void increment() { - lhs.increment(); - rhs.increment(); + public boolean increment() + { + final boolean lhsInBound = lhs.increment(); + final boolean rhsInBound = rhs.increment(); - findIntersection(); + return findIntersection(lhsInBound, rhsInBound); } - private void findIntersection() + private boolean findIntersection(boolean lhsInBound, boolean rhsInBound) { - if (! (lhs.withinBounds() && rhs.withinBounds())) { - return; + if (!(lhsInBound && rhsInBound)) { + return false; } int lhsOffset = lhs.getOffset(); @@ -61,8 +64,8 @@ private void findIntersection() while (lhsOffset != rhsOffset) { while (lhsOffset < rhsOffset) { lhs.increment(); - if (! lhs.withinBounds()) { - return; + if (!lhs.withinBounds()) { + return false; } lhsOffset = lhs.getOffset(); @@ -70,17 +73,19 @@ private void findIntersection() while (rhsOffset < lhsOffset) { rhs.increment(); - if (! rhs.withinBounds()) { - return; + if (!rhs.withinBounds()) { + return false; } rhsOffset = rhs.getOffset(); } } + return true; } @Override - public boolean withinBounds() { + public boolean withinBounds() + { return lhs.withinBounds() && rhs.withinBounds(); } diff --git a/processing/src/main/java/io/druid/segment/data/Offset.java b/processing/src/main/java/io/druid/segment/data/Offset.java index e6421e4c0f08..5d3f2492fc22 100644 --- a/processing/src/main/java/io/druid/segment/data/Offset.java +++ b/processing/src/main/java/io/druid/segment/data/Offset.java @@ -25,7 +25,7 @@ */ public interface Offset extends ReadableOffset { - void increment(); + boolean increment(); boolean withinBounds(); diff --git a/processing/src/main/java/io/druid/segment/data/StartLimitedOffset.java b/processing/src/main/java/io/druid/segment/data/StartLimitedOffset.java index 0d8203765c8c..1769b3a0f13a 100644 --- a/processing/src/main/java/io/druid/segment/data/StartLimitedOffset.java +++ b/processing/src/main/java/io/druid/segment/data/StartLimitedOffset.java @@ -40,9 +40,9 @@ public StartLimitedOffset( } @Override - public void increment() + public boolean increment() { - baseOffset.increment(); + return baseOffset.increment(); } @Override diff --git a/processing/src/main/java/io/druid/segment/data/UnioningOffset.java b/processing/src/main/java/io/druid/segment/data/UnioningOffset.java index 51fd4af2ef20..06f44f85eaee 100644 --- a/processing/src/main/java/io/druid/segment/data/UnioningOffset.java +++ b/processing/src/main/java/io/druid/segment/data/UnioningOffset.java @@ -66,7 +66,7 @@ private UnioningOffset( this.nextOffsetIndex = nextOffsetIndex; } - private void figureOutNextValue() { + private boolean figureOutNextValue() { if (offsets[0] != null) { if (offsets[1] != null) { int lhs = offsetVals[0]; @@ -85,7 +85,9 @@ private void figureOutNextValue() { else { nextOffsetIndex = 0; } + return true; } + return false; } private void rollIndexForward(int i) { @@ -110,10 +112,10 @@ public int getOffset() } @Override - public void increment() + public boolean increment() { rollIndexForward(nextOffsetIndex); - figureOutNextValue(); + return figureOutNextValue(); } @Override diff --git a/processing/src/main/java/io/druid/segment/filter/AndFilter.java b/processing/src/main/java/io/druid/segment/filter/AndFilter.java index 085f86d1c21e..b4ef550a1b89 100644 --- a/processing/src/main/java/io/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/AndFilter.java @@ -26,12 +26,18 @@ import io.druid.query.filter.ValueMatcher; import io.druid.query.filter.ValueMatcherFactory; +import java.util.Arrays; import java.util.List; /** */ -public class AndFilter implements Filter +public class AndFilter extends Filter.AbstractFilter implements Filter.RelationalFilter { + public static Filter of(Filter... filters) + { + return filters == null ? null : filters.length == 1 ? filters[0] : new AndFilter(Arrays.asList(filters)); + } + private final List filters; public AndFilter( @@ -60,7 +66,7 @@ public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector) public ValueMatcher makeMatcher(ValueMatcherFactory factory) { if (filters.size() == 0) { - return new BooleanValueMatcher(false); + return BooleanValueMatcher.FALSE; } final ValueMatcher[] matchers = new ValueMatcher[filters.size()]; @@ -91,4 +97,27 @@ public boolean matches() } }; } + + @Override + public boolean supportsBitmap() + { + for (Filter child : filters) { + if (!child.supportsBitmap()) { + return false; + } + } + return true; + } + + @Override + public List getChildren() + { + return filters; + } + + @Override + public String toString() + { + return "AND " + filters; + } } diff --git a/processing/src/main/java/io/druid/segment/filter/BooleanValueMatcher.java b/processing/src/main/java/io/druid/segment/filter/BooleanValueMatcher.java index a34dadf01323..b026c8cd0fcd 100644 --- a/processing/src/main/java/io/druid/segment/filter/BooleanValueMatcher.java +++ b/processing/src/main/java/io/druid/segment/filter/BooleanValueMatcher.java @@ -36,4 +36,8 @@ public boolean matches() { return matches; } + + public static final BooleanValueMatcher TRUE = new BooleanValueMatcher(true); + + public static final BooleanValueMatcher FALSE = new BooleanValueMatcher(false); } diff --git a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java index b70093b037c6..0d30af4e4f29 100644 --- a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java @@ -34,7 +34,7 @@ import java.util.Comparator; import java.util.Iterator; -public class BoundFilter implements Filter +public class BoundFilter extends Filter.AbstractFilter { private final BoundDimFilter boundDimFilter; private final Comparator comparator; @@ -182,6 +182,11 @@ public boolean apply(String input) { return doesMatch(input); } + + @Override + public String toString() { + return boundDimFilter.toString(); + } } ); } diff --git a/processing/src/main/java/io/druid/segment/filter/ByRowJavaScriptFilter.java b/processing/src/main/java/io/druid/segment/filter/ByRowJavaScriptFilter.java new file mode 100644 index 000000000000..e257d15ed122 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/filter/ByRowJavaScriptFilter.java @@ -0,0 +1,149 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.filter; + +import com.google.common.base.Function; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.extraction.ExtractionFn; +import io.druid.query.extraction.ExtractionFns; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.data.IndexedInts; +import org.mozilla.javascript.Context; + +import java.util.Arrays; + +/** + */ +public class ByRowJavaScriptFilter extends Filter.AbstractFilter +{ + private final String[] dimensions; + private final String script; + private final ExtractionFn[] extractionFns; + + public ByRowJavaScriptFilter(String[] dimension, String script, ExtractionFn[] extractionFns) + { + this.dimensions = dimension; + this.script = script; + this.extractionFns = extractionFns; + } + + @Override + public ValueMatcher makeMatcher(ValueMatcherFactory factory) + { + final Function extractor = ExtractionFns.toTransform(extractionFns); + return factory.makeValueMatcher( + dimensions, new JavaScriptPredicate(script) + { + @Override + final boolean applyInContext(Context cx, Object[] input) + { + return Context.toBoolean(fnApply.call(cx, scope, scope, extractor.apply(input))); + } + } + ); + } + + @Override + public boolean supportsBitmap() + { + return false; + } + + @Override + public ValueMatcher makeMatcher(ColumnSelectorFactory factory) + { + final Function[] extractors = ExtractionFns.toFunctionsWithNull(extractionFns, dimensions.length); + final JavaScriptPredicate predicate = new JavaScriptPredicate(script) + { + @Override + final boolean applyInContext(Context cx, Object[] input) + { + return Context.toBoolean(fnApply.call(cx, scope, scope, input)); + } + }; + final DimensionSelector[] selectors = new DimensionSelector[dimensions.length]; + for (int i = 0; i < dimensions.length; i++) { + selectors[i] = factory.makeDimensionSelector(new DefaultDimensionSpec(dimensions[i], dimensions[i])); + if (selectors[i] == null) { + return BooleanValueMatcher.FALSE; + } + } + return new ValueMatcher() + { + private final Object[] args = new Object[selectors.length]; + + @Override + public boolean matches() + { + for (int i = 0; i < selectors.length; i++) { + IndexedInts row = selectors[i].getRow(); + if (row.size() == 1) { + args[i] = extractors[i].apply(selectors[i].lookupName(row.get(0))); + } else { + String[] array = new String[row.size()]; + for (int j = 0; j < array.length; j++) { + array[j] = extractors[i].apply(selectors[i].lookupName(row.get(j))); + } + args[i] = Context.javaToJS(array, predicate.scope); + } + } + return predicate.apply(args); + } + }; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ByRowJavaScriptFilter that = (ByRowJavaScriptFilter) o; + + if (!Arrays.equals(dimensions, that.dimensions)) { + return false; + } + if (!Arrays.equals(extractionFns, that.extractionFns)) { + return false; + } + if (!script.equals(that.script)) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = Arrays.hashCode(dimensions); + result = 31 * result + script.hashCode(); + result = 31 * result + (extractionFns != null ? Arrays.hashCode(extractionFns) : 0); + return result; + } +} diff --git a/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java b/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java index debdd2a50d2b..3ba8b4eaf2be 100644 --- a/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java @@ -33,7 +33,7 @@ /** */ -public class DimensionPredicateFilter implements Filter +public class DimensionPredicateFilter extends Filter.AbstractFilter { private final String dimension; private final Predicate predicate; diff --git a/processing/src/main/java/io/druid/segment/filter/Filters.java b/processing/src/main/java/io/druid/segment/filter/Filters.java index 67f2838d20ec..3999d39eeaeb 100644 --- a/processing/src/main/java/io/druid/segment/filter/Filters.java +++ b/processing/src/main/java/io/druid/segment/filter/Filters.java @@ -20,11 +20,14 @@ package io.druid.segment.filter; import com.google.common.base.Function; +import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.metamx.common.guava.FunctionalIterable; import io.druid.query.filter.DimFilter; import io.druid.query.filter.Filter; +import java.util.ArrayList; import java.util.List; /** @@ -67,4 +70,204 @@ public static Filter toFilter(DimFilter dimFilter) { return dimFilter == null ? null : dimFilter.toFilter(); } + + public static Filter[] partitionWithBitmapSupport(Filter current) + { + return partitionFilterWith( + current, new Predicate() + { + @Override + public boolean apply(Filter input) + { + return input.supportsBitmap(); + } + } + ); + } + + private static Filter[] partitionFilterWith(Filter current, Predicate predicate) + { + if (current == null) { + return null; + } + current = pushDownNot(current); + current = flatten(current); + current = convertToCNF(current); + current = flatten(current); + + List bitmapIndexSupported = Lists.newArrayList(); + List bitmapIndexNotSupported = Lists.newArrayList(); + + traverse(current, predicate, bitmapIndexSupported, bitmapIndexNotSupported); + + return new Filter[]{andFilter(bitmapIndexSupported), andFilter(bitmapIndexNotSupported)}; + } + + private static void traverse( + Filter current, + Predicate predicate, + List support, + List notSupport + ) + { + if (current instanceof AndFilter) { + for (Filter child : ((AndFilter) current).getChildren()) { + traverse(child, predicate, support, notSupport); + } + } else { + if (predicate.apply(current)) { + support.add(current); + } else { + notSupport.add(current); + } + } + } + + private static Filter andFilter(List filters) { + return filters.isEmpty() ? null : filters.size() == 1 ? filters.get(0) : new AndFilter(filters); + } + + // copied from apache hive + private static Filter pushDownNot(Filter current) + { + if (current instanceof NotFilter) { + Filter child = ((NotFilter) current).getBaseFilter(); + if (child instanceof NotFilter) { + return pushDownNot(((NotFilter) child).getBaseFilter()); + } + if (child instanceof AndFilter) { + List children = Lists.newArrayList(); + for (Filter grandChild : ((AndFilter) child).getChildren()) { + children.add(pushDownNot(new NotFilter(grandChild))); + } + return new OrFilter(children); + } + if (child instanceof OrFilter) { + List children = Lists.newArrayList(); + for (Filter grandChild : ((OrFilter) child).getChildren()) { + children.add(pushDownNot(new NotFilter(grandChild))); + } + return new AndFilter(children); + } + } + if (current instanceof AndFilter) { + List children = Lists.newArrayList(); + for (Filter child : ((AndFilter) current).getChildren()) { + children.add(pushDownNot(child)); + } + return new AndFilter(children); + } + if (current instanceof OrFilter) { + List children = Lists.newArrayList(); + for (Filter child : ((OrFilter) current).getChildren()) { + children.add(pushDownNot(child)); + } + return new OrFilter(children); + } + return current; + } + + // copied from apache hive + private static Filter convertToCNF(Filter current) + { + if (current instanceof NotFilter) { + return new NotFilter(convertToCNF(((NotFilter) current).getBaseFilter())); + } + if (current instanceof AndFilter) { + List children = Lists.newArrayList(); + for (Filter child : ((AndFilter) current).getChildren()) { + children.add(convertToCNF(child)); + } + return new AndFilter(children); + } + if (current instanceof OrFilter) { + // a list of leaves that weren't under AND expressions + List nonAndList = new ArrayList(); + // a list of AND expressions that we need to distribute + List andList = new ArrayList(); + for (Filter child : ((OrFilter) current).getChildren()) { + if (child instanceof AndFilter) { + andList.add(child); + } else if (child instanceof OrFilter) { + // pull apart the kids of the OR expression + for (Filter grandChild : ((OrFilter) child).getChildren()) { + nonAndList.add(grandChild); + } + } else { + nonAndList.add(child); + } + } + if (!andList.isEmpty()) { + List result = Lists.newArrayList(); + generateAllCombinations(result, andList, nonAndList); + return new AndFilter(result); + } + } + return current; + } + + private static Filter flatten(Filter root) { + if (root instanceof Filter.RelationalFilter) { + List children = ((Filter.RelationalFilter)root).getChildren(); + // iterate through the index, so that if we add more children, + // they don't get re-visited + for (int i = 0; i < children.size(); ++i) { + Filter child = flatten(children.get(i)); + // do we need to flatten? + if (child.getClass() == root.getClass() && !(child instanceof NotFilter)) { + boolean first = true; + List grandKids = ((Filter.RelationalFilter)child).getChildren(); + for (Filter grandkid : grandKids) { + // for the first grandkid replace the original parent + if (first) { + first = false; + children.set(i, grandkid); + } else { + children.add(++i, grandkid); + } + } + } else { + children.set(i, child); + } + } + // if we have a singleton AND or OR, just return the child + if (children.size() == 1 && (root instanceof AndFilter || root instanceof OrFilter)) { + return children.get(0); + } + } + return root; + } + + // copied from apache hive + private static void generateAllCombinations( + List result, + List andList, + List nonAndList + ) + { + List children = ((AndFilter) andList.get(0)).getChildren(); + if (result.isEmpty()) { + for (Filter child : children) { + List a = Lists.newArrayList(nonAndList); + a.add(child); + result.add(new OrFilter(a)); + } + } else { + List work = new ArrayList(result); + result.clear(); + for (Filter child : children) { + for (Filter or : work) { + List a = Lists.newArrayList((((OrFilter) or).getChildren())); + a.add(child); + result.add(new OrFilter(a)); + } + } + } + if (andList.size() > 1) { + generateAllCombinations( + result, andList.subList(1, andList.size()), + nonAndList + ); + } + } } diff --git a/processing/src/main/java/io/druid/segment/filter/InFilter.java b/processing/src/main/java/io/druid/segment/filter/InFilter.java index 7af0bdee58bd..dcc03477c16d 100644 --- a/processing/src/main/java/io/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/InFilter.java @@ -24,22 +24,19 @@ import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.metamx.collections.bitmap.BitmapFactory; import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.Filter; import io.druid.query.filter.ValueMatcher; import io.druid.query.filter.ValueMatcherFactory; -import javax.annotation.Nullable; import java.util.List; import java.util.Set; /** */ -public class InFilter implements Filter +public class InFilter extends Filter.AbstractFilter { private final String dimension; private final Set values; diff --git a/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java b/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java index f4d16b829f63..daa2fe5de5cc 100644 --- a/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java @@ -19,155 +19,84 @@ package io.druid.segment.filter; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; +import com.google.common.base.Function; import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.common.guava.FunctionalIterable; +import io.druid.collections.IterableUtils; import io.druid.query.extraction.ExtractionFn; +import io.druid.query.extraction.ExtractionFns; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.Filter; import io.druid.query.filter.ValueMatcher; import io.druid.query.filter.ValueMatcherFactory; import io.druid.segment.data.Indexed; import org.mozilla.javascript.Context; -import org.mozilla.javascript.Function; -import org.mozilla.javascript.ScriptableObject; -import javax.annotation.Nullable; +import java.util.Arrays; -public class JavaScriptFilter implements Filter +public class JavaScriptFilter extends Filter.AbstractFilter { - private final JavaScriptPredicate predicate; - private final String dimension; - private final ExtractionFn extractionFn; + private static final Iterable EMPTY_STR_DIM_VAL = Arrays.asList((String) null); - public JavaScriptFilter(String dimension, final String script, ExtractionFn extractionFn) - { - this.dimension = dimension; - this.predicate = new JavaScriptPredicate(script, extractionFn); - this.extractionFn = extractionFn; - } + private final JavaScriptPredicate predicate; + private final Function extractor; + private final String[] dimensions; - @Override - public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector) + public JavaScriptFilter(String[] dimensions, String script, ExtractionFn[] extractionFns) { - final Context cx = Context.enter(); - try { - final Indexed dimValues = selector.getDimensionValues(dimension); - ImmutableBitmap bitmap; - if (dimValues == null || dimValues.size() == 0) { - bitmap = selector.getBitmapFactory().makeEmptyImmutableBitmap(); - if (predicate.applyInContext(cx, null)) { - bitmap = selector.getBitmapFactory().complement(bitmap, selector.getNumRows()); - } - } else { - bitmap = selector.getBitmapFactory().union( - FunctionalIterable.create(dimValues) - .filter( - new Predicate() - { - @Override - public boolean apply(@Nullable String input) - { - return predicate.applyInContext(cx, input); - } - } - ) - .transform( - new com.google.common.base.Function() - { - @Override - public ImmutableBitmap apply(@Nullable String input) - { - return selector.getBitmapIndex(dimension, input); - } - } - ) - ); + this.dimensions = dimensions; + this.extractor = ExtractionFns.toTransform(extractionFns); + this.predicate = new JavaScriptPredicate(script) { + @Override + final boolean applyInContext(Context cx, Object[] input) + { + return Context.toBoolean(fnApply.call(cx, scope, scope, extractor.apply(input))); } - return bitmap; - } - finally { - Context.exit(); - } + }; } @Override public ValueMatcher makeMatcher(ValueMatcherFactory factory) { // suboptimal, since we need create one context per call to predicate.apply() - return factory.makeValueMatcher(dimension, predicate); + return factory.makeValueMatcher(dimensions, predicate); } - static class JavaScriptPredicate implements Predicate + @Override + public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector) { - final ScriptableObject scope; - final Function fnApply; - final String script; - final ExtractionFn extractionFn; - - public JavaScriptPredicate(final String script, final ExtractionFn extractionFn) - { - Preconditions.checkNotNull(script, "script must not be null"); - this.script = script; - this.extractionFn = extractionFn; - - final Context cx = Context.enter(); - try { - cx.setOptimizationLevel(9); - scope = cx.initStandardObjects(); - - fnApply = cx.compileFunction(scope, script, "script", 1, null); - } - finally { - Context.exit(); - } - } - - @Override - public boolean apply(final String input) - { - // one and only one context per thread - final Context cx = Context.enter(); - try { - return applyInContext(cx, input); - } - finally { - Context.exit(); + Iterable[] dimValuesList = new Iterable[dimensions.length]; + for (int idx = 0; idx < dimensions.length; idx++) { + Indexed dimValues = selector.getDimensionValues(dimensions[idx]); + if (dimValues == null || dimValues.size() == 0) { + dimValuesList[idx] = EMPTY_STR_DIM_VAL; + } else { + dimValuesList[idx] = dimValues; } } - public boolean applyInContext(Context cx, String input) - { - if (extractionFn != null) { - input = extractionFn.apply(input); - } - return Context.toBoolean(fnApply.call(cx, scope, scope, new String[]{input})); - } + ImmutableBitmap bitmap = selector.getBitmapFactory().makeEmptyImmutableBitmap(); - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - JavaScriptPredicate that = (JavaScriptPredicate) o; - - if (!script.equals(that.script)) { - return false; + final Context cx = Context.enter(); + try { + // for dim X with [a,b,c] + dim Y with [1,2,3], we make cartesian of X and Y, which is in this case + // {(a,1), (a,2), (a,3), (b,1), (b,2), (b,3), (c,1), (c,2), (c,3)} + // and apply filter to them and find all possible rows that matched + // This can be very costly if cardinality of dimension is high. For that case it would be safer + // to use `byRow=true`, which applies filter by scanning by-row basis. + for (String[] param : IterableUtils.cartesian(String.class, dimValuesList)) { + if (predicate.applyInContext(cx, param)) { + ImmutableBitmap overlap = null; + for (int idx = 0; idx < dimensions.length; idx++) { + ImmutableBitmap dimBitMap = selector.getBitmapIndex(dimensions[idx], param[idx]); + overlap = overlap == null ? dimBitMap : overlap.intersection(dimBitMap); + } + bitmap = bitmap.union(overlap); + } } - - return true; + return bitmap; } - - @Override - public int hashCode() - { - return script.hashCode(); + finally { + Context.exit(); } } } diff --git a/processing/src/main/java/io/druid/segment/filter/JavaScriptPredicate.java b/processing/src/main/java/io/druid/segment/filter/JavaScriptPredicate.java new file mode 100644 index 000000000000..b71598600a20 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/filter/JavaScriptPredicate.java @@ -0,0 +1,99 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.filter; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import org.mozilla.javascript.Context; +import org.mozilla.javascript.ScriptableObject; + +/** + */ +abstract class JavaScriptPredicate implements Predicate +{ + final ScriptableObject scope; + final org.mozilla.javascript.Function fnApply; + final String script; + + public JavaScriptPredicate(final String script) + { + Preconditions.checkNotNull(script, "script must not be null"); + this.script = script; + + final Context cx = Context.enter(); + try { + cx.setOptimizationLevel(9); + scope = cx.initStandardObjects(); + + fnApply = cx.compileFunction(scope, script, "script", 1, null); + } + finally { + Context.exit(); + } + } + + @Override + public boolean apply(final Object[] input) + { + // one and only one context per thread + final Context cx = Context.enter(); + try { + return applyInContext(cx, input); + } + finally { + Context.exit(); + } + } + + abstract boolean applyInContext(Context cx, Object[] input); + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + JavaScriptPredicate that = (JavaScriptPredicate) o; + + if (!script.equals(that.script)) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + return script.hashCode(); + } + + @Override + public String toString() + { + return "JavaScriptPredicate{" + + "script='" + script + '\'' + + '}'; + } +} diff --git a/processing/src/main/java/io/druid/segment/filter/NotFilter.java b/processing/src/main/java/io/druid/segment/filter/NotFilter.java index 970e19eacb99..44bbf989d5db 100644 --- a/processing/src/main/java/io/druid/segment/filter/NotFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/NotFilter.java @@ -24,11 +24,21 @@ import io.druid.query.filter.Filter; import io.druid.query.filter.ValueMatcher; import io.druid.query.filter.ValueMatcherFactory; +import io.druid.segment.ColumnSelectorFactory; + +import java.util.Arrays; +import java.util.List; /** */ -public class NotFilter implements Filter +public class NotFilter extends Filter.AbstractFilter implements Filter.RelationalFilter { + + public static Filter of(Filter filter) + { + return filter == null ? null : new NotFilter(filter); + } + private final Filter baseFilter; public NotFilter( @@ -38,6 +48,11 @@ public NotFilter( this.baseFilter = baseFilter; } + public Filter getBaseFilter() + { + return baseFilter; + } + @Override public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector) { @@ -61,4 +76,37 @@ public boolean matches() } }; } + + @Override + public ValueMatcher makeMatcher(ColumnSelectorFactory factory) + { + final ValueMatcher baseMatcher = baseFilter.makeMatcher(factory); + + return new ValueMatcher() + { + @Override + public boolean matches() + { + return !baseMatcher.matches(); + } + }; + } + + @Override + public boolean supportsBitmap() + { + return baseFilter.supportsBitmap(); + } + + @Override + public List getChildren() + { + return Arrays.asList(baseFilter); + } + + @Override + public String toString() + { + return "NOT " + baseFilter; + } } diff --git a/processing/src/main/java/io/druid/segment/filter/OrFilter.java b/processing/src/main/java/io/druid/segment/filter/OrFilter.java index 84a7aa6034df..b88552eec797 100644 --- a/processing/src/main/java/io/druid/segment/filter/OrFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/OrFilter.java @@ -25,13 +25,20 @@ import io.druid.query.filter.Filter; import io.druid.query.filter.ValueMatcher; import io.druid.query.filter.ValueMatcherFactory; +import io.druid.segment.ColumnSelectorFactory; +import java.util.Arrays; import java.util.List; /** */ -public class OrFilter implements Filter +public class OrFilter extends Filter.AbstractFilter implements Filter.RelationalFilter { + public static Filter of(Filter... filters) + { + return filters == null ? null : filters.length == 1 ? filters[0] : new OrFilter(Arrays.asList(filters)); + } + private final List filters; public OrFilter( @@ -71,7 +78,18 @@ public ValueMatcher makeMatcher(ValueMatcherFactory factory) return makeMatcher(matchers); } - private ValueMatcher makeMatcher(final ValueMatcher[] baseMatchers){ + public ValueMatcher makeMatcher(ColumnSelectorFactory factory) + { + final ValueMatcher[] matchers = new ValueMatcher[filters.size()]; + + for (int i = 0; i < filters.size(); i++) { + matchers[i] = filters.get(i).makeMatcher(factory); + } + return makeMatcher(matchers); + } + + private ValueMatcher makeMatcher(final ValueMatcher[] baseMatchers) + { if (baseMatchers.length == 1) { return baseMatchers[0]; } @@ -91,4 +109,25 @@ public boolean matches() }; } + @Override + public boolean supportsBitmap() + { + for (Filter child : filters) { + if (!child.supportsBitmap()) { + return false; + } + } + return true; + } + + @Override + public List getChildren() + { + return filters; + } + + @Override + public String toString() { + return "OR " + filters; + } } diff --git a/processing/src/main/java/io/druid/segment/filter/RegexFilter.java b/processing/src/main/java/io/druid/segment/filter/RegexFilter.java index 02a267e05977..9c29e1a713fa 100644 --- a/processing/src/main/java/io/druid/segment/filter/RegexFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/RegexFilter.java @@ -45,6 +45,12 @@ public boolean apply(String input) { return (input != null) && compiled.matcher(input).find(); } + + @Override + public String toString() + { + return pattern; + } }, extractionFn ); diff --git a/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java b/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java index 208d5729d186..ca56eeac6ca7 100644 --- a/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java @@ -47,6 +47,12 @@ public boolean apply(@Nullable String input) { return query.accept(input); } + + @Override + public String toString() + { + return query.toString(); + } }, extractionFn ); diff --git a/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java b/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java index 444f6bbb4968..b7fcda918cc9 100644 --- a/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java @@ -28,15 +28,12 @@ import io.druid.query.filter.Filter; import io.druid.query.filter.ValueMatcher; import io.druid.query.filter.ValueMatcherFactory; -import io.druid.segment.data.Indexed; -import java.util.Arrays; -import java.util.Iterator; import java.util.List; /** */ -public class SelectorFilter implements Filter +public class SelectorFilter extends Filter.AbstractFilter { private final String dimension; private final String value; @@ -104,4 +101,13 @@ private List makeFiltersUsingExtractionFn(BitmapIndexSelector selector) return filters; } + + @Override + public String toString() + { + return "SelectorFilter{" + + "dimension='" + dimension + '\'' + + ", value='" + value + '\'' + + '}'; + } } diff --git a/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java index d9bc9fe2cf5f..aa9b30e33456 100644 --- a/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java @@ -28,11 +28,9 @@ import io.druid.query.filter.ValueMatcherFactory; import io.druid.segment.incremental.SpatialDimensionRowTransformer; -import java.util.Arrays; - /** */ -public class SpatialFilter implements Filter +public class SpatialFilter extends Filter.AbstractFilter { private final String dimension; private final Bound bound; @@ -73,4 +71,13 @@ public boolean apply(Object input) } ); } + + @Override + public String toString() + { + return "SpatialFilter{" + + "dimension='" + dimension + '\'' + + ", bound=" + bound + + '}'; + } } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 64acece175ef..e8f84a0348b9 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -21,15 +21,14 @@ import com.google.common.base.Function; import com.google.common.base.Predicate; -import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.primitives.Ints; -import com.metamx.collections.spatial.search.Bound; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; +import io.druid.collections.IterableUtils; import io.druid.granularity.QueryGranularity; import io.druid.query.QueryInterruptedException; import io.druid.query.dimension.DimensionSpec; @@ -61,6 +60,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -71,7 +71,7 @@ */ public class IncrementalIndexStorageAdapter implements StorageAdapter { - private static final Splitter SPLITTER = Splitter.on(","); + private static final List EMPTY_PARAM = Arrays.asList((Object) null); private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(); private final IncrementalIndex index; @@ -178,7 +178,12 @@ public DateTime getMaxIngestedEventTime() } @Override - public Sequence makeCursors(final Filter filter, final Interval interval, final QueryGranularity gran, final boolean descending) + public Sequence makeCursors( + final Filter filter, + final Interval interval, + final QueryGranularity gran, + final boolean descending + ) { if (index.isEmpty()) { return Sequences.empty(); @@ -618,7 +623,7 @@ private boolean isComparableNullOrEmpty(final Comparable value) private ValueMatcher makeFilterMatcher(final Filter filter, final EntryHolder holder) { return filter == null - ? new BooleanValueMatcher(true) + ? BooleanValueMatcher.TRUE : filter.makeMatcher(new EntryHolderValueMatcherFactory(holder)); } @@ -684,7 +689,7 @@ public boolean matches() } }; } - return new BooleanValueMatcher(false); + return BooleanValueMatcher.FALSE; } return new ValueMatcher() @@ -731,6 +736,54 @@ public boolean matches() } }; } + + @Override + public ValueMatcher makeValueMatcher(String[] dimensions, final Predicate predicate) + { + final int[] dimIndices = new int[dimensions.length]; + final IncrementalIndex.DimDim[] dimValues = new IncrementalIndex.DimDim[dimensions.length]; + + Arrays.fill(dimIndices, -1); + for (int i = 0; i < dimensions.length; i++) { + IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimensions[i]); + if (dimensionDesc != null) { + dimIndices[i] = dimensionDesc.getIndex(); + dimValues[i] = dimensionDesc.getValues(); + } + } + + return new ValueMatcher() + { + private final List[] dimValuesList = new List[dimIndices.length]; + + @Override + public boolean matches() + { + int[][] dims = holder.getKey().getDims(); + for (int i = 0; i < dimIndices.length; i++) { + if (dimIndices[i] < 0 || dimIndices[i] >= dims.length) { + dimValuesList[i] = EMPTY_PARAM; + continue; + } + int[] dimIdx = dims[dimIndices[i]]; + if (dimIdx == null || dimIdx.length == 0) { + dimValuesList[i] = EMPTY_PARAM; + } else { + dimValuesList[i] = Lists.newArrayListWithCapacity(dimIdx.length); + for (int j = 0; j < dimIdx.length; j++) { + dimValuesList[i].add(dimValues[i].getValue(dimIdx[j])); + } + } + } + for (Object[] param : IterableUtils.cartesian(Object.class, dimValuesList)) { + if (predicate.apply(param)) { + return true; + } + } + return false; + } + }; + } } @Override diff --git a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java index 3a02e2f0b5f2..2a462859334f 100644 --- a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java @@ -252,66 +252,6 @@ public Object[] apply(@Nullable Object input) ); } - // simple cartesian iterable - public static Iterable cartesian(final Iterable... iterables) - { - return new Iterable() - { - - @Override - public Iterator iterator() - { - return new Iterator() - { - private final Iterator[] iterators = new Iterator[iterables.length]; - private final Object[] cached = new Object[iterables.length]; - - @Override - public boolean hasNext() - { - return hasNext(0); - } - - private boolean hasNext(int index) - { - if (iterators[index] == null) { - iterators[index] = iterables[index].iterator(); - } - for (; hasMore(index); cached[index] = null) { - if (index == iterables.length - 1 || hasNext(index + 1)) { - return true; - } - } - iterators[index] = null; - return false; - } - - private boolean hasMore(int index) - { - if (cached[index] == null && iterators[index].hasNext()) { - cached[index] = iterators[index].next(); - } - return cached[index] != null; - } - - @Override - public Object[] next() - { - Object[] result = Arrays.copyOf(cached, cached.length); - cached[cached.length - 1] = null; - return result; - } - - @Override - public void remove() - { - throw new UnsupportedOperationException("remove"); - } - }; - } - }; - } - public static > List> makeQueryRunners( QueryRunnerFactory factory ) diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 31bfb2cc11b8..4397481038b1 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -86,7 +86,6 @@ private ColumnSelectorFactory makeColumnSelector(final TestFloatColumnSelector s public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) { final String dimensionName = dimensionSpec.getDimension(); - final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); if (dimensionName.equals("dim")) { return dimensionSpec.decorate( @@ -257,7 +256,7 @@ public void testAggregateWithPredicateFilters() String jsFn = "function(x) { return(x === 'a') }"; factory = new FilteredAggregatorFactory( new DoubleSumAggregatorFactory("billy", "value"), - new JavaScriptDimFilter("dim", jsFn, null) + JavaScriptDimFilter.of("dim", jsFn, null) ); selector = new TestFloatColumnSelector(values); validateFilteredAggs(factory, values, selector); @@ -286,7 +285,7 @@ public void testAggregateWithExtractionFns() ); selector = new TestFloatColumnSelector(values); validateFilteredAggs(factory, values, selector); - + factory = new FilteredAggregatorFactory( new DoubleSumAggregatorFactory("billy", "value"), new BoundDimFilter("dim", "aAARDVARK", "aAARDVARK", false, false, true, extractionFn) @@ -311,7 +310,7 @@ public void testAggregateWithExtractionFns() String jsFn = "function(x) { return(x === 'aAARDVARK') }"; factory = new FilteredAggregatorFactory( new DoubleSumAggregatorFactory("billy", "value"), - new JavaScriptDimFilter("dim", jsFn, extractionFn) + JavaScriptDimFilter.of("dim", jsFn, extractionFn) ); selector = new TestFloatColumnSelector(values); validateFilteredAggs(factory, values, selector); diff --git a/processing/src/test/java/io/druid/query/filter/JavaScriptDimFilterSerDeTest.java b/processing/src/test/java/io/druid/query/filter/JavaScriptDimFilterSerDeTest.java new file mode 100644 index 000000000000..47dfb18d1974 --- /dev/null +++ b/processing/src/test/java/io/druid/query/filter/JavaScriptDimFilterSerDeTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Injector; +import com.google.inject.Key; +import io.druid.guice.GuiceInjectors; +import io.druid.guice.annotations.Json; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +public class JavaScriptDimFilterSerDeTest +{ + private static ObjectMapper mapper; + + private static final String JSFilter = + "{\"type\":\"javascript\",\"extractionFn\":null,\"function\":\"function(d1, d2) { return d1 === d2 }\",\"dimensions\":[\"dimTest1\",\"dimTest2\"]}"; + private static final String JSFilterBackwardCompatible = + "{\"type\":\"javascript\",\"dimension\":\"dimTest\",\"function\":\"function(d1) { return true }\",\"byRow\":false}"; + private static final String JSFilterRewritten = + "{\"type\":\"javascript\",\"extractionFn\":null,\"function\":\"function(d1) { return true }\",\"dimensions\":[\"dimTest\"]}"; + + private static final String JSFilterbyRow = + "{\"type\":\"javascript\",\"extractionFn\":null,\"function\":\"function(d1, d2) { return d1 === d2 }\",\"dimensions\":[\"dimTest1\",\"dimTest2\"]}"; + + @Before + public void setUp() + { + Injector defaultInjector = GuiceInjectors.makeStartupInjector(); + mapper = defaultInjector.getInstance(Key.get(ObjectMapper.class, Json.class)); + } + + @Test + public void testDeserialization1() throws IOException + { + final JavaScriptDimFilter actualJSDimFilter = mapper.reader(DimFilter.class).readValue(JSFilterBackwardCompatible); + final JavaScriptDimFilter expectedJSDimFilter = + JavaScriptDimFilter.of("dimTest", "function(d1) { return true }", null); + Assert.assertEquals(expectedJSDimFilter, actualJSDimFilter); + } + + @Test + public void testDeserialization2() throws IOException + { + final JavaScriptDimFilter actualJSDimFilter = mapper.reader(DimFilter.class).readValue(JSFilter); + final JavaScriptDimFilter expectedJSDimFilter = + JavaScriptDimFilter.of(new String[]{"dimTest1", "dimTest2"}, "function(d1, d2) { return d1 === d2 }", null); + Assert.assertEquals(expectedJSDimFilter, actualJSDimFilter); + } + + @Test + public void testSerialization1() throws IOException + { + final JavaScriptDimFilter JSDimFilter = + JavaScriptDimFilter.of(new String[]{"dimTest1", "dimTest2"}, "function(d1, d2) { return d1 === d2 }", null); + Assert.assertEquals(JSFilter, mapper.writeValueAsString(JSDimFilter)); + } + + @Test + public void testSerialization2() throws IOException + { + final JavaScriptDimFilter JSDimFilter = JavaScriptDimFilter.of("dimTest", "function(d1) { return true }", null); + Assert.assertEquals(JSFilterRewritten, mapper.writeValueAsString(JSDimFilter)); + } + + @Test + public void testSerialization3() throws IOException + { + final JavaScriptDimFilter JSDimFilter = JavaScriptDimFilter.byRow( + new String[]{"dimTest1", "dimTest2"}, + "function(d1, d2) { return d1 === d2 }", + null + ); + Assert.assertEquals(JSFilterbyRow, mapper.writeValueAsString(JSDimFilter)); + } +} diff --git a/processing/src/test/java/io/druid/query/filter/JavaScriptDimFilterTest.java b/processing/src/test/java/io/druid/query/filter/JavaScriptDimFilterTest.java index 3f8cbd84ce9f..588ef72d62d2 100644 --- a/processing/src/test/java/io/druid/query/filter/JavaScriptDimFilterTest.java +++ b/processing/src/test/java/io/druid/query/filter/JavaScriptDimFilterTest.java @@ -29,29 +29,45 @@ public class JavaScriptDimFilterTest { @Test - public void testGetCacheKey() + public void testGetCacheKey1() { - JavaScriptDimFilter javaScriptDimFilter = new JavaScriptDimFilter("dim", "fn", null); - JavaScriptDimFilter javaScriptDimFilter2 = new JavaScriptDimFilter("di", "mfn", null); + JavaScriptDimFilter javaScriptDimFilter = JavaScriptDimFilter.of("dim", "fn", null); + JavaScriptDimFilter javaScriptDimFilter2 = JavaScriptDimFilter.of("di", "mfn", null); + Assert.assertFalse(Arrays.equals(javaScriptDimFilter.getCacheKey(), javaScriptDimFilter2.getCacheKey())); + } + + @Test + public void testGetCacheKey2() + { + JavaScriptDimFilter javaScriptDimFilter = JavaScriptDimFilter.of(new String[]{"dim1", "dim2"}, "fn", null); + JavaScriptDimFilter javaScriptDimFilter2 = JavaScriptDimFilter.of(new String[]{"dim1", "dim"}, "2fn", null); + Assert.assertFalse(Arrays.equals(javaScriptDimFilter.getCacheKey(), javaScriptDimFilter2.getCacheKey())); + } + + @Test + public void testGetCacheKey3() + { + JavaScriptDimFilter javaScriptDimFilter = JavaScriptDimFilter.byRow(new String[]{"dim1", "dim2"}, "fn", null); + JavaScriptDimFilter javaScriptDimFilter2 = JavaScriptDimFilter.byRow(new String[]{"dim1", "dim2"}, "mfn", null); Assert.assertFalse(Arrays.equals(javaScriptDimFilter.getCacheKey(), javaScriptDimFilter2.getCacheKey())); RegexDimExtractionFn regexFn = new RegexDimExtractionFn(".*", false, null); - JavaScriptDimFilter javaScriptDimFilter3 = new JavaScriptDimFilter("dim", "fn", regexFn); + JavaScriptDimFilter javaScriptDimFilter3 = JavaScriptDimFilter.of("dim", "fn", regexFn); Assert.assertFalse(Arrays.equals(javaScriptDimFilter.getCacheKey(), javaScriptDimFilter3.getCacheKey())); } @Test public void testEquals() { - JavaScriptDimFilter javaScriptDimFilter = new JavaScriptDimFilter("dim", "fn", null); - JavaScriptDimFilter javaScriptDimFilter2 = new JavaScriptDimFilter("di", "mfn", null); - JavaScriptDimFilter javaScriptDimFilter3 = new JavaScriptDimFilter("di", "mfn", null); + JavaScriptDimFilter javaScriptDimFilter = JavaScriptDimFilter.of("dim", "fn", null); + JavaScriptDimFilter javaScriptDimFilter2 = JavaScriptDimFilter.of("di", "mfn", null); + JavaScriptDimFilter javaScriptDimFilter3 = JavaScriptDimFilter.of("di", "mfn", null); Assert.assertNotEquals(javaScriptDimFilter, javaScriptDimFilter2); Assert.assertEquals(javaScriptDimFilter2, javaScriptDimFilter3); RegexDimExtractionFn regexFn = new RegexDimExtractionFn(".*", false, null); - JavaScriptDimFilter javaScriptDimFilter4 = new JavaScriptDimFilter("dim", "fn", regexFn); - JavaScriptDimFilter javaScriptDimFilter5 = new JavaScriptDimFilter("dim", "fn", regexFn); + JavaScriptDimFilter javaScriptDimFilter4 = JavaScriptDimFilter.of("dim", "fn", regexFn); + JavaScriptDimFilter javaScriptDimFilter5 = JavaScriptDimFilter.of("dim", "fn", regexFn); Assert.assertNotEquals(javaScriptDimFilter, javaScriptDimFilter3); Assert.assertEquals(javaScriptDimFilter4, javaScriptDimFilter5); } @@ -59,15 +75,15 @@ public void testEquals() @Test public void testHashcode() { - JavaScriptDimFilter javaScriptDimFilter = new JavaScriptDimFilter("dim", "fn", null); - JavaScriptDimFilter javaScriptDimFilter2 = new JavaScriptDimFilter("di", "mfn", null); - JavaScriptDimFilter javaScriptDimFilter3 = new JavaScriptDimFilter("di", "mfn", null); + JavaScriptDimFilter javaScriptDimFilter = JavaScriptDimFilter.of("dim", "fn", null); + JavaScriptDimFilter javaScriptDimFilter2 = JavaScriptDimFilter.of("di", "mfn", null); + JavaScriptDimFilter javaScriptDimFilter3 = JavaScriptDimFilter.of("di", "mfn", null); Assert.assertNotEquals(javaScriptDimFilter.hashCode(), javaScriptDimFilter2.hashCode()); Assert.assertEquals(javaScriptDimFilter2.hashCode(), javaScriptDimFilter3.hashCode()); RegexDimExtractionFn regexFn = new RegexDimExtractionFn(".*", false, null); - JavaScriptDimFilter javaScriptDimFilter4 = new JavaScriptDimFilter("dim", "fn", regexFn); - JavaScriptDimFilter javaScriptDimFilter5 = new JavaScriptDimFilter("dim", "fn", regexFn); + JavaScriptDimFilter javaScriptDimFilter4 = JavaScriptDimFilter.of("dim", "fn", regexFn); + JavaScriptDimFilter javaScriptDimFilter5 = JavaScriptDimFilter.of("dim", "fn", regexFn); Assert.assertNotEquals(javaScriptDimFilter.hashCode(), javaScriptDimFilter3.hashCode()); Assert.assertEquals(javaScriptDimFilter4.hashCode(), javaScriptDimFilter5.hashCode()); } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index ea8911814ee4..48c102cfb2af 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -2722,7 +2722,7 @@ public void testIdenticalSubquery() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }", null)) + .setDimFilter(JavaScriptDimFilter.of("quality", "function(dim){ return true; }", null)) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2781,7 +2781,7 @@ public void testSubqueryWithMultipleIntervalsInOuterQuery() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }", null)) + .setDimFilter(JavaScriptDimFilter.of("quality", "function(dim){ return true; }", null)) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2849,7 +2849,7 @@ public void testSubqueryWithExtractionFnInOuterQuery() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }", null)) + .setDimFilter(JavaScriptDimFilter.of("quality", "function(dim){ return true; }", null)) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3121,7 +3121,7 @@ public void testSubqueryWithPostAggregators() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }", null)) + .setDimFilter(JavaScriptDimFilter.of("quality", "function(dim){ return true; }", null)) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3382,7 +3382,7 @@ public void testSubqueryWithPostAggregatorsAndHaving() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }", null)) + .setDimFilter(JavaScriptDimFilter.of("quality", "function(dim){ return true; }", null)) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3640,7 +3640,7 @@ public void testSubqueryWithMultiColumnAggregators() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("market", "function(dim){ return true; }", null)) + .setDimFilter(JavaScriptDimFilter.of("market", "function(dim){ return true; }", null)) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -4817,7 +4817,7 @@ public void testBySegmentResultsWithAllFiltersWithExtractionFns() superFilterList.add(new BoundDimFilter("quality", "super-mezzanine", "super-mezzanine", false, false, true, extractionFn)); superFilterList.add(new RegexDimFilter("quality", "super-mezzanine", extractionFn)); superFilterList.add(new SearchQueryDimFilter("quality", new ContainsSearchQuerySpec("super-mezzanine", true), extractionFn)); - superFilterList.add(new JavaScriptDimFilter("quality", jsFn, extractionFn)); + superFilterList.add(JavaScriptDimFilter.of("quality", jsFn, extractionFn)); DimFilter superFilter = new AndDimFilter(superFilterList); GroupByQuery.Builder builder = GroupByQuery @@ -4869,21 +4869,128 @@ public void testGroupByWithAllFiltersOnNullDimsWithExtractionFns() superFilterList.add(new InDimFilter("null_column", Arrays.asList("NOT-EMPTY", "FOOBAR", "EMPTY"), extractionFn)); superFilterList.add(new BoundDimFilter("null_column", "EMPTY", "EMPTY", false, false, true, extractionFn)); superFilterList.add(new RegexDimFilter("null_column", "EMPTY", extractionFn)); - superFilterList.add(new SearchQueryDimFilter("null_column", new ContainsSearchQuerySpec("EMPTY", true), extractionFn)); - superFilterList.add(new JavaScriptDimFilter("null_column", jsFn, extractionFn)); + superFilterList.add( + new SearchQueryDimFilter( + "null_column", + new ContainsSearchQuerySpec("EMPTY", true), + extractionFn + ) + ); + superFilterList.add(JavaScriptDimFilter.of("null_column", jsFn, extractionFn)); DimFilter superFilter = new AndDimFilter(superFilterList); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("null_column", "alias"))) + .setDimensions( + Lists.newArrayList( + new DefaultDimensionSpec( + "null_column", + "alias" + ) + ) + ) .setAggregatorSpecs( - Arrays.asList(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index"))) + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", "index") + ) + ) .setGranularity(QueryRunnerTestHelper.dayGran) .setDimFilter(superFilter).build(); List expectedResults = Arrays - .asList(GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", null, "rows", 13L, "idx", 6619L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", null, "rows", 13L, "idx", 5827L)); + .asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", null, "rows", 13L, "idx", 6619L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", null, "rows", 13L, "idx", 5827L) + ); + + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + TestHelper.assertExpectedObjects(expectedResults, results, ""); + } + + @Test + public void testJavaScriptDimFilterWithMultipleDimensions() + { + GroupByQuery query = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) + .setDimensions( + Lists.newArrayList( + new DefaultDimensionSpec("market", "mkt"), + new DefaultDimensionSpec("quality", "alias") + ) + ) + .setDimFilter( + JavaScriptDimFilter.of( + new String[]{"market", "quality"}, "function(d1, d2){ return d1.length == d2.length; }", null + ) + ) + .setAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", "index") + ) + ) + .setGranularity(QueryRunnerTestHelper.dayGran) + .build(); + + List expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", "alias", "news", "mkt", "spot", "rows", 1L, "idx", 121L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", "alias", "premium", "mkt", "upfront", "rows", 1L, "idx", 1234L + ), + + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", "alias", "news", "mkt", "spot", "rows", 1L, "idx", 114L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", "alias", "premium", "mkt", "upfront", "rows", 1L, "idx", 1049L + ) + ); + + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + TestHelper.assertExpectedObjects(expectedResults, results, ""); + } + + + @Test + public void testJavaScriptDimFilterWithMultipleDimensionsOnNotExistingDimension() + { + GroupByQuery query = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) + .setDimensions( + Lists.newArrayList( + new DefaultDimensionSpec("market", "mkt"), + new DefaultDimensionSpec("null_column", "null_field") + ) + ) + .setDimFilter( + JavaScriptDimFilter.of( + new String[]{"market", "null_column"}, "function(d1, d2){ return d1 === 'spot' && d2 === null; }", null + ) + ) + .setAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", "index") + ) + ) + .setGranularity(QueryRunnerTestHelper.dayGran) + .build(); + + List expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", "null_field", null, "mkt", "spot", "rows", 9L, "idx", 1102L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", "null_field", null, "mkt", "spot", "rows", 9L, "idx", 1120L + ) + ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); diff --git a/processing/src/test/java/io/druid/query/select/MultiSegmentSelectQueryTest.java b/processing/src/test/java/io/druid/query/select/MultiSegmentSelectQueryTest.java index 8b0aaae6ced2..22be2ee585a0 100644 --- a/processing/src/test/java/io/druid/query/select/MultiSegmentSelectQueryTest.java +++ b/processing/src/test/java/io/druid/query/select/MultiSegmentSelectQueryTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import com.google.common.io.CharSource; import com.metamx.common.guava.Sequences; +import io.druid.collections.IterableUtils; import io.druid.granularity.QueryGranularity; import io.druid.jackson.DefaultObjectMapper; import io.druid.query.Druids; @@ -193,7 +194,7 @@ public static void clear() @Parameterized.Parameters(name = "fromNext={0}") public static Iterable constructorFeeder() throws IOException { - return QueryRunnerTestHelper.cartesian(Arrays.asList(false, true)); + return IterableUtils.cartesian(Arrays.asList(false, true)); } private final boolean fromNext; diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index 9f8ddd92b0ef..73167dd9834b 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.ObjectArrays; import com.metamx.common.ISE; import com.metamx.common.guava.Sequences; +import io.druid.collections.IterableUtils; import io.druid.jackson.DefaultObjectMapper; import io.druid.query.Druids; import io.druid.query.QueryRunner; @@ -102,7 +103,7 @@ public class SelectQueryRunnerTest @Parameterized.Parameters(name = "{0}:descending={1}") public static Iterable constructorFeeder() throws IOException { - return QueryRunnerTestHelper.cartesian( + return IterableUtils.cartesian( QueryRunnerTestHelper.makeQueryRunners( new SelectQueryRunnerFactory( new SelectQueryQueryToolChest( diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java index 20f2b7e3b47e..73fdf48190e8 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.Maps; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; +import io.druid.collections.IterableUtils; import io.druid.query.Druids; import io.druid.query.Query; import io.druid.query.QueryRunner; @@ -64,7 +65,7 @@ public TimeSeriesUnionQueryRunnerTest( @Parameterized.Parameters(name="{0}:descending={1}") public static Iterable constructorFeeder() throws IOException { - return QueryRunnerTestHelper.cartesian( + return IterableUtils.cartesian( QueryRunnerTestHelper.makeUnionQueryRunners( new TimeseriesQueryRunnerFactory( new TimeseriesQueryQueryToolChest(QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator()), diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index d57436de47f2..56b0839936be 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.metamx.common.guava.Sequences; +import io.druid.collections.IterableUtils; import io.druid.granularity.PeriodGranularity; import io.druid.granularity.QueryGranularity; import io.druid.query.Druids; @@ -41,7 +42,9 @@ import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; import io.druid.query.filter.InDimFilter; +import io.druid.query.filter.JavaScriptDimFilter; import io.druid.query.filter.NotDimFilter; +import io.druid.query.filter.OrDimFilter; import io.druid.query.filter.RegexDimFilter; import io.druid.query.filter.SelectorDimFilter; import io.druid.query.spec.MultipleIntervalSegmentSpec; @@ -72,7 +75,7 @@ public class TimeseriesQueryRunnerTest @Parameterized.Parameters(name="{0}:descending={1}") public static Iterable constructorFeeder() throws IOException { - return QueryRunnerTestHelper.cartesian( + return IterableUtils.cartesian( // runners QueryRunnerTestHelper.makeQueryRunners( new TimeseriesQueryRunnerFactory( @@ -989,6 +992,132 @@ public void testTimeseriesWithFilter3() assertExpectedResults(expectedResults, results); } + @Test + public void testTimeseriesWithMultiJavaScriptFilter() + { + DimFilter filter = JavaScriptDimFilter.byRow( + new String[]{ + QueryRunnerTestHelper.marketDimension, + QueryRunnerTestHelper.qualityDimension + }, + "function(m, q){ return m === \"spot\" && q === \"business\"; }", + null + ); + + TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.dayGran) + .filters(filter) + .intervals(QueryRunnerTestHelper.firstToThird) + .aggregators( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + QueryRunnerTestHelper.indexLongSum, + QueryRunnerTestHelper.qualityUniques + ) + ) + .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) + .build(); + + List> expectedResults = Arrays.asList( + new Result<>( + new DateTime("2011-04-01"), + new TimeseriesResultValue( + ImmutableMap.of( + "rows", 1L, + "index", 118L, + "addRowsIndexConstant", 120.0, + "uniques", QueryRunnerTestHelper.UNIQUES_1 + ) + ) + ), + new Result<>( + new DateTime("2011-04-02"), + new TimeseriesResultValue( + ImmutableMap.of( + "rows", 1L, + "index", 112L, + "addRowsIndexConstant", 114.0, + "uniques", QueryRunnerTestHelper.UNIQUES_1 + ) + ) + ) + ); + + Iterable> results = Sequences.toList( + runner.run(query, CONTEXT), + Lists.>newArrayList() + ); + TestHelper.assertExpectedResults(expectedResults, results); + } + + @Test + public void testTimeseriesWithMultiJavaScriptFilter2() + { + DimFilter filter1 = JavaScriptDimFilter.byRow( + new String[]{ + QueryRunnerTestHelper.marketDimension, + QueryRunnerTestHelper.qualityDimension + }, + "function(m, q){ return m === \"spot\" && q === \"business\"; }", + null + ); + + SelectorDimFilter filter2 = new SelectorDimFilter(QueryRunnerTestHelper.qualityDimension, "mezzanine", null); + + DimFilter filter = new OrDimFilter(Arrays.asList(filter1, filter2)); + + TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.dayGran) + .filters(filter) + .intervals(QueryRunnerTestHelper.firstToThird) + .aggregators( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + QueryRunnerTestHelper.indexLongSum, + QueryRunnerTestHelper.qualityUniques + ) + ) + .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) + .build(); + + + List> expectedResults = Arrays.asList( + new Result<>( + new DateTime("2011-04-01"), + new TimeseriesResultValue( + ImmutableMap.of( + "rows", 4L, + "index", 2988L, + "addRowsIndexConstant", 2993.0, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ) + ) + ), + new Result<>( + new DateTime("2011-04-02"), + new TimeseriesResultValue( + ImmutableMap.of( + "rows", 4L, + "index", 2559L, + "addRowsIndexConstant", 2564.0, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ) + ) + ) + ); + + Iterable> results = Sequences.toList( + runner.run(query, CONTEXT), + Lists.>newArrayList() + ); + for (Result x : results) { + System.out.println("[TimeseriesQueryRunnerTest/testTimeseriesWithMultiJavaScriptFilter2] " + x); + } + TestHelper.assertExpectedResults(expectedResults, results); + } + @Test public void testTimeseriesWithMultiDimFilterAndOr() { diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryTest.java index 3a93a703c17e..c11e4f09aa2a 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryTest.java @@ -20,6 +20,7 @@ package io.druid.query.timeseries; import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.collections.IterableUtils; import io.druid.jackson.DefaultObjectMapper; import io.druid.query.Druids; import io.druid.query.Query; @@ -41,7 +42,7 @@ public class TimeseriesQueryTest @Parameterized.Parameters(name="descending={0}") public static Iterable constructorFeeder() throws IOException { - return QueryRunnerTestHelper.cartesian(Arrays.asList(false, true)); + return IterableUtils.cartesian(Arrays.asList(false, true)); } private final boolean descending; diff --git a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java index a7b27c0f0f3f..f35b466246fd 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.druid.collections.IterableUtils; import io.druid.collections.StupidPool; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; @@ -60,7 +61,7 @@ public TopNUnionQueryTest( @Parameterized.Parameters public static Iterable constructorFeeder() throws IOException { - return QueryRunnerTestHelper.cartesian( + return IterableUtils.cartesian( Iterables.concat( QueryRunnerTestHelper.makeUnionQueryRunners( new TopNQueryRunnerFactory( diff --git a/processing/src/test/java/io/druid/segment/filter/FiltersTest.java b/processing/src/test/java/io/druid/segment/filter/FiltersTest.java new file mode 100644 index 000000000000..d083f60f1a60 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/filter/FiltersTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.filter; + +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.filter.AndDimFilter; +import io.druid.query.filter.DimFilter; +import io.druid.query.filter.Filter; +import io.druid.query.filter.JavaScriptDimFilter; +import io.druid.query.filter.NotDimFilter; +import io.druid.query.filter.OrDimFilter; +import io.druid.query.filter.SelectorDimFilter; +import org.junit.Assert; +import org.junit.Test; + +public class FiltersTest +{ + @Test + public void testPartitionWithBitmapSupport() throws Exception + { + SelectorDimFilter dim1 = new SelectorDimFilter(QueryRunnerTestHelper.qualityDimension, "mezzanine", null); + + DimFilter dim2 = JavaScriptDimFilter.byRow( + new String[]{ + QueryRunnerTestHelper.marketDimension, + QueryRunnerTestHelper.qualityDimension + }, + "function(m, q){ return m === \"spot\" && q === \"business\"; }", + null + ); + + DimFilter dim3 = AndDimFilter.of(dim1, dim2); + + DimFilter dim4 = OrDimFilter.of(dim1, dim2); + + Filter filter1 = dim1.toFilter(); + Filter filter2 = dim2.toFilter(); + Filter filter3 = dim3.toFilter(); + Filter filter4 = dim4.toFilter(); + + // DIM1 AND !(DIM1 OR DIM2) -> DIM1 AND !DIM1 AND !DIM2 + Filter filter5 = AndDimFilter.of(dim1, NotDimFilter.of(dim4)).toFilter(); + + Filter[] filters; + + filters = Filters.partitionWithBitmapSupport(null); + Assert.assertNull(filters); + + filters = Filters.partitionWithBitmapSupport(filter1); + assertEquals(filter1, filters[0]); + Assert.assertNull(filters[1]); + + filters = Filters.partitionWithBitmapSupport(filter2); + Assert.assertNull(filters[0]); + assertEquals(filter2, filters[1]); + + filters = Filters.partitionWithBitmapSupport(filter3); + assertEquals(filter1, filters[0]); + assertEquals(filter2, filters[1]); + + filters = Filters.partitionWithBitmapSupport(filter4); + Assert.assertNull(filters[0]); + assertEquals(filter4, filters[1]); + + filters = Filters.partitionWithBitmapSupport(filter5); + assertEquals(AndFilter.of(filter1, NotFilter.of(filter1)), filters[0]); + assertEquals(NotFilter.of(filter2), filters[1]); + } + + private void assertEquals(Filter expected, Filter result) { + Assert.assertEquals(expected.toString(), result.toString()); + } +} diff --git a/processing/src/test/java/io/druid/segment/filter/JavaScriptDimFilterTest.java b/processing/src/test/java/io/druid/segment/filter/JavaScriptDimFilterTest.java new file mode 100644 index 000000000000..1c7ab22dfb6e --- /dev/null +++ b/processing/src/test/java/io/druid/segment/filter/JavaScriptDimFilterTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.filter; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.metamx.collections.bitmap.BitmapFactory; +import com.metamx.collections.bitmap.ConciseBitmapFactory; +import com.metamx.collections.bitmap.ImmutableBitmap; +import com.metamx.collections.bitmap.MutableBitmap; +import com.metamx.collections.bitmap.RoaringBitmapFactory; +import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.DimFilters; +import io.druid.query.filter.Filter; +import io.druid.query.filter.JavaScriptDimFilter; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.data.ArrayIndexed; +import io.druid.segment.data.Indexed; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * + */ +@RunWith(Parameterized.class) +public class JavaScriptDimFilterTest +{ + /* + * 3 row values of three dimensions + * + * foo | bar | baz + * ==================== + * foo1 | bar1 | foo1 + * foo2 | foo3 | foo3 + * foo3 | foo1 | foo3 + */ + private static final Map DIM_VALS = ImmutableMap.of( + "foo", new String[]{"foo1", "foo2", "foo3"}, + "bar", new String[]{"bar1", "foo1", "foo3"}, + "baz", new String[]{"foo1", "foo3"} + ); + + @Parameterized.Parameters + public static Iterable constructorFeeder() + { + return ImmutableList.of( + new Object[]{new ConciseBitmapFactory()}, + new Object[]{new RoaringBitmapFactory()} + ); + } + + public JavaScriptDimFilterTest(BitmapFactory bitmapFactory) + { + bitmapMap = new LinkedHashMap<>(); + final MutableBitmap mutableBitmapfoo1 = bitmapFactory.makeEmptyMutableBitmap(); + final MutableBitmap mutableBitmapfoo2 = bitmapFactory.makeEmptyMutableBitmap(); + final MutableBitmap mutableBitmapfoo3 = bitmapFactory.makeEmptyMutableBitmap(); + mutableBitmapfoo1.add(0); + mutableBitmapfoo2.add(1); + mutableBitmapfoo3.add(2); + Map foo = new LinkedHashMap<>(); + foo.put("foo1", bitmapFactory.makeImmutableBitmap(mutableBitmapfoo1)); + foo.put("foo2", bitmapFactory.makeImmutableBitmap(mutableBitmapfoo2)); + foo.put("foo3", bitmapFactory.makeImmutableBitmap(mutableBitmapfoo3)); + this.bitmapMap.put("foo", foo); + + final MutableBitmap mutableBitmapbar1 = bitmapFactory.makeEmptyMutableBitmap(); + final MutableBitmap mutableBitmapbar2 = bitmapFactory.makeEmptyMutableBitmap(); + final MutableBitmap mutableBitmapbar3 = bitmapFactory.makeEmptyMutableBitmap(); + mutableBitmapbar1.add(0); + mutableBitmapbar2.add(2); + mutableBitmapbar3.add(1); + Map bar = new LinkedHashMap<>(); + bar.put("bar1", bitmapFactory.makeImmutableBitmap(mutableBitmapbar1)); + bar.put("foo1", bitmapFactory.makeImmutableBitmap(mutableBitmapbar2)); + bar.put("foo3", bitmapFactory.makeImmutableBitmap(mutableBitmapbar3)); + this.bitmapMap.put("bar", bar); + + final MutableBitmap mutableBitmapbaz1 = bitmapFactory.makeEmptyMutableBitmap(); + final MutableBitmap mutableBitmapbaz2 = bitmapFactory.makeEmptyMutableBitmap(); + mutableBitmapbaz1.add(0); + mutableBitmapbaz2.add(1); + mutableBitmapbaz2.add(2); + Map baz = new LinkedHashMap<>(); + baz.put("foo1", bitmapFactory.makeImmutableBitmap(mutableBitmapbaz1)); + baz.put("foo3", bitmapFactory.makeImmutableBitmap(mutableBitmapbaz2)); + this.bitmapMap.put("baz", baz); + + this.factory = bitmapFactory; + } + + private final BitmapFactory factory; + private final Map> bitmapMap; + + private final BitmapIndexSelector BITMAP_INDEX_SELECTOR = new BitmapIndexSelector() + { + @Override + public Indexed getDimensionValues(String dimension) + { + final String[] vals = DIM_VALS.get(dimension); + return vals == null ? null : new ArrayIndexed(vals, String.class); + } + + @Override + public int getNumRows() + { + return 3; + } + + @Override + public BitmapFactory getBitmapFactory() + { + return factory; + } + + @Override + public BitmapIndex getBitmapIndex(String dimension) + { + return null; + } + + @Override + public ImmutableBitmap getBitmapIndex(String dimension, String value) + { + return bitmapMap.get(dimension).get(value); + } + + @Override + public ImmutableRTree getSpatialIndex(String dimension) + { + return null; + } + }; + private static final String javaScript = "function(dim1, dim2) { return dim1 === dim2 }"; + private static final String[] dimensions1 = {"foo", "bar"}; + private static final String[] dimensions2 = {"foo", "baz"}; + + @Test + public void testEmpty() + { + Filter javaScriptFilter = JavaScriptDimFilter.of(dimensions1, javaScript, null).toFilter(); + ImmutableBitmap immutableBitmap = javaScriptFilter.getBitmapIndex(BITMAP_INDEX_SELECTOR); + Assert.assertEquals(0, immutableBitmap.size()); + } + + @Test + public void testNormal() + { + Filter javaScriptFilter = JavaScriptDimFilter.of(dimensions2, javaScript, null).toFilter(); + ImmutableBitmap immutableBitmap = javaScriptFilter.getBitmapIndex(BITMAP_INDEX_SELECTOR); + Assert.assertEquals(2, immutableBitmap.size()); + } + + @Test + public void testOr() + { + Assert.assertEquals( + 2, + DimFilters.or( + JavaScriptDimFilter.of(dimensions2, javaScript, null) + ).toFilter().getBitmapIndex(BITMAP_INDEX_SELECTOR).size() + ); + + Assert.assertEquals( + 2, + DimFilters.or( + JavaScriptDimFilter.of(dimensions1, javaScript, null), + JavaScriptDimFilter.of(dimensions2, javaScript, null) + ).toFilter().getBitmapIndex(BITMAP_INDEX_SELECTOR).size() + ); + } + + @Test + public void testAnd() + { + Assert.assertEquals( + 2, + DimFilters.or( + JavaScriptDimFilter.of(dimensions2, javaScript, null) + ).toFilter().getBitmapIndex(BITMAP_INDEX_SELECTOR).size() + ); + + Assert.assertEquals( + 0, + DimFilters.and( + JavaScriptDimFilter.of(dimensions1, javaScript, null), + JavaScriptDimFilter.of(dimensions2, javaScript, null) + ).toFilter().getBitmapIndex(BITMAP_INDEX_SELECTOR).size() + ); + } + + @Test + public void testNot() + { + Assert.assertEquals( + 2, DimFilters.or( + JavaScriptDimFilter.of(dimensions2, javaScript, null) + ).toFilter().getBitmapIndex(BITMAP_INDEX_SELECTOR).size() + ); + + Assert.assertEquals( + 1, + DimFilters.not( + JavaScriptDimFilter.of(dimensions2, javaScript, null) + ).toFilter().getBitmapIndex(BITMAP_INDEX_SELECTOR).size() + ); + } +} diff --git a/processing/src/test/java/io/druid/segment/filter/JavascriptFilterTest.java b/processing/src/test/java/io/druid/segment/filter/JavascriptFilterTest.java index 3bc471c42f38..cce8e51c5e4e 100644 --- a/processing/src/test/java/io/druid/segment/filter/JavascriptFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/JavascriptFilterTest.java @@ -93,51 +93,51 @@ private String jsValueFilter(String value) @Test public void testSingleValueStringColumnWithoutNulls() { - assertFilterMatches(new JavaScriptDimFilter("dim0", jsNullFilter, null), ImmutableList.of()); - assertFilterMatches(new JavaScriptDimFilter("dim0", jsValueFilter(""), null), ImmutableList.of()); - assertFilterMatches(new JavaScriptDimFilter("dim0", jsValueFilter("0"), null), ImmutableList.of("0")); - assertFilterMatches(new JavaScriptDimFilter("dim0", jsValueFilter("1"), null), ImmutableList.of("1")); + assertFilterMatches(JavaScriptDimFilter.of("dim0", jsNullFilter, null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim0", jsValueFilter(""), null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim0", jsValueFilter("0"), null), ImmutableList.of("0")); + assertFilterMatches(JavaScriptDimFilter.of("dim0", jsValueFilter("1"), null), ImmutableList.of("1")); } @Test public void testSingleValueStringColumnWithNulls() { - assertFilterMatches(new JavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of("0")); - assertFilterMatches(new JavaScriptDimFilter("dim1", jsValueFilter("10"), null), ImmutableList.of("1")); - assertFilterMatches(new JavaScriptDimFilter("dim1", jsValueFilter("2"), null), ImmutableList.of("2")); - assertFilterMatches(new JavaScriptDimFilter("dim1", jsValueFilter("1"), null), ImmutableList.of("3")); - assertFilterMatches(new JavaScriptDimFilter("dim1", jsValueFilter("def"), null), ImmutableList.of("4")); - assertFilterMatches(new JavaScriptDimFilter("dim1", jsValueFilter("abc"), null), ImmutableList.of("5")); - assertFilterMatches(new JavaScriptDimFilter("dim1", jsValueFilter("ab"), null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim1", jsNullFilter, null), ImmutableList.of("0")); + assertFilterMatches(JavaScriptDimFilter.of("dim1", jsValueFilter("10"), null), ImmutableList.of("1")); + assertFilterMatches(JavaScriptDimFilter.of("dim1", jsValueFilter("2"), null), ImmutableList.of("2")); + assertFilterMatches(JavaScriptDimFilter.of("dim1", jsValueFilter("1"), null), ImmutableList.of("3")); + assertFilterMatches(JavaScriptDimFilter.of("dim1", jsValueFilter("def"), null), ImmutableList.of("4")); + assertFilterMatches(JavaScriptDimFilter.of("dim1", jsValueFilter("abc"), null), ImmutableList.of("5")); + assertFilterMatches(JavaScriptDimFilter.of("dim1", jsValueFilter("ab"), null), ImmutableList.of()); } @Test public void testMultiValueStringColumn() { // multi-val null...... - assertFilterMatches(new JavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "2", "5")); - assertFilterMatches(new JavaScriptDimFilter("dim2", jsValueFilter("a"), null), ImmutableList.of("0", "3")); - assertFilterMatches(new JavaScriptDimFilter("dim2", jsValueFilter("b"), null), ImmutableList.of("0")); - assertFilterMatches(new JavaScriptDimFilter("dim2", jsValueFilter("c"), null), ImmutableList.of("4")); - assertFilterMatches(new JavaScriptDimFilter("dim2", jsValueFilter("d"), null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim2", jsNullFilter, null), ImmutableList.of("1", "2", "5")); + assertFilterMatches(JavaScriptDimFilter.of("dim2", jsValueFilter("a"), null), ImmutableList.of("0", "3")); + assertFilterMatches(JavaScriptDimFilter.of("dim2", jsValueFilter("b"), null), ImmutableList.of("0")); + assertFilterMatches(JavaScriptDimFilter.of("dim2", jsValueFilter("c"), null), ImmutableList.of("4")); + assertFilterMatches(JavaScriptDimFilter.of("dim2", jsValueFilter("d"), null), ImmutableList.of()); } @Test public void testMissingColumnSpecifiedInDimensionList() { - assertFilterMatches(new JavaScriptDimFilter("dim3", jsNullFilter, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - assertFilterMatches(new JavaScriptDimFilter("dim3", jsValueFilter("a"), null), ImmutableList.of()); - assertFilterMatches(new JavaScriptDimFilter("dim3", jsValueFilter("b"), null), ImmutableList.of()); - assertFilterMatches(new JavaScriptDimFilter("dim3", jsValueFilter("c"), null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim3", jsNullFilter, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + assertFilterMatches(JavaScriptDimFilter.of("dim3", jsValueFilter("a"), null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim3", jsValueFilter("b"), null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim3", jsValueFilter("c"), null), ImmutableList.of()); } @Test public void testMissingColumnNotSpecifiedInDimensionList() { - assertFilterMatches(new JavaScriptDimFilter("dim4", jsNullFilter, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - assertFilterMatches(new JavaScriptDimFilter("dim4", jsValueFilter("a"), null), ImmutableList.of()); - assertFilterMatches(new JavaScriptDimFilter("dim4", jsValueFilter("b"), null), ImmutableList.of()); - assertFilterMatches(new JavaScriptDimFilter("dim4", jsValueFilter("c"), null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim4", jsNullFilter, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + assertFilterMatches(JavaScriptDimFilter.of("dim4", jsValueFilter("a"), null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim4", jsValueFilter("b"), null), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim4", jsValueFilter("c"), null), ImmutableList.of()); } @Test @@ -152,20 +152,20 @@ public void testJavascriptFilterWithLookupExtractionFn() LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false); LookupExtractionFn lookupFn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true); - assertFilterMatches(new JavaScriptDimFilter("dim0", jsValueFilter("HELLO"), lookupFn), ImmutableList.of("1")); - assertFilterMatches(new JavaScriptDimFilter("dim0", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "2", "3", "4", "5")); + assertFilterMatches(JavaScriptDimFilter.of("dim0", jsValueFilter("HELLO"), lookupFn), ImmutableList.of("1")); + assertFilterMatches(JavaScriptDimFilter.of("dim0", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "2", "3", "4", "5")); - assertFilterMatches(new JavaScriptDimFilter("dim1", jsValueFilter("HELLO"), lookupFn), ImmutableList.of("3", "4")); - assertFilterMatches(new JavaScriptDimFilter("dim1", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "5")); + assertFilterMatches(JavaScriptDimFilter.of("dim1", jsValueFilter("HELLO"), lookupFn), ImmutableList.of("3", "4")); + assertFilterMatches(JavaScriptDimFilter.of("dim1", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "5")); - assertFilterMatches(new JavaScriptDimFilter("dim2", jsValueFilter("HELLO"), lookupFn), ImmutableList.of("0", "3")); - assertFilterMatches(new JavaScriptDimFilter("dim2", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "4", "5")); + assertFilterMatches(JavaScriptDimFilter.of("dim2", jsValueFilter("HELLO"), lookupFn), ImmutableList.of("0", "3")); + assertFilterMatches(JavaScriptDimFilter.of("dim2", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "4", "5")); - assertFilterMatches(new JavaScriptDimFilter("dim3", jsValueFilter("HELLO"), lookupFn), ImmutableList.of()); - assertFilterMatches(new JavaScriptDimFilter("dim3", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); + assertFilterMatches(JavaScriptDimFilter.of("dim3", jsValueFilter("HELLO"), lookupFn), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim3", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); - assertFilterMatches(new JavaScriptDimFilter("dim4", jsValueFilter("HELLO"), lookupFn), ImmutableList.of()); - assertFilterMatches(new JavaScriptDimFilter("dim4", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); + assertFilterMatches(JavaScriptDimFilter.of("dim4", jsValueFilter("HELLO"), lookupFn), ImmutableList.of()); + assertFilterMatches(JavaScriptDimFilter.of("dim4", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); } private void assertFilterMatches(