Javascript filter with multiple columns

docs/content/querying/filters.md

@@ -58,15 +58,15 @@ The filter specified at field can be any other filter defined on this page.
 
 ### JavaScript filter
 
-The JavaScript filter matches a dimension against the specified JavaScript function predicate. The filter matches values for which the function returns true.
+The JavaScript filter matches dimensions against the specified JavaScript function predicate. The filter matches values for which the function returns true.
 
-The function takes a single argument, the dimension value, and returns either true or false.
+The function takes the same number of arguments as the dimension values, and returns either true or false.
 
 ```json
 {
   "type" : "javascript",
-  "dimension" : <dimension_string>,
-  "function" : "function(value) { <...> }"
+  "dimensions" : <array of dimension_strings>,
+  "function" : "function(value1, value2, ...) { <...> }"
 }
 ```
 
@@ -81,6 +81,15 @@ The following matches any dimension values for the dimension `name` between `'ba
 }
 ```
 
+The following matches values of the given two dimensions `dim1` and `dim2` are the same
+```json
+{
+  "type" : "javascript",
+  "dimensions" : ["dim1","dim2"]
+  "function" : "function(x, y) { return x === y }"
+}
+```
+
 ### Extraction filter
 
 Extraction filter matches a dimension using some specific [Extraction function](./dimensionspecs.html#extraction-functions).

processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java

@@ -21,32 +21,36 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.metamx.common.StringUtils;
 
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 
 public class JavaScriptDimFilter implements DimFilter
 {
-  private final String dimension;
+  private final String[] dimensions;
   private final String function;
 
   @JsonCreator
   public JavaScriptDimFilter(
+      // for backwards compatibility
       @JsonProperty("dimension") String dimension,
+      @JsonProperty("dimensions") String[] dimensions,
       @JsonProperty("function") String function
   )
   {
-    Preconditions.checkArgument(dimension != null, "dimension must not be null");
+    Preconditions.checkArgument(dimension != null ^ dimensions != null, "dimensions(xor dimension) must not be null");
     Preconditions.checkArgument(function != null, "function must not be null");
-    this.dimension = dimension;
+    this.dimensions = (dimensions != null) ? dimensions : new String[]{dimension};
     this.function = function;
   }
 
   @JsonProperty
-  public String getDimension()
+  public String[] getDimensions()
   {
-    return dimension;
+    return dimensions;
   }
 
   @JsonProperty
@@ -58,14 +62,22 @@ public String getFunction()
   @Override
   public byte[] getCacheKey()
   {
-    final byte[] dimensionBytes = StringUtils.toUtf8(dimension);
     final byte[] functionBytes = StringUtils.toUtf8(function);
+    byte[][] dimensionsBytes = new byte[dimensions.length][];
+    int totalDimensionsBytes = 0;
 
-    return ByteBuffer.allocate(2 + dimensionBytes.length + functionBytes.length)
-                     .put(DimFilterCacheHelper.JAVASCRIPT_CACHE_ID)
-                     .put(dimensionBytes)
-                     .put(DimFilterCacheHelper.STRING_SEPARATOR)
-                     .put(functionBytes)
+    for (int idx = 0; idx < dimensions.length; idx++) {
+      dimensionsBytes[idx] = StringUtils.toUtf8(dimensions[idx]);
+      totalDimensionsBytes += dimensionsBytes[idx].length;
+    }
+
+    ByteBuffer byteBuffer = ByteBuffer.allocate(2 + dimensions.length + totalDimensionsBytes + functionBytes.length)
+                     .put(DimFilterCacheHelper.JAVASCRIPT_CACHE_ID);
+    for (byte[] dimBytes: dimensionsBytes) {
+      byteBuffer.put(dimBytes)
+                .put(DimFilterCacheHelper.STRING_SEPARATOR);
+    }
+    return byteBuffer.put(functionBytes)
                      .array();
   }
 
@@ -79,8 +91,26 @@ public DimFilter optimize()
   public String toString()
   {
     return "JavaScriptDimFilter{" +
-           "dimension='" + dimension + '\'' +
+           "dimensions=['" + Joiner.on("', '").join(dimensions) + "']" +
            ", function='" + function + '\'' +
            '}';
   }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof JavaScriptDimFilter)) {
+      return false;
+    }
+
+    JavaScriptDimFilter that = (JavaScriptDimFilter) o;
+
+    if (!function.equals(that.function)) {
+      return false;
+    }
+    return Arrays.equals(that.dimensions, dimensions);
+  }
 }

processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java

@@ -28,5 +28,6 @@ public interface ValueMatcherFactory
 {
   public ValueMatcher makeValueMatcher(String dimension, String value);
   public ValueMatcher makeValueMatcher(String dimension, Predicate<String> value);
+  public ValueMatcher makeValueMatcher(String[] dimensions, Predicate<String[]> value);
   public ValueMatcher makeValueMatcher(String dimension, Bound bound);
 }

processing/src/main/java/io/druid/segment/filter/Filters.java

@@ -93,7 +93,7 @@ public static Filter convertDimensionFilters(DimFilter dimFilter)
     } else if (dimFilter instanceof JavaScriptDimFilter) {
       final JavaScriptDimFilter javaScriptDimFilter = (JavaScriptDimFilter) dimFilter;
 
-      filter = new JavaScriptFilter(javaScriptDimFilter.getDimension(), javaScriptDimFilter.getFunction());
+      filter = new JavaScriptFilter(javaScriptDimFilter.getDimensions(), javaScriptDimFilter.getFunction());
     } else if (dimFilter instanceof SpatialDimFilter) {
       final SpatialDimFilter spatialDimFilter = (SpatialDimFilter) dimFilter;
 

processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java

@@ -22,7 +22,6 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
 import com.metamx.collections.bitmap.ImmutableBitmap;
-import com.metamx.common.guava.FunctionalIterable;
 import io.druid.query.filter.BitmapIndexSelector;
 import io.druid.query.filter.Filter;
 import io.druid.query.filter.ValueMatcher;
@@ -33,14 +32,14 @@
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptableObject;
 
-import javax.annotation.Nullable;
+import java.util.Iterator;
 
 public class JavaScriptFilter implements Filter
 {
   private final JavaScriptPredicate predicate;
-  private final String dimension;
+  private final String[] dimension;
 
-  public JavaScriptFilter(String dimension, final String script)
+  public JavaScriptFilter(String[] dimension, final String script)
   {
     this.dimension = dimension;
     this.predicate = new JavaScriptPredicate(script);
@@ -51,34 +50,57 @@ public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector)
   {
     final Context cx = Context.enter();
     try {
-      final Indexed<String> dimValues = selector.getDimensionValues(dimension);
       ImmutableBitmap bitmap;
-      if (dimValues == null) {
-        bitmap = selector.getBitmapFactory().makeEmptyImmutableBitmap();
-      } else {
-        bitmap = selector.getBitmapFactory().union(
-            FunctionalIterable.create(dimValues)
-                              .filter(
-                                  new Predicate<String>()
-                                  {
-                                    @Override
-                                    public boolean apply(@Nullable String input)
-                                    {
-                                      return predicate.applyInContext(cx, input);
-                                    }
-                                  }
-                              )
-                              .transform(
-                                  new com.google.common.base.Function<String, ImmutableBitmap>()
-                                  {
-                                    @Override
-                                    public ImmutableBitmap apply(@Nullable String input)
-                                    {
-                                      return selector.getBitmapIndex(dimension, input);
-                                    }
-                                  }
-                              )
-        );
+
+      boolean hasEmptyDimension = false;
+      Indexed<String> [] dimValuesList = new Indexed[dimension.length];
+      Iterator<String> [] dimValuesIterator = new Iterator[dimension.length];
+      String[] currentDim = new String[dimension.length];
+      for(int idx = 0; idx < dimension.length; idx++) {
+        dimValuesList[idx] = selector.getDimensionValues(dimension[idx]);
+        if (dimValuesList[idx].size() == 0) {
+          hasEmptyDimension = true;
+          break;
+        }
+        dimValuesIterator[idx] = dimValuesList[idx].iterator();
+        if (idx != 0) {
+          currentDim[idx] = dimValuesIterator[idx].next();
+        }
+      }
+
+      bitmap = selector.getBitmapFactory().makeEmptyImmutableBitmap();
+      if (!hasEmptyDimension) {
+        int iteratingIndex = 0;
+        while(true) {
+          // advance iterator
+          Iterator<String> iterator = dimValuesIterator[iteratingIndex];
+          if (iterator.hasNext()) {
+            currentDim[iteratingIndex] = iterator.next();
+            if (iteratingIndex > 0) {
+              // reset inner loop iterators
+              for (int idx = 0; idx < iteratingIndex; idx++) {
+                dimValuesIterator[idx] = dimValuesList[idx].iterator();
+                currentDim[idx] = dimValuesIterator[idx].next();
+              }
+              // move to the most inner loop
+              iteratingIndex = 0;
+            }
+            if (predicate.applyInContext(cx, currentDim))
+            {
+              // update bitmap
+              ImmutableBitmap overlap = null;
+              for (int idx = 0; idx < dimension.length; idx++) {
+                ImmutableBitmap dimBitMap = selector.getBitmapIndex(dimension[idx], currentDim[idx]);
+                overlap = (overlap == null) ? dimBitMap : overlap.intersection(dimBitMap);
+              }
+              bitmap = bitmap.union(overlap);
+            }
+
+          } else {
+            iteratingIndex++;
+            if (iteratingIndex == dimension.length) break;
+          }
+        }
       }
       return bitmap;
     }
@@ -94,7 +116,7 @@ public ValueMatcher makeMatcher(ValueMatcherFactory factory)
     return factory.makeValueMatcher(dimension, predicate);
   }
 
-  static class JavaScriptPredicate implements Predicate<String>
+  static class JavaScriptPredicate implements Predicate<String[]>
   {
     final ScriptableObject scope;
     final Function fnApply;
@@ -118,7 +140,7 @@ public JavaScriptPredicate(final String script)
     }
 
     @Override
-    public boolean apply(final String input)
+    public boolean apply(final String[] input)
     {
       // one and only one context per thread
       final Context cx = Context.enter();
@@ -131,9 +153,9 @@ public boolean apply(final String input)
 
     }
 
-    public boolean applyInContext(Context cx, String input)
+    public boolean applyInContext(Context cx, String[] input)
     {
-      return Context.toBoolean(fnApply.call(cx, scope, scope, new String[]{input}));
+      return Context.toBoolean(fnApply.call(cx, scope, scope, input));
     }
 
     @Override

processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java

@@ -684,6 +684,66 @@ public boolean matches()
       };
     }
 
+    @Override
+    public ValueMatcher makeValueMatcher(final String[] dimensions, final Predicate<String[]> predicate)
+    {
+      Integer[] dimIndexObject = new Integer[dimensions.length];
+      final int[] dimIndex = new int[dimensions.length];
+      for (int idx = 0; idx < dimensions.length; idx++) {
+        dimIndexObject[idx] = index.getDimensionIndex(dimensions[idx]);
+        if (dimIndexObject[idx] == null) {
+          return new BooleanValueMatcher(false);
+        }
+        dimIndex[idx] = dimIndexObject[idx];
+      }
+
+      return new ValueMatcher()
+      {
+        @Override
+        public boolean matches()
+        {
+          String[][] dims = holder.getKey().getDims();
+          String[][] selected = new String[dimensions.length][];
+          String[] current = new String[dimensions.length];
+          int[] currentIdx = new int[dimensions.length];
+          for (int idx = 0; idx < dimensions.length; idx++) {
+            if (dimIndex[idx] >= dims.length || dims[dimIndex[idx]] == null) {
+              return predicate.apply(null);
+            }
+            selected[idx] = dims[dimIndex[idx]];
+            current[idx] = selected[idx][0];
+            currentIdx[idx] = 0;
+          }
+
+          if (predicate.apply(current)) {
+            return true;
+          }
+
+          int currentIteratorIdx = 0;
+          while (true) {
+            currentIdx[currentIteratorIdx]++;
+            if (currentIdx[currentIteratorIdx] < selected[currentIteratorIdx].length) {
+              current[currentIteratorIdx] = selected[currentIteratorIdx][currentIdx[currentIteratorIdx]];
+              if (currentIteratorIdx > 0) {
+                for (int idx = 0; idx < currentIteratorIdx; idx++) {
+                  currentIdx[idx] = 0;
+                  current[idx] = selected[idx][0];
+                }
+                currentIteratorIdx = 0;
+              }
+              if (predicate.apply(current)) {
+                return true;
+              }
+            } else {
+              currentIteratorIdx++;
+              if (currentIteratorIdx == dimensions.length) break;
+            }
+          }
+          return false;
+        }
+      };
+    }
+
     @Override
     public ValueMatcher makeValueMatcher(final String dimension, final Bound bound)
     {