More methods in QueryMetrics and TopNQueryMetrics (the last part of #3798)

benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java

@@ -508,7 +508,7 @@ public void readComplexOrFilterCNF(Blackhole blackhole) throws Exception
 
   private Sequence<Cursor> makeCursors(StorageAdapter sa, Filter filter)
   {
-    return sa.makeCursors(filter, schemaInfo.getDataInterval(), VirtualColumns.EMPTY, Granularities.ALL, false);
+    return sa.makeCursors(filter, schemaInfo.getDataInterval(), VirtualColumns.EMPTY, Granularities.ALL, false, null);
   }
 
   private Sequence<List<String>> readCursors(Sequence<Cursor> cursors, final Blackhole blackhole)

benchmarks/src/main/java/io/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java

@@ -205,7 +205,8 @@ private Sequence<Cursor> makeCursors(IncrementalIndexStorageAdapter sa, DimFilte
         schemaInfo.getDataInterval(),
         VirtualColumns.EMPTY,
         Granularities.ALL,
-        false
+        false,
+        null
     );
   }
 }

extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountTopNQueryTest.java

@@ -112,10 +112,7 @@ public ByteBuffer get()
                           .build();
 
     final Iterable<Result<TopNResultValue>> results = Sequences.toList(
-        engine.query(
-            query,
-            new IncrementalIndexStorageAdapter(index)
-        ),
+        engine.query(query, new IncrementalIndexStorageAdapter(index), null),
         Lists.<Result<TopNResultValue>>newLinkedList()
     );
 

extensions-contrib/scan-query/src/main/java/io/druid/query/scan/ScanQueryEngine.java

@@ -116,7 +116,8 @@ public Sequence<ScanResultValue> process(
                 intervals.get(0),
                 VirtualColumns.EMPTY,
                 Granularities.ALL,
-                query.isDescending()
+                query.isDescending(),
+                null
             ),
             new Function<Cursor, Sequence<ScanResultValue>>()
             {

processing/src/main/java/io/druid/query/BitmapResultFactory.java

@@ -0,0 +1,107 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query;
+
+import io.druid.collections.bitmap.BitmapFactory;
+import io.druid.collections.bitmap.ImmutableBitmap;
+
+/**
+ * BitmapResultFactory is an abstraction that allows to record something along with preFilter bitmap construction, and
+ * emit this information as dimension(s) of query metrics. BitmapResultFactory is similar to {@link
+ * io.druid.collections.bitmap.BitmapFactory}: it has the same methods with the exception that it accepts generic type
+ * T (bitmap wrapper type) instead of {@link ImmutableBitmap}.
+ *
+ * {@link DefaultBitmapResultFactory} is a no-op implementation, where "wrapper" type is {@code ImmutableBitmap} itself.
+ *
+ * BitmapResultFactory delegates actual operations on bitmaps to a {@code BitmapFactory}, which it accepts in
+ * constructor, called from {@link QueryMetrics#makeBitmapResultFactory(BitmapFactory)}.
+ *
+ * Emitting of query metric dimension(s) should be done from {@link #toImmutableBitmap(Object)}, the "unwrapping"
+ * method, called only once to obtain the final preFilter bitmap.
+ *
+ * Implementors expectations
+ * -------------------------
+ * BitmapResultFactory is a part of the {@link QueryMetrics} subsystem, so this interface could be changed often, in
+ * every Druid release (including "patch" releases). Users who create their custom implementations of
+ * BitmapResultFactory should be ready to fix the code of their code to accommodate interface changes (e. g. implement
+ * new methods) when they update Druid. See {@link QueryMetrics} Javadoc for more info.
+ *
+ * @param <T> the bitmap result (wrapper) type
+ * @see QueryMetrics#makeBitmapResultFactory(BitmapFactory)
+ * @see QueryMetrics#reportBitmapConstructionTime(long)
+ */
+public interface BitmapResultFactory<T>
+{
+  /**
+   * Wraps a bitmap of unknown nature.
+   */
+  T wrapUnknown(ImmutableBitmap bitmap);
+
+  /**
+   * Wraps a bitmap which designates rows in a segment with some specific dimension value.
+   */
+  T wrapDimensionValue(ImmutableBitmap bitmap);
+
+  /**
+   * Wraps a bitmap which is a result of {@link BitmapFactory#makeEmptyImmutableBitmap()} call.
+   */
+  T wrapAllFalse(ImmutableBitmap allFalseBitmap);
+
+  /**
+   * Wraps a bitmap which is a result of {@link BitmapFactory#complement(ImmutableBitmap)} called with
+   * {@link BitmapFactory#makeEmptyImmutableBitmap()} as argument.
+   */
+  T wrapAllTrue(ImmutableBitmap allTrueBitmap);
+
+  /**
+   * Checks that the wrapped bitmap is empty, see {@link ImmutableBitmap#isEmpty()}.
+   */
+  boolean isEmpty(T bitmapResult);
+
+  /**
+   * Delegates to {@link BitmapFactory#intersection(Iterable)} on the wrapped bitmaps, and returns a bitmap result
+   * wrapping the resulting intersection ImmutableBitmap.
+   */
+  T intersection(Iterable<T> bitmapResults);
+
+  /**
+   * Delegates to {@link BitmapFactory#union(Iterable)} on the wrapped bitmaps, and returns a bitmap result wrapping
+   * the resulting union ImmutableBitmap.
+   */
+  T union(Iterable<T> bitmapResults);
+
+  /**
+   * Equivalent of intersection(Iterables.transform(dimensionValueBitmaps, factory::wrapDimensionValue)), but doesn't
+   * create a lot of bitmap result objects.
+   */
+  T unionDimensionValueBitmaps(Iterable<ImmutableBitmap> dimensionValueBitmaps);
+
+  /**
+   * Delegates to {@link BitmapFactory#complement(ImmutableBitmap, int)} on the wrapped bitmap, and returns a bitmap
+   * result wrapping the resulting complement ImmutableBitmap.
+   */
+  T complement(T bitmapResult, int numRows);
+
+  /**
+   * Unwraps bitmapResult back to ImmutableBitmap. BitmapResultFactory should emit query metric dimension(s) in the
+   * implementation of this method.
+   */
+  ImmutableBitmap toImmutableBitmap(T bitmapResult);
+}

processing/src/main/java/io/druid/query/DefaultBitmapResultFactory.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query;
+
+import io.druid.collections.bitmap.BitmapFactory;
+import io.druid.collections.bitmap.ImmutableBitmap;
+
+public final class DefaultBitmapResultFactory implements BitmapResultFactory<ImmutableBitmap>
+{
+  private final BitmapFactory factory;
+
+  public DefaultBitmapResultFactory(BitmapFactory factory)
+  {
+    this.factory = factory;
+  }
+
+  @Override
+  public ImmutableBitmap wrapUnknown(ImmutableBitmap bitmap)
+  {
+    return bitmap;
+  }
+
+  @Override
+  public ImmutableBitmap wrapDimensionValue(ImmutableBitmap bitmap)
+  {
+    return bitmap;
+  }
+
+  @Override
+  public ImmutableBitmap wrapAllFalse(ImmutableBitmap allFalseBitmap)
+  {
+    return allFalseBitmap;
+  }
+
+  @Override
+  public ImmutableBitmap wrapAllTrue(ImmutableBitmap allTrueBitmap)
+  {
+    return allTrueBitmap;
+  }
+
+  @Override
+  public boolean isEmpty(ImmutableBitmap bitmapResult)
+  {
+    return bitmapResult.isEmpty();
+  }
+
+  @Override
+  public ImmutableBitmap intersection(Iterable<ImmutableBitmap> bitmapResults)
+  {
+    return factory.intersection(bitmapResults);
+  }
+
+  @Override
+  public ImmutableBitmap union(Iterable<ImmutableBitmap> bitmapResults)
+  {
+    return factory.union(bitmapResults);
+  }
+
+  @Override
+  public ImmutableBitmap unionDimensionValueBitmaps(Iterable<ImmutableBitmap> dimensionValueBitmaps)
+  {
+    return factory.union(dimensionValueBitmaps);
+  }
+
+  @Override
+  public ImmutableBitmap complement(ImmutableBitmap bitmapResult, int numRows)
+  {
+    return factory.complement(bitmapResult, numRows);
+  }
+
+  @Override
+  public ImmutableBitmap toImmutableBitmap(ImmutableBitmap bitmapResult)
+  {
+    return bitmapResult;
+  }
+}

processing/src/main/java/io/druid/query/DefaultQueryMetrics.java

@@ -25,9 +25,12 @@
 import com.google.common.collect.ImmutableMap;
 import com.metamx.emitter.service.ServiceEmitter;
 import com.metamx.emitter.service.ServiceMetricEvent;
+import io.druid.collections.bitmap.BitmapFactory;
+import io.druid.query.filter.Filter;
 import org.joda.time.Interval;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -168,6 +171,24 @@ public void chunkInterval(Interval interval)
     setDimension("chunkInterval", interval.toString());
   }
 
+  @Override
+  public void preFilters(List<Filter> preFilters)
+  {
+    // Emit nothing by default.
+  }
+
+  @Override
+  public void postFilters(List<Filter> postFilters)
+  {
+    // Emit nothing by default.
+  }
+
+  @Override
+  public BitmapResultFactory<?> makeBitmapResultFactory(BitmapFactory factory)
+  {
+    return new DefaultBitmapResultFactory(factory);
+  }
+
   @Override
   public QueryMetrics<QueryType> reportQueryTime(long timeNs)
   {
@@ -240,6 +261,27 @@ public QueryMetrics<QueryType> reportNodeBytes(long byteCount)
     return reportMetric("query/node/bytes", byteCount);
   }
 
+  @Override
+  public QueryMetrics<QueryType> reportBitmapConstructionTime(long timeNs)
+  {
+    // Don't emit by default.
+    return this;
+  }
+
+  @Override
+  public QueryMetrics<QueryType> reportSegmentRows(long numRows)
+  {
+    // Don't emit by default.
+    return this;
+  }
+
+  @Override
+  public QueryMetrics<QueryType> reportPreFilteredRows(long numRows)
+  {
+    // Don't emit by default.
+    return this;
+  }
+
   @Override
   public void emit(ServiceEmitter emitter)
   {

processing/src/main/java/io/druid/query/QueryMetrics.java

@@ -20,8 +20,12 @@
 package io.druid.query;
 
 import com.metamx.emitter.service.ServiceEmitter;
+import io.druid.collections.bitmap.BitmapFactory;
+import io.druid.query.filter.Filter;
 import org.joda.time.Interval;
 
+import java.util.List;
+
 /**
  * Abstraction wrapping {@link com.metamx.emitter.service.ServiceMetricEvent.Builder} and allowing to control what
  * metrics are actually emitted, what dimensions do they have, etc.
@@ -196,6 +200,18 @@ public interface QueryMetrics<QueryType extends Query<?>>
 
   void chunkInterval(Interval interval);
 
+  void preFilters(List<Filter> preFilters);
+
+  void postFilters(List<Filter> postFilters);
+
+  /**
+   * Creates a {@link BitmapResultFactory} which may record some information along bitmap construction from {@link
+   * #preFilters(List)}. The returned BitmapResultFactory may add some dimensions to this QueryMetrics from it's {@link
+   * BitmapResultFactory#toImmutableBitmap(Object)} method. See {@link BitmapResultFactory} Javadoc for more
+   * information.
+   */
+  BitmapResultFactory<?> makeBitmapResultFactory(BitmapFactory factory);
+
   /**
    * Registers "query time" metric.
    */
@@ -246,6 +262,23 @@ public interface QueryMetrics<QueryType extends Query<?>>
    */
   QueryMetrics<QueryType> reportNodeBytes(long byteCount);
 
+  /**
+   * Reports the time spent constructing bitmap from {@link #preFilters(List)} of the query. Not reported, if there are
+   * no preFilters.
+   */
+  QueryMetrics<QueryType> reportBitmapConstructionTime(long timeNs);
+
+  /**
+   * Reports the total number of rows in the processed segment.
+   */
+  QueryMetrics<QueryType> reportSegmentRows(long numRows);
+
+  /**
+   * Reports the number of rows to scan in the segment after applying {@link #preFilters(List)}. If the are no
+   * preFilters, this metric is equal to {@link #reportSegmentRows(long)}.
+   */
+  QueryMetrics<QueryType> reportPreFilteredRows(long numRows);
+
   /**
    * Emits all metrics, registered since the last {@code emit()} call on this QueryMetrics object.
    */

processing/src/main/java/io/druid/query/QueryRunnerHelper.java

@@ -58,7 +58,7 @@ public static <T> Sequence<Result<T>> makeCursorBasedQuery(
 
     return Sequences.filter(
         Sequences.map(
-            adapter.makeCursors(filter, queryIntervals.get(0), virtualColumns, granularity, descending),
+            adapter.makeCursors(filter, queryIntervals.get(0), virtualColumns, granularity, descending, null),
             new Function<Cursor, Result<T>>()
             {
               @Override