Query vectorization.

.travis.yml

@@ -60,7 +60,7 @@ matrix:
       before_script: unset _JAVA_OPTIONS
       script:
         # Set MAVEN_OPTS for Surefire launcher
-        - MAVEN_OPTS='-Xmx512m' mvn test -B -pl processing
+        - MAVEN_OPTS='-Xmx800m' mvn test -B -pl processing
         - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
         - free -m
 
@@ -71,7 +71,7 @@ matrix:
       before_script: unset _JAVA_OPTIONS
       script:
         # Set MAVEN_OPTS for Surefire launcher
-        - MAVEN_OPTS='-Xmx512m' mvn test -B -Ddruid.generic.useDefaultValueForNull=false -pl processing
+        - MAVEN_OPTS='-Xmx800m' mvn test -B -Ddruid.generic.useDefaultValueForNull=false -pl processing
         - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
         - free -m
 
@@ -82,7 +82,7 @@ matrix:
       before_script: unset _JAVA_OPTIONS
       script:
         # Set MAVEN_OPTS for Surefire launcher
-        - MAVEN_OPTS='-Xmx512m' mvn test -B -pl server
+        - MAVEN_OPTS='-Xmx800m' mvn test -B -pl server
 
       # server module test with SQL Compatibility enabled
     - env:
@@ -91,7 +91,7 @@ matrix:
       before_script: unset _JAVA_OPTIONS
       script:
         # Set MAVEN_OPTS for Surefire launcher
-        - MAVEN_OPTS='-Xmx512m' mvn test -B -pl server -Ddruid.generic.useDefaultValueForNull=false
+        - MAVEN_OPTS='-Xmx800m' mvn test -B -pl server -Ddruid.generic.useDefaultValueForNull=false
 
 
       # other modules test
@@ -101,7 +101,7 @@ matrix:
       before_script: unset _JAVA_OPTIONS
       script:
         # Set MAVEN_OPTS for Surefire launcher
-        - MAVEN_OPTS='-Xmx512m' mvn test -B -pl '!processing,!server'
+        - MAVEN_OPTS='-Xmx800m' mvn test -B -pl '!processing,!server'
         - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
         - free -m
 
@@ -112,7 +112,7 @@ matrix:
       before_script: unset _JAVA_OPTIONS
       script:
         # Set MAVEN_OPTS for Surefire launcher
-        - MAVEN_OPTS='-Xmx512m' mvn test -B -Ddruid.generic.useDefaultValueForNull=false -pl '!processing,!server'
+        - MAVEN_OPTS='-Xmx800m' mvn test -B -Ddruid.generic.useDefaultValueForNull=false -pl '!processing,!server'
         - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
         - free -m
 

benchmarks/pom.xml

@@ -83,12 +83,24 @@
       <version>${project.parent.version}</version>
       <type>test-jar</type>
     </dependency>
+    <dependency>
+      <groupId>org.apache.druid</groupId>
+      <artifactId>druid-core</artifactId>
+      <version>${project.parent.version}</version>
+      <type>test-jar</type>
+    </dependency>
     <dependency>
       <groupId>org.apache.druid</groupId>
       <artifactId>druid-processing</artifactId>
       <version>${project.parent.version}</version>
       <type>test-jar</type>
     </dependency>
+    <dependency>
+      <groupId>org.apache.druid</groupId>
+      <artifactId>druid-server</artifactId>
+      <version>${project.parent.version}</version>
+      <type>test-jar</type>
+    </dependency>
     <dependency>
       <groupId>org.apache.druid</groupId>
       <artifactId>druid-sql</artifactId>

benchmarks/src/main/java/org/apache/druid/benchmark/ExpressionAggregationBenchmark.java

@@ -27,6 +27,7 @@
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.js.JavaScriptConfig;
 import org.apache.druid.query.aggregation.BufferAggregator;
 import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
@@ -73,15 +74,17 @@ public class ExpressionAggregationBenchmark
   @Param({"1000000"})
   private int rowsPerSegment;
 
-  private SegmentGenerator segmentGenerator;
   private QueryableIndex index;
   private JavaScriptAggregatorFactory javaScriptAggregatorFactory;
   private DoubleSumAggregatorFactory expressionAggregatorFactory;
   private ByteBuffer aggregationBuffer = ByteBuffer.allocate(Double.BYTES);
+  private Closer closer;
 
   @Setup(Level.Trial)
   public void setup()
   {
+    this.closer = Closer.create();
+
     final BenchmarkSchemaInfo schemaInfo = new BenchmarkSchemaInfo(
         ImmutableList.of(
             BenchmarkColumnSchema.makeNormal("x", ValueType.FLOAT, false, 1, 0d, 0d, 10000d, false),
@@ -99,8 +102,10 @@ public void setup()
                                                .shardSpec(new LinearShardSpec(0))
                                                .build();
 
-    this.segmentGenerator = new SegmentGenerator();
-    this.index = segmentGenerator.generate(dataSegment, schemaInfo, Granularities.NONE, rowsPerSegment);
+    final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
+    this.index = closer.register(
+        segmentGenerator.generate(dataSegment, schemaInfo, Granularities.NONE, rowsPerSegment)
+    );
     this.javaScriptAggregatorFactory = new JavaScriptAggregatorFactory(
         "name",
         ImmutableList.of("x", "y"),
@@ -120,15 +125,7 @@ public void setup()
   @TearDown(Level.Trial)
   public void tearDown() throws Exception
   {
-    if (index != null) {
-      index.close();
-      index = null;
-    }
-
-    if (segmentGenerator != null) {
-      segmentGenerator.close();
-      segmentGenerator = null;
-    }
+    closer.close();
   }
 
   @Benchmark

benchmarks/src/main/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java

@@ -26,6 +26,7 @@
 import org.apache.druid.java.util.common.Intervals;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.java.util.common.io.Closer;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
 import org.apache.druid.query.dimension.ExtractionDimensionSpec;
 import org.apache.druid.query.expression.TestExprMacroTable;
@@ -72,12 +73,14 @@ public class ExpressionSelectorBenchmark
   @Param({"1000000"})
   private int rowsPerSegment;
 
-  private SegmentGenerator segmentGenerator;
   private QueryableIndex index;
+  private Closer closer;
 
   @Setup(Level.Trial)
   public void setup()
   {
+    this.closer = Closer.create();
+
     final BenchmarkSchemaInfo schemaInfo = new BenchmarkSchemaInfo(
         ImmutableList.of(
             BenchmarkColumnSchema.makeZipf(
@@ -113,22 +116,16 @@ public void setup()
                                                .shardSpec(new LinearShardSpec(0))
                                                .build();
 
-    this.segmentGenerator = new SegmentGenerator();
-    this.index = segmentGenerator.generate(dataSegment, schemaInfo, Granularities.HOUR, rowsPerSegment);
+    final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
+    this.index = closer.register(
+        segmentGenerator.generate(dataSegment, schemaInfo, Granularities.HOUR, rowsPerSegment)
+    );
   }
 
   @TearDown(Level.Trial)
   public void tearDown() throws Exception
   {
-    if (index != null) {
-      index.close();
-      index = null;
-    }
-
-    if (segmentGenerator != null) {
-      segmentGenerator.close();
-      segmentGenerator = null;
-    }
+    closer.close();
   }
 
   @Benchmark

benchmarks/src/main/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java

@@ -20,6 +20,7 @@
 package org.apache.druid.benchmark;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.io.Files;
 import org.apache.commons.io.FileUtils;
 import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator;
@@ -32,7 +33,6 @@
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.guava.Sequence;
 import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.js.JavaScriptConfig;
 import org.apache.druid.query.Druids;
 import org.apache.druid.query.FinalizeResultsQueryRunner;
 import org.apache.druid.query.Query;
@@ -48,7 +48,6 @@
 import org.apache.druid.query.filter.BoundDimFilter;
 import org.apache.druid.query.filter.DimFilter;
 import org.apache.druid.query.filter.InDimFilter;
-import org.apache.druid.query.filter.JavaScriptDimFilter;
 import org.apache.druid.query.filter.OrDimFilter;
 import org.apache.druid.query.filter.RegexDimFilter;
 import org.apache.druid.query.filter.SearchQueryDimFilter;
@@ -107,6 +106,9 @@ public class FilteredAggregatorBenchmark
   @Param({"basic"})
   private String schema;
 
+  @Param({"false", "true"})
+  private String vectorize;
+
   private static final Logger log = new Logger(FilteredAggregatorBenchmark.class);
   private static final int RNG_SEED = 9999;
   private static final IndexMergerV9 INDEX_MERGER_V9;
@@ -161,12 +163,6 @@ public void setup() throws IOException
     filter = new OrDimFilter(
         Arrays.asList(
             new BoundDimFilter("dimSequential", "-1", "-1", true, true, null, null, StringComparators.ALPHANUMERIC),
-            new JavaScriptDimFilter(
-                "dimSequential",
-                "function(x) { return false }",
-                null,
-                JavaScriptConfig.getEnabledInstance()
-            ),
             new RegexDimFilter("dimSequential", "X", null),
             new SearchQueryDimFilter("dimSequential", new ContainsSearchQuerySpec("X", false), null),
             new InDimFilter("dimSequential", Collections.singletonList("X"), null)
@@ -234,15 +230,18 @@ private IncrementalIndex makeIncIndex(AggregatorFactory[] metrics)
         .buildOnheap();
   }
 
-  private static <T> List<T> runQuery(QueryRunnerFactory factory, QueryRunner runner, Query<T> query)
+  private static <T> List<T> runQuery(QueryRunnerFactory factory, QueryRunner runner, Query<T> query, String vectorize)
   {
     QueryToolChest toolChest = factory.getToolchest();
     QueryRunner<T> theRunner = new FinalizeResultsQueryRunner<>(
         toolChest.mergeResults(toolChest.preMergeQueryDecoration(runner)),
         toolChest
     );
 
-    Sequence<T> queryResult = theRunner.run(QueryPlus.wrap(query), new HashMap<>());
+    final QueryPlus<T> queryToRun = QueryPlus.wrap(
+        query.withOverriddenContext(ImmutableMap.of("vectorize", vectorize))
+    );
+    Sequence<T> queryResult = theRunner.run(queryToRun, new HashMap<>());
     return queryResult.toList();
   }
 
@@ -269,7 +268,12 @@ public void querySingleIncrementalIndex(Blackhole blackhole)
         new IncrementalIndexSegment(incIndex, SegmentId.dummy("incIndex"))
     );
 
-    List<Result<TimeseriesResultValue>> results = FilteredAggregatorBenchmark.runQuery(factory, runner, query);
+    List<Result<TimeseriesResultValue>> results = FilteredAggregatorBenchmark.runQuery(
+        factory,
+        runner,
+        query,
+        vectorize
+    );
     for (Result<TimeseriesResultValue> result : results) {
       blackhole.consume(result);
     }
@@ -286,7 +290,12 @@ public void querySingleQueryableIndex(Blackhole blackhole)
         new QueryableIndexSegment(qIndex, SegmentId.dummy("qIndex"))
     );
 
-    List<Result<TimeseriesResultValue>> results = FilteredAggregatorBenchmark.runQuery(factory, runner, query);
+    List<Result<TimeseriesResultValue>> results = FilteredAggregatorBenchmark.runQuery(
+        factory,
+        runner,
+        query,
+        vectorize
+    );
     for (Result<TimeseriesResultValue> result : results) {
       blackhole.consume(result);
     }

benchmarks/src/main/java/org/apache/druid/benchmark/datagen/BenchmarkColumnSchema.java

@@ -424,4 +424,26 @@ public static BenchmarkColumnSchema makeEnumerated(
     schema.enumeratedProbabilities = enumeratedProbabilities;
     return schema;
   }
+
+  @Override
+  public String toString()
+  {
+    return "BenchmarkColumnSchema{" +
+           "distributionType=" + distributionType +
+           ", name='" + name + '\'' +
+           ", type=" + type +
+           ", isMetric=" + isMetric +
+           ", rowSize=" + rowSize +
+           ", nullProbability=" + nullProbability +
+           ", enumeratedValues=" + enumeratedValues +
+           ", enumeratedProbabilities=" + enumeratedProbabilities +
+           ", startInt=" + startInt +
+           ", endInt=" + endInt +
+           ", startDouble=" + startDouble +
+           ", endDouble=" + endDouble +
+           ", zipfExponent=" + zipfExponent +
+           ", mean=" + mean +
+           ", standardDeviation=" + standardDeviation +
+           '}';
+  }
 }

benchmarks/src/main/java/org/apache/druid/benchmark/datagen/BenchmarkSchemaInfo.java

@@ -68,4 +68,15 @@ public boolean isWithRollup()
   {
     return withRollup;
   }
+
+  @Override
+  public String toString()
+  {
+    return "BenchmarkSchemaInfo{" +
+           "columnSchemas=" + columnSchemas +
+           ", aggs=" + aggs +
+           ", dataInterval=" + dataInterval +
+           ", withRollup=" + withRollup +
+           '}';
+  }
 }

benchmarks/src/main/java/org/apache/druid/benchmark/datagen/BenchmarkSchemas.java

@@ -94,7 +94,7 @@ public class BenchmarkSchemas
     basicSchemaIngestAggsExpression.add(new DoubleMinAggregatorFactory("minFloatZipf", "metFloatZipf"));
     basicSchemaIngestAggsExpression.add(new HyperUniquesAggregatorFactory("hyper", "dimHyperUnique"));
 
-    Interval basicSchemaDataInterval = Intervals.utc(0, 1000000);
+    Interval basicSchemaDataInterval = Intervals.of("2000-01-01/P1D");
 
     BenchmarkSchemaInfo basicSchema = new BenchmarkSchemaInfo(
         basicSchemaColumns,