Migrating bytebuffercollections from Metamarkets.

NOTICE

@@ -36,3 +36,11 @@ This product contains a modified version of TestNG 6.8.7
     * http://testng.org/license/ (Apache License, Version 2.0)
   * HOMEPAGE:
     * http://testng.org/
+
+This product contains a modified version of Metamarkets bytebuffer-collections library
+  * LICENSE:
+    * https://github.com/metamx/bytebuffer-collections/blob/master/LICENSE (Apache License, Version 2.0)
+  * HOMEPAGE:
+    * https://github.com/metamx/bytebuffer-collections
+  * COMMIT TAG:
+    * https://github.com/metamx/bytebuffer-collections/commit/3d1e7c8

benchmarks/src/main/java/io/druid/benchmark/BoundFilterBenchmark.java

@@ -19,14 +19,32 @@
 
 package io.druid.benchmark;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.FluentIterable;
-import com.metamx.collections.bitmap.BitmapFactory;
-import com.metamx.collections.bitmap.ImmutableBitmap;
-import com.metamx.collections.bitmap.MutableBitmap;
-import com.metamx.collections.bitmap.RoaringBitmapFactory;
-import com.metamx.collections.spatial.ImmutableRTree;
+
+import io.druid.collections.bitmap.BitmapFactory;
+import io.druid.collections.bitmap.ImmutableBitmap;
+import io.druid.collections.bitmap.MutableBitmap;
+import io.druid.collections.bitmap.RoaringBitmapFactory;
+import io.druid.collections.spatial.ImmutableRTree;
 import io.druid.query.filter.BitmapIndexSelector;
 import io.druid.query.filter.BoundDimFilter;
 import io.druid.query.ordering.StringComparators;
@@ -38,22 +56,6 @@
 import io.druid.segment.filter.BoundFilter;
 import io.druid.segment.serde.BitmapIndexColumnPartSupplier;
 import it.uniroma3.mat.extendedset.intset.ConciseSetUtils;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.Warmup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 @State(Scope.Benchmark)
 @Fork(value = 1)

benchmarks/src/main/java/io/druid/benchmark/ConciseComplementBenchmark.java

@@ -20,7 +20,8 @@
 package io.druid.benchmark;
 
 
-import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet;
+import java.util.concurrent.TimeUnit;
+
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Mode;
@@ -30,7 +31,7 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.infra.Blackhole;
 
-import java.util.concurrent.TimeUnit;
+import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet;
 
 @State(Scope.Benchmark)
 public class ConciseComplementBenchmark

benchmarks/src/main/java/io/druid/benchmark/DimensionPredicateFilterBenchmark.java

@@ -23,11 +23,11 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
 import com.google.common.collect.FluentIterable;
-import com.metamx.collections.bitmap.BitmapFactory;
-import com.metamx.collections.bitmap.ImmutableBitmap;
-import com.metamx.collections.bitmap.MutableBitmap;
-import com.metamx.collections.bitmap.RoaringBitmapFactory;
-import com.metamx.collections.spatial.ImmutableRTree;
+import io.druid.collections.bitmap.BitmapFactory;
+import io.druid.collections.bitmap.ImmutableBitmap;
+import io.druid.collections.bitmap.MutableBitmap;
+import io.druid.collections.bitmap.RoaringBitmapFactory;
+import io.druid.collections.spatial.ImmutableRTree;
 import io.druid.query.filter.BitmapIndexSelector;
 import io.druid.query.filter.DruidLongPredicate;
 import io.druid.query.filter.DruidPredicateFactory;

benchmarks/src/main/java/io/druid/benchmark/LikeFilterBenchmark.java

@@ -21,11 +21,11 @@
 
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
-import com.metamx.collections.bitmap.BitmapFactory;
-import com.metamx.collections.bitmap.ImmutableBitmap;
-import com.metamx.collections.bitmap.MutableBitmap;
-import com.metamx.collections.bitmap.RoaringBitmapFactory;
-import com.metamx.collections.spatial.ImmutableRTree;
+import io.druid.collections.bitmap.BitmapFactory;
+import io.druid.collections.bitmap.ImmutableBitmap;
+import io.druid.collections.bitmap.MutableBitmap;
+import io.druid.collections.bitmap.RoaringBitmapFactory;
+import io.druid.collections.spatial.ImmutableRTree;
 import io.druid.query.filter.BitmapIndexSelector;
 import io.druid.query.filter.BoundDimFilter;
 import io.druid.query.filter.Filter;

bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html

@@ -0,0 +1,145 @@
+<!--
+  ~ Licensed to Metamarkets Group Inc. (Metamarkets) under one
+  ~ or more contributor license agreements. See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership. Metamarkets licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License. You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied. See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
+    <title>Benchmark results for methods in class RangeBitmapBenchmarkTest</title>
+
+    <script type="text/javascript" src="http://www.google.com/jsapi"></script>
+    <script type="text/javascript">
+      google.load("jquery", "1.4.2");
+      google.load('visualization', '1', {packages: ['linechart', 'table']});
+
+
+    </script>
+
+    <style type="text/css">
+        #content {
+          width: 800px ;
+          margin-left: auto;
+          margin-right: auto;
+        }
+
+        #chart {
+          width: 600px;
+          height: 400px;
+          margin-left: auto;
+          margin-right: auto;
+          margin-bottom: 10px;
+          display: block;
+        }
+
+        ul.expander {
+          display: inline;
+          margin-left: 0;
+          padding-left: 0;
+          font-size: .8em;
+        }
+
+        ul.expander li {
+          margin-left: 0;
+          list-style: none;
+          display: inline;
+        }
+
+        ul.expander li:before {
+          content: "\25BA\0020";  // 25B8
+        }
+
+        ul.expander li.expanded:before {
+          content: "\25BC\0020";  // 25BE
+        }
+
+
+    </style>
+
+    <script type="text/javascript">
+      var chart; 
+
+      function drawData(data, labelColumn) {
+        var labelColumns = [0, 1, 2];
+        labelColumns.splice(labelColumn, 1);
+
+        var view = new google.visualization.DataView(data);
+        view.hideColumns(labelColumns);
+
+        chart.draw(view, {
+            titleY: "Round time [s]",
+
+        });
+      }
+
+      function visualize(jsonData, className) {
+        var data = new google.visualization.DataTable(jsonData);
+
+        var table = new google.visualization.Table(document.getElementById('raw-data'));
+        table.draw(data, {
+            sort: "event",
+        });
+
+        google.visualization.events.addListener(table, 'sort',
+         function(event) {
+           if (event.column >= 0 && event.column <= 2) {
+              drawData(data, event.column);
+           }
+         }
+        );
+
+        chart = new google.visualization.LineChart(document.getElementById('chart'));
+        drawData(data, 01);
+      }
+
+      var data;
+      function receiveJsonpData(jsonpData) {
+        data = jsonpData;
+      }
+
+      google.setOnLoadCallback(function() {
+          visualize(data, "RangeBitmapBenchmarkTest");
+
+          $("#properties").click(function () {
+              $(this).toggleClass("expanded"); 
+              $("#proplist").slideToggle();
+          });
+      });
+
+
+    </script>
+</head>
+
+<body style="font-family: Arial; margin: 30px;">
+<div id="content">
+    <h1>Benchmark results for methods in class RangeBitmapBenchmarkTest</h1>
+
+    <div id="chart"></div>
+    <div id="raw-data"></div>
+
+    <div>
+        <ul class="expander">
+            <li id="properties"><a href="#">Properties</a></li>
+        </ul>
+        <pre id="proplist" style="display: none; background-color: #e0e0e0;">Shows historical runs: 20</pre>
+    </div>
+</div>
+
+<script src="RangeBitmapBenchmarkTest.jsonp"></script>
+</body>
+</html>

bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp

@@ -0,0 +1,22 @@
+receiveJsonpData({
+"cols": [
+{"label": "Run", "type": "string"},
+{"label": "Custom key", "type": "string"},
+{"label": "Timestamp", "type": "string"},
+{"label": "timeConciseUnion", "type": "string"} ,
+{"label": "timeGenericConciseIntersection", "type": "string"} ,
+{"label": "timeGenericConciseUnion", "type": "string"} ,
+{"label": "timeGenericRoaringIntersection", "type": "string"} ,
+{"label": "timeGenericRoaringUnion", "type": "string"} ,
+{"label": "timeImmutableRoaringUnion", "type": "string"} ,
+{"label": "timeOffheapConciseUnion", "type": "string"} ,
+{"label": "timeOffheapRoaringUnion", "type": "string"} ,
+{"label": "timeRoaringUnion", "type": "string"} ],
+"rows": [
+{"c": [{"v": "5"}, {"v": "0.00001"}, {"v": "2014-11-04 14:03:04.268"}, {"v": 80.304}, {"v": 79.758}, {"v": 65.896}, {"v": 0.008}, {"v": 0.278}, {"v": 0.596}, {"v": 70.89}, {"v": 0.275}, {"v": 0.202}]},
+{"c": [{"v": "1"}, {"v": "0.0001"}, {"v": "2014-11-04 13:32:21.752"}, {"v": 30.843}, {"v": 30.863}, {"v": 32.306}, {"v": 0.012}, {"v": 0.272}, {"v": 0.546}, {"v": 32.727}, {"v": 0.327}, {"v": 0.158}]},
+{"c": [{"v": "2"}, {"v": "0.0010"}, {"v": "2014-11-04 13:41:55.608"}, {"v": 3.801}, {"v": 3.441}, {"v": 3.421}, {"v": 0.019}, {"v": 0.272}, {"v": 0.524}, {"v": 3.76}, {"v": 0.271}, {"v": 0.171}]},
+{"c": [{"v": "3"}, {"v": "0.0100"}, {"v": "2014-11-04 13:45:36.077"}, {"v": 0.341}, {"v": 0.541}, {"v": 0.628}, {"v": 0.025}, {"v": 0.276}, {"v": 0.576}, {"v": 0.352}, {"v": 0.276}, {"v": 0.263}]},
+{"c": [{"v": "4"}, {"v": "0.1000"}, {"v": "2014-11-04 13:49:27.509"}, {"v": 0.051}, {"v": 0.062}, {"v": 0.046}, {"v": 0.039}, {"v": 0.295}, {"v": 0.47}, {"v": 0.045}, {"v": 0.299}, {"v": 0.181}]},
+{"c": [{"v": "38"}, {"v": "0.25000"}, {"v": "2014-11-04 15:22:42.446"}, {"v": 0.036}, {"v": 0.041}, {"v": 0.04}, {"v": 0.052}, {"v": 0.179}, {"v": 0.391}, {"v": 0.042}, {"v": 0.18}, {"v": 0.105}]}
+]});