Fix bugs in ImmutableRTree; Merge bytebuffer-collections module into druid-processing (#5275)

pom.xml

@@ -99,7 +99,6 @@
         <module>benchmarks</module>
         <module>aws-common</module>
         <module>java-util</module>
-        <module>bytebuffer-collections</module>
         <module>extendedset</module>
         <module>hll</module>
         <!-- Core extensions -->

processing/pom.xml

@@ -47,9 +47,25 @@
         </dependency>
         <dependency>
             <groupId>io.druid</groupId>
-            <artifactId>bytebuffer-collections</artifactId>
+            <artifactId>extendedset</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.roaringbitmap</groupId>
+            <artifactId>RoaringBitmap</artifactId>
+        </dependency>
         <dependency>
             <groupId>it.unimi.dsi</groupId>
             <artifactId>fastutil</artifactId>
@@ -125,6 +141,17 @@
             <artifactId>JUnitParams</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>1.4.182</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava-testlib</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
@@ -156,7 +183,32 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <excludedGroups>io.druid.collections.test.annotation.Benchmark</excludedGroups>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 
+    <profiles>
+        <profile>
+            <id>benchmark</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <argLine>-server -Xms3G -Xmx3G -Djub.consumers=CONSOLE,H2 -Djub.db.file=benchmarks/benchmarks</argLine>
+                            <groups>io.druid.collections.test.annotation.Benchmark</groups>
+                            <excludedGroups>io.druid.collections.test.annotation.Dummy</excludedGroups>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
 </project>

bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java → processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java

@@ -64,11 +64,6 @@ public byte[] toBytes()
     return bitmap.toBytes();
   }
 
-  public int compareTo(ImmutableBitmap other)
-  {
-    return bitmap.compareTo(((WrappedImmutableConciseBitmap) other).getBitmap());
-  }
-
   @Override
   public String toString()
   {

bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java → processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java

@@ -27,32 +27,44 @@
 import io.druid.collections.spatial.search.Bound;
 import io.druid.collections.spatial.search.GutmanSearchStrategy;
 import io.druid.collections.spatial.search.SearchStrategy;
+import io.druid.io.Channels;
+import io.druid.segment.writeout.WriteOutBytes;
+import it.unimi.dsi.fastutil.bytes.ByteArrays;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
  * An immutable representation of an {@link RTree} for spatial indexing.
  */
-public class ImmutableRTree
+public final class ImmutableRTree
 {
-  private static byte VERSION = 0x0;
+  private static final byte VERSION = 0x0;
+
+  private static final ImmutableRTree EMPTY = new ImmutableRTree();
+
+  public static ImmutableRTree empty()
+  {
+    return EMPTY;
+  }
+
   private final int numDims;
   private final ImmutableNode root;
   private final ByteBuffer data;
   private final SearchStrategy defaultSearchStrategy = new GutmanSearchStrategy();
 
-  public ImmutableRTree()
+  private ImmutableRTree()
   {
     this.numDims = 0;
-    this.data = ByteBuffer.wrap(new byte[]{});
+    this.data = ByteBuffer.wrap(ByteArrays.EMPTY_ARRAY);
     this.root = null;
   }
 
   public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory)
   {
     data = data.asReadOnlyBuffer();
     final int initPosition = data.position();
-    Preconditions.checkArgument(data.get(0) == VERSION, "Mismatching versions");
+    Preconditions.checkArgument(data.get(initPosition) == VERSION, "Mismatching versions");
     this.numDims = data.getInt(1 + initPosition) & 0x7FFF;
     this.data = data;
     this.root = new ImmutableNode(numDims, initPosition, 1 + Ints.BYTES, data, bitmapFactory);
@@ -61,7 +73,7 @@ public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory)
   public static ImmutableRTree newImmutableFromMutable(RTree rTree)
   {
     if (rTree.getSize() == 0) {
-      return new ImmutableRTree();
+      return empty();
     }
 
     ByteBuffer buffer = ByteBuffer.wrap(new byte[calcNumBytes(rTree)]);
@@ -103,7 +115,7 @@ private static int calcNodeBytes(Node node)
 
   public int size()
   {
-    return data.capacity();
+    return data.remaining();
   }
 
   public Iterable<ImmutableBitmap> search(Bound bound)
@@ -123,9 +135,19 @@ public Iterable<ImmutableBitmap> search(SearchStrategy strategy, Bound bound)
 
   public byte[] toBytes()
   {
-    ByteBuffer buf = ByteBuffer.allocate(data.capacity());
-    buf.put(data.asReadOnlyBuffer());
-    return buf.array();
+    if (size() == 0) {
+      return ByteArrays.EMPTY_ARRAY;
+    }
+    byte[] res = new byte[data.remaining()];
+    data.duplicate().get(res);
+    return res;
+  }
+
+  public void writeTo(WriteOutBytes out) throws IOException
+  {
+    if (size() != 0) {
+      Channels.writeFully(out, data.duplicate());
+    }
   }
 
   public int compareTo(ImmutableRTree other)

processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java

@@ -229,12 +229,12 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value)
   public ImmutableRTree getSpatialIndex(String dimension)
   {
     if (isVirtualColumn(dimension)) {
-      return new ImmutableRTree();
+      return ImmutableRTree.empty();
     }
 
     final Column column = index.getColumn(dimension);
     if (column == null || !column.getCapabilities().hasSpatialIndexes()) {
-      return new ImmutableRTree();
+      return ImmutableRTree.empty();
     }
 
     return column.getSpatialIndex().getRTree();

processing/src/main/java/io/druid/segment/data/GenericIndexed.java

@@ -498,7 +498,9 @@ private static <T, U> GenericIndexed<U> fromIterableVersionOne(
 
         // for compatibility with the format, but this field is unused
         valuesOut.writeInt(0);
-        strategy.writeTo(next, valuesOut);
+        if (next != null) {
+          strategy.writeTo(next, valuesOut);
+        }
         headerOut.writeInt(Ints.checkedCast(valuesOut.size()));
 
         if (prevVal instanceof Closeable) {

processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java

@@ -217,7 +217,9 @@ public void write(T objectToWrite) throws IOException
     // for compatibility with the format (see GenericIndexed javadoc for description of the format), but this field is
     // unused.
     valuesOut.writeInt(0);
-    strategy.writeTo(objectToWrite, valuesOut);
+    if (objectToWrite != null) {
+      strategy.writeTo(objectToWrite, valuesOut);
+    }
 
     if (!requireMultipleFiles) {
       headerOut.writeInt(Ints.checkedCast(valuesOut.size()));

processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java

@@ -22,8 +22,9 @@
 import com.google.common.collect.Ordering;
 import io.druid.collections.bitmap.BitmapFactory;
 import io.druid.collections.spatial.ImmutableRTree;
-import it.unimi.dsi.fastutil.bytes.ByteArrays;
+import io.druid.segment.writeout.WriteOutBytes;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 public class ImmutableRTreeObjectStrategy implements ObjectStrategy<ImmutableRTree>
@@ -71,12 +72,15 @@ public ImmutableRTree fromByteBuffer(ByteBuffer buffer, int numBytes)
   @Override
   public byte[] toBytes(ImmutableRTree val)
   {
-    if (val == null || val.size() == 0) {
-      return ByteArrays.EMPTY_ARRAY;
-    }
     return val.toBytes();
   }
 
+  @Override
+  public void writeTo(ImmutableRTree val, WriteOutBytes out) throws IOException
+  {
+    val.writeTo(out);
+  }
+
   @Override
   public int compare(ImmutableRTree o1, ImmutableRTree o2)
   {