[DO NOT Merge] Very initial pass - ByteBuffer -> Memory

benchmarks/src/main/java/io/druid/benchmark/CompressedIndexedIntsBenchmark.java

@@ -20,6 +20,7 @@
 package io.druid.benchmark;
 
 import com.google.common.primitives.Ints;
+import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion;
 import io.druid.segment.data.CompressedObjectStrategy;
 import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier;
 import io.druid.segment.data.IndexedInts;
@@ -88,7 +89,7 @@ public void setup() throws IOException
             )
         )
     );
-    this.uncompressed = VSizeIndexedInts.readFromByteBuffer(bufferUncompressed);
+    this.uncompressed = VSizeIndexedInts.readFromMemory(new PositionalMemoryRegion(bufferUncompressed));
 
     filter = new BitSet();
     for (int i = 0; i < filteredRowCount; i++) {

benchmarks/src/main/java/io/druid/benchmark/CompressedVSizeIndexedBenchmark.java

@@ -22,6 +22,7 @@
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion;
 import io.druid.segment.CompressedVSizeIndexedSupplier;
 import io.druid.segment.data.CompressedObjectStrategy;
 import io.druid.segment.data.IndexedInts;
@@ -98,10 +99,8 @@ public IndexedInts apply(int[] input)
             ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4
         )
     );
-    this.compressed = CompressedVSizeIndexedSupplier.fromByteBuffer(
-        bufferCompressed,
-        ByteOrder.nativeOrder(),
-        null
+    this.compressed = CompressedVSizeIndexedSupplier.fromMemory(
+        new PositionalMemoryRegion(bufferCompressed), ByteOrder.nativeOrder()
     ).get();
 
     final ByteBuffer bufferUncompressed = serialize(
@@ -119,7 +118,7 @@ public VSizeIndexedInts apply(int[] input)
             )
         ).asWritableSupplier()
     );
-    this.uncompressed = VSizeIndexed.readFromByteBuffer(bufferUncompressed);
+    this.uncompressed = VSizeIndexed.readFromMemory(new PositionalMemoryRegion(bufferUncompressed));
 
     filter = new BitSet();
     for (int i = 0; i < filteredRowCount; i++) {

benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmark.java

@@ -23,6 +23,7 @@
 
 import com.google.common.base.Supplier;
 import com.google.common.io.Files;
+import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion;
 import io.druid.segment.data.CompressedFloatsIndexedSupplier;
 import io.druid.segment.data.IndexedFloats;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -72,7 +73,7 @@ public void setup() throws Exception
     File compFile = new File(dir, file + "-" + strategy);
     rand = new Random();
     ByteBuffer buffer = Files.map(compFile);
-    supplier = CompressedFloatsIndexedSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder(), null);
+    supplier = CompressedFloatsIndexedSupplier.fromMemory(new PositionalMemoryRegion(buffer), ByteOrder.nativeOrder());
   }
 
   @Benchmark

benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmark.java

@@ -21,6 +21,7 @@
 
 import com.google.common.base.Supplier;
 import com.google.common.io.Files;
+import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion;
 import io.druid.segment.data.CompressedLongsIndexedSupplier;
 import io.druid.segment.data.IndexedLongs;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -75,7 +76,7 @@ public void setup() throws Exception
     File compFile = new File(dir, file + "-" + strategy + "-" + format);
     rand = new Random();
     ByteBuffer buffer = Files.map(compFile);
-    supplier = CompressedLongsIndexedSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder(), null);
+    supplier = CompressedLongsIndexedSupplier.fromMemory(new PositionalMemoryRegion(buffer), ByteOrder.nativeOrder());
   }
 
   @Benchmark

benchmarks/src/main/java/io/druid/benchmark/VSizeSerdeBenchmark.java

@@ -20,6 +20,7 @@
 package io.druid.benchmark;
 
 import com.google.common.io.Files;
+import com.yahoo.memory.NativeMemory;
 import io.druid.segment.data.VSizeLongSerde;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -84,19 +85,19 @@ public void setup() throws IOException, URISyntaxException
       }
     }
     ByteBuffer buffer = Files.map(dummy);
-    d1 = VSizeLongSerde.getDeserializer(1, buffer, 10);
-    d2 = VSizeLongSerde.getDeserializer(2, buffer, 10);
-    d4 = VSizeLongSerde.getDeserializer(4, buffer, 10);
-    d8 = VSizeLongSerde.getDeserializer(8, buffer, 10);
-    d12 = VSizeLongSerde.getDeserializer(12, buffer, 10);
-    d16 = VSizeLongSerde.getDeserializer(16, buffer, 10);
-    d20 = VSizeLongSerde.getDeserializer(20, buffer, 10);
-    d24 = VSizeLongSerde.getDeserializer(24, buffer, 10);
-    d32 = VSizeLongSerde.getDeserializer(32, buffer, 10);
-    d40 = VSizeLongSerde.getDeserializer(40, buffer, 10);
-    d48 = VSizeLongSerde.getDeserializer(48, buffer, 10);
-    d56 = VSizeLongSerde.getDeserializer(56, buffer, 10);
-    d64 = VSizeLongSerde.getDeserializer(64, buffer, 10);
+    d1 = VSizeLongSerde.getDeserializer(1, new NativeMemory(buffer), 10);
+    d2 = VSizeLongSerde.getDeserializer(2, new NativeMemory(buffer), 10);
+    d4 = VSizeLongSerde.getDeserializer(4, new NativeMemory(buffer), 10);
+    d8 = VSizeLongSerde.getDeserializer(8, new NativeMemory(buffer), 10);
+    d12 = VSizeLongSerde.getDeserializer(12, new NativeMemory(buffer), 10);
+    d16 = VSizeLongSerde.getDeserializer(16, new NativeMemory(buffer), 10);
+    d20 = VSizeLongSerde.getDeserializer(20, new NativeMemory(buffer), 10);
+    d24 = VSizeLongSerde.getDeserializer(24, new NativeMemory(buffer), 10);
+    d32 = VSizeLongSerde.getDeserializer(32, new NativeMemory(buffer), 10);
+    d40 = VSizeLongSerde.getDeserializer(40, new NativeMemory(buffer), 10);
+    d48 = VSizeLongSerde.getDeserializer(48, new NativeMemory(buffer), 10);
+    d56 = VSizeLongSerde.getDeserializer(56, new NativeMemory(buffer), 10);
+    d64 = VSizeLongSerde.getDeserializer(64, new NativeMemory(buffer), 10);
   }
 
   @TearDown

bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java

@@ -26,6 +26,8 @@
 import java.util.Collection;
 import java.util.Iterator;
 
+import com.yahoo.memory.NativeMemory;
+
 /**
  * As the name suggests, this class instantiates bitmaps of the types
  * WrappedConciseBitmap and WrappedImmutableConciseBitmap.
@@ -104,7 +106,7 @@ public ImmutableBitmap makeImmutableBitmap(MutableBitmap mutableBitmap)
   @Override
   public ImmutableBitmap mapImmutableBitmap(ByteBuffer b)
   {
-    return new WrappedImmutableConciseBitmap(b);
+    return new WrappedImmutableConciseBitmap(new NativeMemory(b));
   }
 
   @Override

bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java

@@ -19,22 +19,20 @@
 
 package io.druid.collections.bitmap;
 
-
 import io.druid.extendedset.intset.ImmutableConciseSet;
+import com.yahoo.memory.Memory;
 import org.roaringbitmap.IntIterator;
 
-import java.nio.ByteBuffer;
-
 public class WrappedImmutableConciseBitmap implements ImmutableBitmap
 {
   /**
    * Underlying bitmap.
    */
   private final ImmutableConciseSet bitmap;
 
-  public WrappedImmutableConciseBitmap(ByteBuffer byteBuffer)
+  public WrappedImmutableConciseBitmap(Memory memory)
   {
-    this.bitmap = new ImmutableConciseSet(byteBuffer.asReadOnlyBuffer());
+    this.bitmap = new ImmutableConciseSet(memory);
   }
 
   /**

bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java

@@ -21,6 +21,7 @@
 
 import com.google.common.primitives.Floats;
 import com.google.common.primitives.Ints;
+import com.yahoo.memory.Memory;
 import io.druid.collections.bitmap.BitmapFactory;
 import io.druid.collections.bitmap.ImmutableBitmap;
 
@@ -51,27 +52,27 @@ public class ImmutableNode
   private final boolean isLeaf;
   private final int childrenOffset;
 
-  private final ByteBuffer data;
+  private final Memory data;
 
   private final BitmapFactory bitmapFactory;
 
   public ImmutableNode(
       int numDims,
       int initialOffset,
       int offsetFromInitial,
-      ByteBuffer data,
+      Memory data,
       BitmapFactory bitmapFactory
   )
   {
     this.bitmapFactory = bitmapFactory;
     this.numDims = numDims;
     this.initialOffset = initialOffset;
     this.offsetFromInitial = offsetFromInitial;
-    short header = data.getShort(initialOffset + offsetFromInitial);
+    short header = Short.reverseBytes(data.getShort(initialOffset + offsetFromInitial));
     this.isLeaf = (header & 0x8000) != 0;
     this.numChildren = (short) (header & 0x7FFF);
     final int sizePosition = initialOffset + offsetFromInitial + HEADER_NUM_BYTES + 2 * numDims * Floats.BYTES;
-    int bitmapSize = data.getInt(sizePosition);
+    int bitmapSize = Integer.reverseBytes(data.getInt(sizePosition));
     this.childrenOffset = initialOffset
                           + offsetFromInitial
                           + HEADER_NUM_BYTES
@@ -88,7 +89,7 @@ public ImmutableNode(
       int offsetFromInitial,
       short numChildren,
       boolean leaf,
-      ByteBuffer data,
+      Memory data,
       BitmapFactory bitmapFactory
   )
   {
@@ -153,11 +154,10 @@ public float[] getMaxCoordinates()
   public ImmutableBitmap getImmutableBitmap()
   {
     final int sizePosition = initialOffset + offsetFromInitial + HEADER_NUM_BYTES + 2 * numDims * Floats.BYTES;
-    int numBytes = data.getInt(sizePosition);
-    data.position(sizePosition + Ints.BYTES);
-    ByteBuffer tmpBuffer = data.slice();
-    tmpBuffer.limit(numBytes);
-    return bitmapFactory.mapImmutableBitmap(tmpBuffer.asReadOnlyBuffer());
+    int numBytes = Integer.reverseBytes(data.getInt(sizePosition));
+    byte[] bytes = new byte[numBytes];
+    data.getByteArray(sizePosition + Ints.BYTES, bytes, 0, numBytes);
+    return bitmapFactory.mapImmutableBitmap(ByteBuffer.wrap(bytes));
   }
 
   public Iterable<ImmutableNode> getChildren()
@@ -184,15 +184,15 @@ public ImmutableNode next()
               return new ImmutablePoint(
                   numDims,
                   initialOffset,
-                  data.getInt(childrenOffset + (count++) * Ints.BYTES),
+                  Integer.reverseBytes(data.getInt(childrenOffset + (count++) * Ints.BYTES)),
                   data,
                   bitmapFactory
               );
             }
             return new ImmutableNode(
                 numDims,
                 initialOffset,
-                data.getInt(childrenOffset + (count++) * Ints.BYTES),
+                Integer.reverseBytes(data.getInt(childrenOffset + (count++) * Ints.BYTES)),
                 data,
                 bitmapFactory
             );
@@ -208,19 +208,15 @@ public void remove()
     };
   }
 
-  public ByteBuffer getData()
+  public Memory getData()
   {
     return data;
   }
 
   private float[] getCoords(int offset)
   {
     final float[] retVal = new float[numDims];
-
-    final ByteBuffer readOnlyBuffer = data.asReadOnlyBuffer();
-    readOnlyBuffer.position(offset);
-    readOnlyBuffer.asFloatBuffer().get(retVal);
-
+    data.getFloatArray(offset, retVal, 0, numDims);
     return retVal;
   }
 }

bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java

@@ -19,17 +19,16 @@
 
 package io.druid.collections.spatial;
 
+import com.yahoo.memory.Memory;
 import io.druid.collections.bitmap.BitmapFactory;
 
-import java.nio.ByteBuffer;
-
 public class ImmutablePoint extends ImmutableNode
 {
   public ImmutablePoint(
       int numDims,
       int initialOffset,
       int offsetFromInitial,
-      ByteBuffer data,
+      Memory data,
       BitmapFactory bitmapFactory
   )
   {

bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java

@@ -22,6 +22,8 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Ints;
+import com.yahoo.memory.Memory;
+import com.yahoo.memory.NativeMemory;
 import io.druid.collections.bitmap.BitmapFactory;
 import io.druid.collections.bitmap.ImmutableBitmap;
 import io.druid.collections.spatial.search.Bound;
@@ -38,21 +40,21 @@ public class ImmutableRTree
   private static byte VERSION = 0x0;
   private final int numDims;
   private final ImmutableNode root;
-  private final ByteBuffer data;
+  private final Memory data;
   private final SearchStrategy defaultSearchStrategy = new GutmanSearchStrategy();
 
   public ImmutableRTree()
   {
     this.numDims = 0;
-    this.data = ByteBuffer.wrap(new byte[]{});
+    this.data = new NativeMemory(new byte[]{});
     this.root = null;
   }
 
-  public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory)
+  public ImmutableRTree(Memory data, BitmapFactory bitmapFactory)
   {
-    final int initPosition = data.position();
-    Preconditions.checkArgument(data.get(0) == VERSION, "Mismatching versions");
-    this.numDims = data.getInt(1 + initPosition) & 0x7FFF;
+    final int initPosition = 0;
+    Preconditions.checkArgument(data.getByte(0) == VERSION, "Mismatching versions");
+    this.numDims = Integer.reverseBytes(data.getInt(1 + initPosition)) & 0x7FFF;
     this.data = data;
     this.root = new ImmutableNode(numDims, initPosition, 1 + Ints.BYTES, data, bitmapFactory);
   }
@@ -69,7 +71,7 @@ public static ImmutableRTree newImmutableFromMutable(RTree rTree)
     buffer.putInt(rTree.getNumDims());
     rTree.getRoot().storeInByteBuffer(buffer, buffer.position());
     buffer.position(0);
-    return new ImmutableRTree(buffer.asReadOnlyBuffer(), rTree.getBitmapFactory());
+    return new ImmutableRTree(new NativeMemory(buffer), rTree.getBitmapFactory());
   }
 
   private static int calcNumBytes(RTree tree)
@@ -102,7 +104,7 @@ private static int calcNodeBytes(Node node)
 
   public int size()
   {
-    return data.capacity();
+    return (int)data.getCapacity();
   }
 
   public ImmutableNode getRoot()
@@ -132,13 +134,22 @@ public Iterable<ImmutableBitmap> search(SearchStrategy strategy, Bound bound)
 
   public byte[] toBytes()
   {
-    ByteBuffer buf = ByteBuffer.allocate(data.capacity());
-    buf.put(data.asReadOnlyBuffer());
-    return buf.array();
+    byte[] bytes = new byte[(int)data.getCapacity()];
+    data.getByteArray(0, bytes, 0, bytes.length);
+    return bytes;
   }
 
-  public int compareTo(ImmutableRTree other)
+  //TODO Remove this and implement Memory.compareTo in Memory
+  public int compareTo(ImmutableRTree o)
   {
-    return this.data.compareTo(other.data);
+    int n = (int)Math.min(this.data.getCapacity(), o.data.getCapacity())/Ints.BYTES;
+    int i, j;
+    for (i = 0, j = 0; i < n; i++, j++) {
+      int cmp = Byte.compare(this.data.getByte(i), o.data.getByte(j));
+      if (cmp != 0) {
+        return cmp;
+      }
+    }
+    return (int)((this.data.getCapacity()-i) - (o.data.getCapacity()-j));
   }
 }

bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java

@@ -25,6 +25,7 @@
 import java.nio.ByteBuffer;
 import java.util.Random;
 
+import com.yahoo.memory.NativeMemory;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -72,7 +73,7 @@ protected static ImmutableConciseSet makeOffheapConcise(ImmutableConciseSet conc
     conciseCount++;
     final ByteBuffer buf = ByteBuffer.allocateDirect(bytes.length).put(bytes);
     buf.rewind();
-    return new ImmutableConciseSet(buf);
+    return new ImmutableConciseSet(new NativeMemory(buf));
   }
 
   protected static ImmutableRoaringBitmap writeImmutable(MutableRoaringBitmap r, ByteBuffer buf) throws IOException