diff --git a/benchmarks/src/main/java/io/druid/benchmark/CompressedIndexedIntsBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/CompressedIndexedIntsBenchmark.java index 9a5bf336e65e..90142631abb1 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/CompressedIndexedIntsBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/CompressedIndexedIntsBenchmark.java @@ -20,6 +20,7 @@ package io.druid.benchmark; import com.google.common.primitives.Ints; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier; import io.druid.segment.data.IndexedInts; @@ -88,7 +89,7 @@ public void setup() throws IOException ) ) ); - this.uncompressed = VSizeIndexedInts.readFromByteBuffer(bufferUncompressed); + this.uncompressed = VSizeIndexedInts.readFromMemory(new PositionalMemoryRegion(bufferUncompressed)); filter = new BitSet(); for (int i = 0; i < filteredRowCount; i++) { diff --git a/benchmarks/src/main/java/io/druid/benchmark/CompressedVSizeIndexedBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/CompressedVSizeIndexedBenchmark.java index 106854cd0386..b29c0235304d 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/CompressedVSizeIndexedBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/CompressedVSizeIndexedBenchmark.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.CompressedVSizeIndexedSupplier; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.IndexedInts; @@ -98,10 +99,8 @@ public IndexedInts apply(int[] input) ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4 ) ); - this.compressed = CompressedVSizeIndexedSupplier.fromByteBuffer( - bufferCompressed, - ByteOrder.nativeOrder(), - null + this.compressed = CompressedVSizeIndexedSupplier.fromMemory( + new PositionalMemoryRegion(bufferCompressed), ByteOrder.nativeOrder() ).get(); final ByteBuffer bufferUncompressed = serialize( @@ -119,7 +118,7 @@ public VSizeIndexedInts apply(int[] input) ) ).asWritableSupplier() ); - this.uncompressed = VSizeIndexed.readFromByteBuffer(bufferUncompressed); + this.uncompressed = VSizeIndexed.readFromMemory(new PositionalMemoryRegion(bufferUncompressed)); filter = new BitSet(); for (int i = 0; i < filteredRowCount; i++) { diff --git a/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmark.java index b06ecb0657a2..197b093bb8a6 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmark.java @@ -23,6 +23,7 @@ import com.google.common.base.Supplier; import com.google.common.io.Files; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.data.CompressedFloatsIndexedSupplier; import io.druid.segment.data.IndexedFloats; import org.openjdk.jmh.annotations.Benchmark; @@ -72,7 +73,7 @@ public void setup() throws Exception File compFile = new File(dir, file + "-" + strategy); rand = new Random(); ByteBuffer buffer = Files.map(compFile); - supplier = CompressedFloatsIndexedSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder(), null); + supplier = CompressedFloatsIndexedSupplier.fromMemory(new PositionalMemoryRegion(buffer), ByteOrder.nativeOrder()); } @Benchmark diff --git a/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmark.java index ac41c6874571..8ba2e42972b6 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmark.java @@ -21,6 +21,7 @@ import com.google.common.base.Supplier; import com.google.common.io.Files; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.data.CompressedLongsIndexedSupplier; import io.druid.segment.data.IndexedLongs; import org.openjdk.jmh.annotations.Benchmark; @@ -75,7 +76,7 @@ public void setup() throws Exception File compFile = new File(dir, file + "-" + strategy + "-" + format); rand = new Random(); ByteBuffer buffer = Files.map(compFile); - supplier = CompressedLongsIndexedSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder(), null); + supplier = CompressedLongsIndexedSupplier.fromMemory(new PositionalMemoryRegion(buffer), ByteOrder.nativeOrder()); } @Benchmark diff --git a/benchmarks/src/main/java/io/druid/benchmark/VSizeSerdeBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/VSizeSerdeBenchmark.java index 490128cc5f60..a1d4aa8fde90 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/VSizeSerdeBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/VSizeSerdeBenchmark.java @@ -20,6 +20,7 @@ package io.druid.benchmark; import com.google.common.io.Files; +import com.yahoo.memory.NativeMemory; import io.druid.segment.data.VSizeLongSerde; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -84,19 +85,19 @@ public void setup() throws IOException, URISyntaxException } } ByteBuffer buffer = Files.map(dummy); - d1 = VSizeLongSerde.getDeserializer(1, buffer, 10); - d2 = VSizeLongSerde.getDeserializer(2, buffer, 10); - d4 = VSizeLongSerde.getDeserializer(4, buffer, 10); - d8 = VSizeLongSerde.getDeserializer(8, buffer, 10); - d12 = VSizeLongSerde.getDeserializer(12, buffer, 10); - d16 = VSizeLongSerde.getDeserializer(16, buffer, 10); - d20 = VSizeLongSerde.getDeserializer(20, buffer, 10); - d24 = VSizeLongSerde.getDeserializer(24, buffer, 10); - d32 = VSizeLongSerde.getDeserializer(32, buffer, 10); - d40 = VSizeLongSerde.getDeserializer(40, buffer, 10); - d48 = VSizeLongSerde.getDeserializer(48, buffer, 10); - d56 = VSizeLongSerde.getDeserializer(56, buffer, 10); - d64 = VSizeLongSerde.getDeserializer(64, buffer, 10); + d1 = VSizeLongSerde.getDeserializer(1, new NativeMemory(buffer), 10); + d2 = VSizeLongSerde.getDeserializer(2, new NativeMemory(buffer), 10); + d4 = VSizeLongSerde.getDeserializer(4, new NativeMemory(buffer), 10); + d8 = VSizeLongSerde.getDeserializer(8, new NativeMemory(buffer), 10); + d12 = VSizeLongSerde.getDeserializer(12, new NativeMemory(buffer), 10); + d16 = VSizeLongSerde.getDeserializer(16, new NativeMemory(buffer), 10); + d20 = VSizeLongSerde.getDeserializer(20, new NativeMemory(buffer), 10); + d24 = VSizeLongSerde.getDeserializer(24, new NativeMemory(buffer), 10); + d32 = VSizeLongSerde.getDeserializer(32, new NativeMemory(buffer), 10); + d40 = VSizeLongSerde.getDeserializer(40, new NativeMemory(buffer), 10); + d48 = VSizeLongSerde.getDeserializer(48, new NativeMemory(buffer), 10); + d56 = VSizeLongSerde.getDeserializer(56, new NativeMemory(buffer), 10); + d64 = VSizeLongSerde.getDeserializer(64, new NativeMemory(buffer), 10); } @TearDown diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java index 3416cd827869..1b9fc4cd3574 100755 --- a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java @@ -26,6 +26,8 @@ import java.util.Collection; import java.util.Iterator; +import com.yahoo.memory.NativeMemory; + /** * As the name suggests, this class instantiates bitmaps of the types * WrappedConciseBitmap and WrappedImmutableConciseBitmap. @@ -104,7 +106,7 @@ public ImmutableBitmap makeImmutableBitmap(MutableBitmap mutableBitmap) @Override public ImmutableBitmap mapImmutableBitmap(ByteBuffer b) { - return new WrappedImmutableConciseBitmap(b); + return new WrappedImmutableConciseBitmap(new NativeMemory(b)); } @Override diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java index 137b28d7ef23..4430a2adb618 100755 --- a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java @@ -19,12 +19,10 @@ package io.druid.collections.bitmap; - import io.druid.extendedset.intset.ImmutableConciseSet; +import com.yahoo.memory.Memory; import org.roaringbitmap.IntIterator; -import java.nio.ByteBuffer; - public class WrappedImmutableConciseBitmap implements ImmutableBitmap { /** @@ -32,9 +30,9 @@ public class WrappedImmutableConciseBitmap implements ImmutableBitmap */ private final ImmutableConciseSet bitmap; - public WrappedImmutableConciseBitmap(ByteBuffer byteBuffer) + public WrappedImmutableConciseBitmap(Memory memory) { - this.bitmap = new ImmutableConciseSet(byteBuffer.asReadOnlyBuffer()); + this.bitmap = new ImmutableConciseSet(memory); } /** diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java index e29e2002ad7f..6b0b46b2f7f6 100755 --- a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; @@ -51,7 +52,7 @@ public class ImmutableNode private final boolean isLeaf; private final int childrenOffset; - private final ByteBuffer data; + private final Memory data; private final BitmapFactory bitmapFactory; @@ -59,7 +60,7 @@ public ImmutableNode( int numDims, int initialOffset, int offsetFromInitial, - ByteBuffer data, + Memory data, BitmapFactory bitmapFactory ) { @@ -67,11 +68,11 @@ public ImmutableNode( this.numDims = numDims; this.initialOffset = initialOffset; this.offsetFromInitial = offsetFromInitial; - short header = data.getShort(initialOffset + offsetFromInitial); + short header = Short.reverseBytes(data.getShort(initialOffset + offsetFromInitial)); this.isLeaf = (header & 0x8000) != 0; this.numChildren = (short) (header & 0x7FFF); final int sizePosition = initialOffset + offsetFromInitial + HEADER_NUM_BYTES + 2 * numDims * Floats.BYTES; - int bitmapSize = data.getInt(sizePosition); + int bitmapSize = Integer.reverseBytes(data.getInt(sizePosition)); this.childrenOffset = initialOffset + offsetFromInitial + HEADER_NUM_BYTES @@ -88,7 +89,7 @@ public ImmutableNode( int offsetFromInitial, short numChildren, boolean leaf, - ByteBuffer data, + Memory data, BitmapFactory bitmapFactory ) { @@ -153,11 +154,10 @@ public float[] getMaxCoordinates() public ImmutableBitmap getImmutableBitmap() { final int sizePosition = initialOffset + offsetFromInitial + HEADER_NUM_BYTES + 2 * numDims * Floats.BYTES; - int numBytes = data.getInt(sizePosition); - data.position(sizePosition + Ints.BYTES); - ByteBuffer tmpBuffer = data.slice(); - tmpBuffer.limit(numBytes); - return bitmapFactory.mapImmutableBitmap(tmpBuffer.asReadOnlyBuffer()); + int numBytes = Integer.reverseBytes(data.getInt(sizePosition)); + byte[] bytes = new byte[numBytes]; + data.getByteArray(sizePosition + Ints.BYTES, bytes, 0, numBytes); + return bitmapFactory.mapImmutableBitmap(ByteBuffer.wrap(bytes)); } public Iterable getChildren() @@ -184,7 +184,7 @@ public ImmutableNode next() return new ImmutablePoint( numDims, initialOffset, - data.getInt(childrenOffset + (count++) * Ints.BYTES), + Integer.reverseBytes(data.getInt(childrenOffset + (count++) * Ints.BYTES)), data, bitmapFactory ); @@ -192,7 +192,7 @@ public ImmutableNode next() return new ImmutableNode( numDims, initialOffset, - data.getInt(childrenOffset + (count++) * Ints.BYTES), + Integer.reverseBytes(data.getInt(childrenOffset + (count++) * Ints.BYTES)), data, bitmapFactory ); @@ -208,7 +208,7 @@ public void remove() }; } - public ByteBuffer getData() + public Memory getData() { return data; } @@ -216,11 +216,7 @@ public ByteBuffer getData() private float[] getCoords(int offset) { final float[] retVal = new float[numDims]; - - final ByteBuffer readOnlyBuffer = data.asReadOnlyBuffer(); - readOnlyBuffer.position(offset); - readOnlyBuffer.asFloatBuffer().get(retVal); - + data.getFloatArray(offset, retVal, 0, numDims); return retVal; } } diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java index 1ee07020571f..7e72051df21b 100755 --- a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java @@ -19,17 +19,16 @@ package io.druid.collections.spatial; +import com.yahoo.memory.Memory; import io.druid.collections.bitmap.BitmapFactory; -import java.nio.ByteBuffer; - public class ImmutablePoint extends ImmutableNode { public ImmutablePoint( int numDims, int initialOffset, int offsetFromInitial, - ByteBuffer data, + Memory data, BitmapFactory bitmapFactory ) { diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java index 9da5957170bc..ff7e501f2698 100755 --- a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java @@ -22,6 +22,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; +import com.yahoo.memory.NativeMemory; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.spatial.search.Bound; @@ -38,21 +40,21 @@ public class ImmutableRTree private static byte VERSION = 0x0; private final int numDims; private final ImmutableNode root; - private final ByteBuffer data; + private final Memory data; private final SearchStrategy defaultSearchStrategy = new GutmanSearchStrategy(); public ImmutableRTree() { this.numDims = 0; - this.data = ByteBuffer.wrap(new byte[]{}); + this.data = new NativeMemory(new byte[]{}); this.root = null; } - public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory) + public ImmutableRTree(Memory data, BitmapFactory bitmapFactory) { - final int initPosition = data.position(); - Preconditions.checkArgument(data.get(0) == VERSION, "Mismatching versions"); - this.numDims = data.getInt(1 + initPosition) & 0x7FFF; + final int initPosition = 0; + Preconditions.checkArgument(data.getByte(0) == VERSION, "Mismatching versions"); + this.numDims = Integer.reverseBytes(data.getInt(1 + initPosition)) & 0x7FFF; this.data = data; this.root = new ImmutableNode(numDims, initPosition, 1 + Ints.BYTES, data, bitmapFactory); } @@ -69,7 +71,7 @@ public static ImmutableRTree newImmutableFromMutable(RTree rTree) buffer.putInt(rTree.getNumDims()); rTree.getRoot().storeInByteBuffer(buffer, buffer.position()); buffer.position(0); - return new ImmutableRTree(buffer.asReadOnlyBuffer(), rTree.getBitmapFactory()); + return new ImmutableRTree(new NativeMemory(buffer), rTree.getBitmapFactory()); } private static int calcNumBytes(RTree tree) @@ -102,7 +104,7 @@ private static int calcNodeBytes(Node node) public int size() { - return data.capacity(); + return (int)data.getCapacity(); } public ImmutableNode getRoot() @@ -132,13 +134,22 @@ public Iterable search(SearchStrategy strategy, Bound bound) public byte[] toBytes() { - ByteBuffer buf = ByteBuffer.allocate(data.capacity()); - buf.put(data.asReadOnlyBuffer()); - return buf.array(); + byte[] bytes = new byte[(int)data.getCapacity()]; + data.getByteArray(0, bytes, 0, bytes.length); + return bytes; } - public int compareTo(ImmutableRTree other) + //TODO Remove this and implement Memory.compareTo in Memory + public int compareTo(ImmutableRTree o) { - return this.data.compareTo(other.data); + int n = (int)Math.min(this.data.getCapacity(), o.data.getCapacity())/Ints.BYTES; + int i, j; + for (i = 0, j = 0; i < n; i++, j++) { + int cmp = Byte.compare(this.data.getByte(i), o.data.getByte(j)); + if (cmp != 0) { + return cmp; + } + } + return (int)((this.data.getCapacity()-i) - (o.data.getCapacity()-j)); } } diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java index 617e4ea6de0c..1ec02e9676c4 100755 --- a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java +++ b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java @@ -25,6 +25,7 @@ import java.nio.ByteBuffer; import java.util.Random; +import com.yahoo.memory.NativeMemory; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -72,7 +73,7 @@ protected static ImmutableConciseSet makeOffheapConcise(ImmutableConciseSet conc conciseCount++; final ByteBuffer buf = ByteBuffer.allocateDirect(bytes.length).put(bytes); buf.rewind(); - return new ImmutableConciseSet(buf); + return new ImmutableConciseSet(new NativeMemory(buf)); } protected static ImmutableRoaringBitmap writeImmutable(MutableRoaringBitmap r, ByteBuffer buf) throws IOException diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java index 4ae8bc744316..1a77cb401cf9 100755 --- a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java +++ b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java @@ -23,6 +23,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import com.yahoo.memory.NativeMemory; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; @@ -36,6 +37,7 @@ import org.roaringbitmap.IntIterator; import java.nio.ByteBuffer; +import java.util.Iterator; import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -58,7 +60,7 @@ public void testToAndFromByteBuffer() ImmutableRTree firstTree = ImmutableRTree.newImmutableFromMutable(tree); ByteBuffer buffer = ByteBuffer.wrap(firstTree.toBytes()); - ImmutableRTree secondTree = new ImmutableRTree(buffer, bf); + ImmutableRTree secondTree = new ImmutableRTree(new NativeMemory(buffer), bf); Iterable points = secondTree.search(new RadiusBound(new float[]{0, 0}, 10)); ImmutableBitmap finalSet = bf.union(points); Assert.assertTrue(finalSet.size() >= 5); @@ -83,7 +85,7 @@ public void testToAndFromByteBufferRoaring() ImmutableRTree firstTree = ImmutableRTree.newImmutableFromMutable(tree); ByteBuffer buffer = ByteBuffer.wrap(firstTree.toBytes()); - ImmutableRTree secondTree = new ImmutableRTree(buffer, bf); + ImmutableRTree secondTree = new ImmutableRTree(new NativeMemory(buffer), bf); Iterable points = secondTree.search(new RadiusBound(new float[]{0, 0}, 10)); ImmutableBitmap finalSet = bf.union(points); @@ -170,21 +172,29 @@ public void testSearchWithSplit() Random rand = new Random(); for (int i = 0; i < 95; i++) { tree.insert( - new float[]{(float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0)}, + new float[]{i+100, i+100}, i ); } ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); Iterable points = searchTree.search(new RadiusBound(new float[]{0, 0}, 5)); - ImmutableBitmap finalSet = bf.union(points); - Assert.assertTrue(finalSet.size() >= 5); - Set expected = Sets.newHashSet(1, 2, 3, 4, 5); - IntIterator iter = finalSet.iterator(); - while (iter.hasNext()) { - Assert.assertTrue(expected.contains(iter.next())); + int i =0; + Iterator itr = points.iterator(); + while(itr.hasNext()){ + itr.next(); + i++; } + Assert.assertEquals(5, i); +// ImmutableBitmap finalSet = bf.union(points); +// Assert.assertTrue(finalSet.size() >= 5); +// +// Set expected = Sets.newHashSet(1, 2, 3, 4, 5); +// IntIterator iter = finalSet.iterator(); +// while (iter.hasNext()) { +// Assert.assertTrue(expected.contains(iter.next())); +// } } @Test @@ -213,7 +223,10 @@ public void testSearchWithSplitRoaring() Set expected = Sets.newHashSet(1, 2, 3, 4, 5); IntIterator iter = finalSet.iterator(); + int i = 0; while (iter.hasNext()) { + System.out.println("i=" + i); + i++; Assert.assertTrue(expected.contains(iter.next())); } } diff --git a/common/src/main/java/io/druid/common/utils/SerializerUtils.java b/common/src/main/java/io/druid/common/utils/SerializerUtils.java index d0ea17099746..3e23d62966b0 100644 --- a/common/src/main/java/io/druid/common/utils/SerializerUtils.java +++ b/common/src/main/java/io/druid/common/utils/SerializerUtils.java @@ -22,8 +22,9 @@ import com.google.common.io.ByteStreams; import com.google.common.io.OutputSupplier; import com.google.common.primitives.Ints; - +import com.yahoo.memory.Memory; import io.druid.collections.IntList; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.io.IOException; import java.io.InputStream; @@ -67,12 +68,61 @@ public String readString(InputStream in) throws IOException return StringUtils.fromUtf8(stringBytes); } + public String readString(PositionalMemoryRegion pMemory, int numBytes) throws IOException + { + return StringUtils.fromUtf8(pMemory.getBytes(numBytes)); + } + + public String readString(Memory memory, long offSet) throws IOException + { + final int length = memory.getInt(offSet); + byte[] bytes = new byte[length]; + memory.getByteArray(offSet + 4, bytes, 0, length); + return StringUtils.fromUtf8(bytes); + } + + public String readStringReverse(Memory memory, long offSet) throws IOException + { + final int length = Integer.reverseBytes(memory.getInt(offSet)); + byte[] bytes = new byte[length]; + memory.getByteArray(offSet + 4, bytes, 0, length); + return StringUtils.fromUtf8(bytes); + } + + public String readStringReverse(Memory memory, int length) throws IOException + { + byte[] bytes = new byte[length]; + memory.getByteArray(4, bytes, 0, length); + return StringUtils.fromUtf8(bytes); + } + + public String readStringReverse(PositionalMemoryRegion pMemory, int length) throws IOException + { + return StringUtils.fromUtf8(pMemory.getBytes(length)); + } + + public String readStringReverse(PositionalMemoryRegion pMemory) throws IOException + { + return StringUtils.fromUtf8(pMemory.getBytes(Integer.reverseBytes(pMemory.getInt()))); + } + public String readString(ByteBuffer in) throws IOException { final int length = in.getInt(); return StringUtils.fromUtf8(readBytes(in, length)); } - + + public String readString(PositionalMemoryRegion pMemory) throws IOException + { + final int length = pMemory.getInt(); + return StringUtils.fromUtf8(readBytes(pMemory, length)); + } + + public byte[] readBytes(PositionalMemoryRegion pMemory, int length) throws IOException + { + return pMemory.getBytes(length); + } + public byte[] readBytes(ByteBuffer in, int length) throws IOException { byte[] bytes = new byte[length]; @@ -80,6 +130,13 @@ public byte[] readBytes(ByteBuffer in, int length) throws IOException return bytes; } + public byte[] readBytes(Memory in, int length) throws IOException + { + byte[] bytes = new byte[length]; + in.getByteArray(0, bytes, 0, length); + return bytes; + } + public void writeStrings(OutputStream out, String[] names) throws IOException { writeStrings(out, Arrays.asList(names)); diff --git a/extendedset/pom.xml b/extendedset/pom.xml index b402948e27ad..be8876af5381 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -54,6 +54,11 @@ 4.8.1 test + + com.yahoo.datasketches + memory + 0.8.4 + diff --git a/extendedset/src/main/java/io/druid/extendedset/intset/BitIterator.java b/extendedset/src/main/java/io/druid/extendedset/intset/BitIterator.java index 7b76026361b4..5c92b9f945a9 100644 --- a/extendedset/src/main/java/io/druid/extendedset/intset/BitIterator.java +++ b/extendedset/src/main/java/io/druid/extendedset/intset/BitIterator.java @@ -19,6 +19,8 @@ package io.druid.extendedset.intset; +import com.google.common.primitives.Ints; + import java.util.NoSuchElementException; // Based on the ConciseSet implementation by Alessandro Colantonio @@ -75,7 +77,7 @@ private int advance() { int wordExpanderNext; while ((wordExpanderNext = wordExpanderAdvance()) < 0) { - if (nextIndex > immutableConciseSet.lastWordIndex) { + if (nextIndex > immutableConciseSet.getLastWordIndex()) { return -1; } nextWord(); @@ -114,7 +116,7 @@ public void skipAllBefore(int element) next = wordExpanderNext; return; } - if (nextIndex > immutableConciseSet.lastWordIndex) { + if (nextIndex > immutableConciseSet.getLastWordIndex()) { next = -1; return; } @@ -150,7 +152,7 @@ public IntSet.IntIterator clone() private void nextWord() { - final int word = immutableConciseSet.words.get(nextIndex++); + final int word = immutableConciseSet.words.getInt(nextIndex++ * Ints.BYTES); literalAndZeroFill = wordExpanderReset(nextOffset, word); // prepare next offset diff --git a/extendedset/src/main/java/io/druid/extendedset/intset/ImmutableConciseSet.java b/extendedset/src/main/java/io/druid/extendedset/intset/ImmutableConciseSet.java index c2a54da2b9d7..fbf9098e8981 100755 --- a/extendedset/src/main/java/io/druid/extendedset/intset/ImmutableConciseSet.java +++ b/extendedset/src/main/java/io/druid/extendedset/intset/ImmutableConciseSet.java @@ -20,11 +20,13 @@ import com.google.common.collect.Iterators; import com.google.common.collect.UnmodifiableIterator; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; +import com.yahoo.memory.NativeMemory; import io.druid.extendedset.utilities.IntList; import java.nio.ByteBuffer; -import java.nio.IntBuffer; import java.util.ArrayList; +import java.nio.ByteOrder; import java.util.Arrays; import java.util.Comparator; import java.util.Iterator; @@ -34,6 +36,9 @@ public class ImmutableConciseSet { private final static int CHUNK_SIZE = 10000; + final Memory words; + private final int lastWordIndex; + private final int size; private static final Comparator UNION_COMPARATOR = new Comparator() { @@ -121,12 +126,39 @@ public int compare(WordIterator i1, WordIterator i2) } }; + public ImmutableConciseSet(Memory memory) + { + this.words = memory; + this.lastWordIndex = (int)memory.getCapacity()/Ints.BYTES - 1; + this.size = calcSize(); + } + +// public ImmutableConciseSet(ByteBuffer byteBuffer) +// { +//// this.words = byteBuffer.asIntBuffer(); +//// this.lastWordIndex = words.capacity() - 1; +//// this.size = calcSize(); +// } +// +// public ImmutableConciseSet(IntBuffer buffer) +// { +//// this.words = buffer; +//// this.lastWordIndex = (words == null || buffer.capacity() == 0) ? -1 : words.capacity() - 1; +//// this.size = calcSize(); +// } + public static ImmutableConciseSet newImmutableFromMutable(ConciseSet conciseSet) { if (conciseSet == null || conciseSet.isEmpty()) { return new ImmutableConciseSet(); } - return new ImmutableConciseSet(IntBuffer.wrap(conciseSet.getWords())); + //TODO Replace with new constructor from Memory. This is a hack till then + int[] words = conciseSet.getWords(); + ByteBuffer bb = ByteBuffer.allocate(words.length*Ints.BYTES).order(ByteOrder.nativeOrder()); + for(int i = 0; i < words.length; i++){ + bb.putInt(words[i]); + } + return new ImmutableConciseSet(new NativeMemory(bb)); } public static ImmutableConciseSet union(ImmutableConciseSet... sets) @@ -262,7 +294,8 @@ public static ImmutableConciseSet complement(ImmutableConciseSet set, int length if (retVal.isEmpty()) { return new ImmutableConciseSet(); } - return compact(new ImmutableConciseSet(IntBuffer.wrap(retVal.toArray()))); + + return compact(new ImmutableConciseSet(convertToNativeMem(retVal))); } public static ImmutableConciseSet compact(ImmutableConciseSet set) @@ -272,7 +305,8 @@ public static ImmutableConciseSet compact(ImmutableConciseSet set) while (itr.hasNext()) { addAndCompact(retVal, itr.next()); } - return new ImmutableConciseSet(IntBuffer.wrap(retVal.toArray())); + + return new ImmutableConciseSet(convertToNativeMem(retVal)); } private static void addAndCompact(IntList set, int wordToAdd) @@ -482,7 +516,8 @@ private static ImmutableConciseSet doUnion(Iterator sets) if (retVal.isEmpty()) { return new ImmutableConciseSet(); } - return new ImmutableConciseSet(IntBuffer.wrap(retVal.toArray())); + + return new ImmutableConciseSet(convertToNativeMem(retVal)); } public static ImmutableConciseSet doIntersection(Iterator sets) @@ -683,7 +718,19 @@ public static ImmutableConciseSet doIntersection(Iterator s if (retVal.isEmpty()) { return new ImmutableConciseSet(); } - return new ImmutableConciseSet(IntBuffer.wrap(retVal.toArray())); + + return new ImmutableConciseSet(convertToNativeMem(retVal)); + } + + private static Memory convertToNativeMem(IntList retVal) + { + //TODO Replace with new constructor from Memory. This is a hack till then + ByteBuffer bb = ByteBuffer.allocate(retVal.length()*Ints.BYTES).order(ByteOrder.nativeOrder()); + for(int i = 0; i < retVal.length(); i++){ + bb.putInt(retVal.get(i)); + } + + return new NativeMemory(bb); } /** @@ -747,7 +794,8 @@ public static ImmutableConciseSet doComplement(ImmutableConciseSet set) if (retVal.isEmpty()) { return new ImmutableConciseSet(); } - return new ImmutableConciseSet(IntBuffer.wrap(retVal.toArray())); + + return new ImmutableConciseSet(convertToNativeMem(retVal)); } // Based on the ConciseSet implementation by Alessandro Colantonio @@ -780,10 +828,6 @@ private static void trimZeros(IntList set) } while (true); } - final IntBuffer words; - final int lastWordIndex; - private final int size; - public ImmutableConciseSet() { this.words = null; @@ -791,28 +835,29 @@ public ImmutableConciseSet() this.size = 0; } - public ImmutableConciseSet(ByteBuffer byteBuffer) - { - this.words = byteBuffer.asIntBuffer(); - this.lastWordIndex = words.capacity() - 1; - this.size = calcSize(); - } - - public ImmutableConciseSet(IntBuffer buffer) - { - this.words = buffer; - this.lastWordIndex = (words == null || buffer.capacity() == 0) ? -1 : words.capacity() - 1; - this.size = calcSize(); - } +// public ImmutableConciseSet(ByteBuffer byteBuffer) +// { +// this.words = byteBuffer.asIntBuffer(); +// this.lastWordIndex = words.capacity() - 1; +// this.size = calcSize(); +// } +// +// public ImmutableConciseSet(IntBuffer buffer) +// { +// this.words = buffer; +// this.lastWordIndex = (words == null || buffer.capacity() == 0) ? -1 : words.capacity() - 1; +// this.size = calcSize(); +// } public byte[] toBytes() { if (words == null) { return new byte[]{}; } - ByteBuffer buf = ByteBuffer.allocate(words.capacity() * Ints.BYTES); - buf.asIntBuffer().put(words.asReadOnlyBuffer()); - return buf.array(); + + byte[] bytes = new byte[(int)words.getCapacity()]; + words.getByteArray(0, bytes, 0, bytes.length); + return bytes; } public int getLastWordIndex() @@ -825,7 +870,7 @@ private int calcSize() { int retVal = 0; for (int i = 0; i <= lastWordIndex; i++) { - int w = words.get(i); + int w = words.getInt(i*Ints.BYTES); if (ConciseSetUtils.isLiteral(w)) { retVal += ConciseSetUtils.getLiteralBitCount(w); } else { @@ -859,7 +904,7 @@ public int getLast() int last = 0; for (int i = 0; i <= lastWordIndex; i++) { - int w = words.get(i); + int w = words.getInt(i*Ints.BYTES); if (ConciseSetUtils.isLiteral(w)) { last += ConciseSetUtils.MAX_LITERAL_LENGTH; } else { @@ -867,7 +912,7 @@ public int getLast() } } - int w = words.get(lastWordIndex); + int w = words.getInt(lastWordIndex*Ints.BYTES); if (ConciseSetUtils.isLiteral(w)) { last -= Integer.numberOfLeadingZeros(ConciseSetUtils.getLiteralBits(w)); } else { @@ -898,7 +943,7 @@ public int get(int i) int position = i; int setBitsInCurrentWord = 0; for (int j = 0; j <= lastWordIndex; j++) { - int w = words.get(j); + int w = words.getInt(j*Ints.BYTES); if (ConciseSetUtils.isLiteral(w)) { // number of bits in the current word setBitsInCurrentWord = ConciseSetUtils.getLiteralBitCount(w); @@ -958,12 +1003,21 @@ public int get(int i) public int compareTo(ImmutableConciseSet other) { - return words.asReadOnlyBuffer().compareTo(other.words.asReadOnlyBuffer()); + long n = Math.min(words.getCapacity(), other.words.getCapacity()); + long i, j; + for (i = 0, j = 0; i < n; i++, j++) { + int cmp = Long.compare(this.words.getInt(i*Ints.BYTES), other.words.getInt(j*Ints.BYTES)); + if (cmp != 0) + { + return cmp; + } + } + return (int)((words.getCapacity()-i) - (other.words.getCapacity()-j)); } private boolean isEmpty() { - return words == null || words.limit() == 0; + return words == null || words.getCapacity() == 0; } @Override @@ -1044,7 +1098,7 @@ public boolean hasNext() if (hasNextWord) { return true; } - return currRow < (words.capacity() - 1); + return currRow < (words.getCapacity()/Ints.BYTES - 1); } @Override @@ -1056,7 +1110,7 @@ public int next() return currWord; } - currWord = words.get(++currRow); + currWord = words.getInt(++currRow*Ints.BYTES); if (ConciseSetUtils.isLiteral(currWord)) { startIndex = wordsWalked++; } else { diff --git a/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetTest.java b/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetTest.java index 2438bdad4e16..eac349e181af 100755 --- a/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetTest.java +++ b/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetTest.java @@ -17,12 +17,16 @@ package io.druid.extendedset.intset; import com.google.common.collect.Lists; +import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; +import com.yahoo.memory.NativeMemory; import junit.framework.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import java.nio.IntBuffer; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -127,7 +131,7 @@ public void testCompactOneLitOneLit() { int[] words = {-1, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); @@ -140,7 +144,7 @@ public void testCompactOneLitPureOneFill() { int[] words = {-1, 0x40000004}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x40000005, itr.next()); @@ -152,7 +156,7 @@ public void testCompactOneLitDirtyOneFill() { int[] words = {-1, 0x42000004}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(-1, itr.next()); @@ -165,7 +169,7 @@ public void testCompactOneFillOneLit() { int[] words = {0x40000004, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x40000005, itr.next()); @@ -177,7 +181,7 @@ public void testCompactOneFillPureOneFill() { int[] words = {0x40000004, 0x40000004}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x40000009, itr.next()); @@ -189,7 +193,7 @@ public void testCompactOneFillDirtyOneFill() { int[] words = {0x40000004, 0x42000004}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x40000004, itr.next()); @@ -202,7 +206,7 @@ public void testCompactZeroLitZeroLit() { int[] words = {0x80000000, 0x80000000, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x00000001, itr.next()); @@ -215,7 +219,7 @@ public void testCompactZeroLitPureZeroFill() { int[] words = {0x80000000, 0x00000004, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x00000005, itr.next()); @@ -228,7 +232,7 @@ public void testCompactZeroLitDirtyZeroFill() { int[] words = {0x80000000, 0x02000004, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x80000000, itr.next()); @@ -242,7 +246,7 @@ public void testCompactZeroFillZeroLit() { int[] words = {0x00000004, 0x80000000, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x00000005, itr.next()); @@ -255,7 +259,7 @@ public void testCompactZeroFillPureZeroFill() { int[] words = {0x00000004, 0x00000004, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x00000009, itr.next()); @@ -268,7 +272,7 @@ public void testCompactZeroFillDirtyZeroFill() { int[] words = {0x00000004, 0x02000004, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x00000004, itr.next()); @@ -282,7 +286,7 @@ public void testCompactSingleOneBitLitZeroLit() { int[] words = {0x80000001, 0x80000000, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x02000001, itr.next()); @@ -295,7 +299,7 @@ public void testCompactDoubleOneBitLitZeroLit() { int[] words = {0x80000003, 0x80000000, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x80000003, itr.next()); @@ -309,7 +313,7 @@ public void testCompactSingleOneBitLitPureZeroFill() { int[] words = {0x80000001, 0x00000004, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x02000005, itr.next()); @@ -322,7 +326,7 @@ public void testCompactDoubleOneBitLitPureZeroFill() { int[] words = {0x80000003, 0x00000004, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x80000003, itr.next()); @@ -336,7 +340,7 @@ public void testCompactSingleOneBitLitDirtyZeroFill() { int[] words = {0x80000001, 0x02000004, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x80000001, itr.next()); @@ -350,7 +354,7 @@ public void testCompactSingleZeroBitLitOneLit() { int[] words = {0xFFFFFFFE, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x42000001, itr.next()); @@ -362,7 +366,7 @@ public void testCompactDoubleZeroBitLitOneLit() { int[] words = {0xFFFFFFEE, -1}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0xFFFFFFEE, itr.next()); @@ -375,7 +379,7 @@ public void testCompactSingleZeroBitLitPureOneFill() { int[] words = {0xFFFFFFFE, 0x40000004}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0x42000005, itr.next()); @@ -387,7 +391,7 @@ public void testCompactDoubleZeroBitLitPureOneFill() { int[] words = {0xFFFFFFFC, 0x40000004}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0xFFFFFFFC, itr.next()); @@ -400,7 +404,7 @@ public void testCompactSingleZeroBitLitDirtyOneFill() { int[] words = {0xFFFFFFFE, 0x42000004}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0xFFFFFFFE, itr.next()); @@ -413,7 +417,7 @@ public void testCompactTwoLiterals() { int[] words = {0xFFFFFFFE, 0xFFEFFEFF}; - ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(IntBuffer.wrap(words))); + ImmutableConciseSet res = ImmutableConciseSet.compact(new ImmutableConciseSet(createMemoryFromIntArray(words))); ImmutableConciseSet.WordIterator itr = res.newWordIterator(); Assert.assertEquals(0xFFFFFFFE, itr.next()); @@ -1718,7 +1722,7 @@ public void testComplement1() } ImmutableConciseSet testSet = ImmutableConciseSet.newImmutableFromMutable(set); - + testSet.toString(); verifyComplement(expected, testSet, NO_COMPLEMENT_LENGTH); } @@ -2029,4 +2033,14 @@ public void testContains() Assert.assertEquals(s, integerSet.contains(i), immutableConciseSet.contains(i)); } } + + private Memory createMemoryFromIntArray(int[] words) + { + //TODO Replace with new constructor from Memory. This is a hack till then + ByteBuffer bb = ByteBuffer.allocate(words.length* Ints.BYTES).order(ByteOrder.nativeOrder()); + for(int i = 0; i < words.length; i++){ + bb.putInt(words[i]); + } + return new NativeMemory(bb); + } } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchHolder.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchHolder.java index 54fa865d7970..826acebd978b 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchHolder.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchHolder.java @@ -25,6 +25,7 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; import com.yahoo.memory.NativeMemory; import com.yahoo.sketches.Family; import com.yahoo.sketches.theta.AnotB; @@ -114,7 +115,7 @@ public int compare(Memory o1, Memory o2) private SketchHolder(Object obj) { Preconditions.checkArgument( - obj instanceof Sketch || obj instanceof Union || obj instanceof Memory, + obj instanceof Sketch || obj instanceof Union || obj instanceof MemoryRegion || obj instanceof Memory , "unknown sketch representation type [%s]", obj.getClass().getName() ); this.obj = obj; diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java index 45ddd390dc17..42db885d2783 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java @@ -21,6 +21,7 @@ import com.yahoo.sketches.theta.Sketch; import io.druid.data.input.InputRow; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -28,8 +29,6 @@ import io.druid.segment.serde.ComplexMetricExtractor; import io.druid.segment.serde.ComplexMetricSerde; -import java.nio.ByteBuffer; - public class SketchMergeComplexMetricSerde extends ComplexMetricSerde { private SketchObjectStrategy strategy = new SketchObjectStrategy(); @@ -64,10 +63,10 @@ public Object extractValue(InputRow inputRow, String metricName) } @Override - public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder) + public void deserializeColumn(PositionalMemoryRegion pMemory, ColumnBuilder columnBuilder) { - GenericIndexed ge = GenericIndexed.read(buffer, strategy); - builder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), ge)); + GenericIndexed ge = GenericIndexed.read(pMemory, strategy); + columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), ge)); } @Override diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java index 2e2550a0b3b4..b4a1fc40ec9c 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java @@ -19,14 +19,11 @@ package io.druid.query.aggregation.datasketches.theta; -import com.yahoo.memory.MemoryRegion; -import com.yahoo.memory.NativeMemory; +import com.yahoo.memory.Memory; import com.yahoo.sketches.theta.Sketch; import io.druid.java.util.common.IAE; import io.druid.segment.data.ObjectStrategy; -import java.nio.ByteBuffer; - public class SketchObjectStrategy implements ObjectStrategy { @@ -45,13 +42,13 @@ public Class getClazz() } @Override - public Object fromByteBuffer(ByteBuffer buffer, int numBytes) + public Object fromMemory(Memory buffer) { - if (numBytes == 0) { + if (buffer.getCapacity() == 0) { return SketchHolder.EMPTY; } - return SketchHolder.of(new MemoryRegion(new NativeMemory(buffer), buffer.position(), numBytes)); + return SketchHolder.of(buffer); } @Override diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java index 6d7dd4fd93e4..6e1b335996b0 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java @@ -27,6 +27,11 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.primitives.Shorts; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; +import com.yahoo.memory.NativeMemory; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; +import javafx.geometry.Pos; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -1287,32 +1292,34 @@ public void toBytesCompact(ByteBuffer buf) */ public static ApproximateHistogram fromBytes(byte[] bytes) { - ByteBuffer buf = ByteBuffer.wrap(bytes); - return fromBytes(buf); + return fromBytes(new NativeMemory(bytes)); } /** * Constructs an ApproximateHistogram object from the given dense byte-buffer representation * - * @param buf ByteBuffer to construct an ApproximateHistogram from * * @return ApproximateHistogram constructed from the given ByteBuffer */ - public static ApproximateHistogram fromBytesDense(ByteBuffer buf) + public static ApproximateHistogram fromBytesDense(Memory memory) { - int size = buf.getInt(); - int binCount = buf.getInt(); + PositionalMemoryRegion pMemory = new PositionalMemoryRegion( + new MemoryRegion(memory, 0, memory.getCapacity())); + int size = Integer.reverseBytes(pMemory.getInt()); + int binCount = Integer.reverseBytes(pMemory.getInt()); float[] positions = new float[size]; long[] bins = new long[size]; - buf.asFloatBuffer().get(positions); - buf.position(buf.position() + Floats.BYTES * positions.length); - buf.asLongBuffer().get(bins); - buf.position(buf.position() + Longs.BYTES * bins.length); + for(int i = 0; i < size; i++){ + positions[i] = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); + } + for(int i = 0; i < size; i++){ + bins[i] = Long.reverseBytes(pMemory.getLong()); + } - float min = buf.getFloat(); - float max = buf.getFloat(); + float min = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); + float max = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); return new ApproximateHistogram(binCount, positions, bins, min, max); } @@ -1320,27 +1327,27 @@ public static ApproximateHistogram fromBytesDense(ByteBuffer buf) /** * Constructs an ApproximateHistogram object from the given dense byte-buffer representation * - * @param buf ByteBuffer to construct an ApproximateHistogram from * * @return ApproximateHistogram constructed from the given ByteBuffer */ - public static ApproximateHistogram fromBytesSparse(ByteBuffer buf) + public static ApproximateHistogram fromBytesSparse(Memory memory) { - int size = buf.getInt(); - int binCount = -1 * buf.getInt(); + PositionalMemoryRegion pMemory = new PositionalMemoryRegion(new MemoryRegion(memory, 0, memory.getCapacity())); + int size = Integer.reverseBytes(pMemory.getInt()); + int binCount = -1 * Integer.reverseBytes(pMemory.getInt()); float[] positions = new float[size]; long[] bins = new long[size]; for (int i = 0; i < binCount; ++i) { - positions[i] = buf.getFloat(); + positions[i] = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); } for (int i = 0; i < binCount; ++i) { - bins[i] = buf.getLong(); + bins[i] = Long.reverseBytes(pMemory.getLong()); } - float min = buf.getFloat(); - float max = buf.getFloat(); + float min = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); + float max = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); return new ApproximateHistogram(binCount, positions, bins, min, max); } @@ -1348,20 +1355,21 @@ public static ApproximateHistogram fromBytesSparse(ByteBuffer buf) /** * Constructs an ApproximateHistogram object from the given compact byte-buffer representation * - * @param buf ByteBuffer to construct an ApproximateHistogram from * * @return ApproximateHistogram constructed from the given ByteBuffer */ - public static ApproximateHistogram fromBytesCompact(ByteBuffer buf) + public static ApproximateHistogram fromBytesCompact(Memory memory) { - short size = (short) (-1 * buf.getShort()); - byte count = buf.get(); + PositionalMemoryRegion pMemory = new PositionalMemoryRegion( + new MemoryRegion(memory, 0, memory.getCapacity())); + short size = (short) (-1 * Short.reverseBytes(pMemory.getShort())); + byte count = pMemory.getByte(); if (count >= 0) { // only exact bins ApproximateHistogram histogram = new ApproximateHistogram(size); for (int i = 0; i < count; ++i) { - histogram.offer(buf.getFloat()); + histogram.offer(Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat())))); } return histogram; } else { @@ -1370,7 +1378,7 @@ public static ApproximateHistogram fromBytesCompact(ByteBuffer buf) Map approx = Maps.newHashMap(); for (int i = 0; i < approxCount; ++i) { - final float value = buf.getFloat(); + final float value = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); if (approx.containsKey(value)) { approx.put(value, approx.get(value) + 1); } else { @@ -1378,15 +1386,15 @@ public static ApproximateHistogram fromBytesCompact(ByteBuffer buf) } } - float min = buf.getFloat(); - float max = buf.getFloat(); + float min = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); + float max = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); - byte exactCount = buf.get(); + byte exactCount = pMemory.getByte(); Map exact = Maps.newHashMap(); for (int i = 0; i < exactCount; ++i) { - final float value = buf.getFloat(); + final float value = Float.intBitsToFloat(Integer.reverseBytes(Float.floatToRawIntBits(pMemory.getFloat()))); if (exact.containsKey(value)) { exact.put(value, exact.get(value) + 1); } else { @@ -1424,25 +1432,23 @@ public static ApproximateHistogram fromBytesCompact(ByteBuffer buf) /** * Constructs an ApproximateHistogram object from the given byte-buffer representation * - * @param buf ByteBuffer to construct an ApproximateHistogram from * * @return ApproximateHistogram constructed from the given ByteBuffer */ - public static ApproximateHistogram fromBytes(ByteBuffer buf) + public static ApproximateHistogram fromBytes(Memory memory) { - ByteBuffer copy = buf.asReadOnlyBuffer(); // negative size indicates compact representation // this works regardless of whether we use int or short for the size since the leftmost bit is the sign bit - if (copy.getShort(buf.position()) < 0) { - return fromBytesCompact(buf); + if (Short.reverseBytes(memory.getShort(0)) < 0) { + return fromBytesCompact(memory); } else { // ignore size - copy.getInt(); + memory.getInt(0); // determine if sparse or dense based on sign of binCount - if (copy.getInt() < 0) { - return fromBytesSparse(buf); + if (Integer.reverseBytes(memory.getInt(Ints.BYTES)) < 0) { + return fromBytesSparse(memory); } else { - return fromBytesDense(buf); + return fromBytesDense(memory); } } } diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java index ddbf948530b5..47639055eecb 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java @@ -27,6 +27,7 @@ import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; +import com.yahoo.memory.NativeMemory; import io.druid.java.util.common.StringUtils; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; @@ -162,7 +163,8 @@ public Object deserialize(Object object) return ah; } else if (object instanceof ByteBuffer) { - final ApproximateHistogram ah = ApproximateHistogram.fromBytes((ByteBuffer) object); + final ApproximateHistogram ah = ApproximateHistogram.fromBytes( + new NativeMemory((ByteBuffer) object)); ah.setLowerLimit(lowerLimit); ah.setUpperLimit(upperLimit); diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramBufferAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramBufferAggregator.java index 297fde021d09..6f15ab72031f 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramBufferAggregator.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramBufferAggregator.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation.histogram; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.query.aggregation.BufferAggregator; import io.druid.segment.FloatColumnSelector; @@ -66,7 +67,7 @@ public void aggregate(ByteBuffer buf, int position) ByteBuffer mutationBuffer = buf.duplicate(); mutationBuffer.position(position); - ApproximateHistogram h0 = ApproximateHistogram.fromBytesDense(mutationBuffer); + ApproximateHistogram h0 = ApproximateHistogram.fromBytesDense((new PositionalMemoryRegion(mutationBuffer)).getRemainingMemory()); h0.offer(selector.get()); mutationBuffer.position(position); @@ -78,7 +79,7 @@ public Object get(ByteBuffer buf, int position) { ByteBuffer mutationBuffer = buf.duplicate(); mutationBuffer.position(position); - return ApproximateHistogram.fromBytes(mutationBuffer); + return ApproximateHistogram.fromBytes((new PositionalMemoryRegion(mutationBuffer)).getRemainingMemory()); } @Override diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingBufferAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingBufferAggregator.java index bd52b603badb..566962688172 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingBufferAggregator.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingBufferAggregator.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation.histogram; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.query.aggregation.BufferAggregator; import io.druid.segment.ObjectColumnSelector; @@ -67,7 +68,8 @@ public void aggregate(ByteBuffer buf, int position) ByteBuffer mutationBuffer = buf.duplicate(); mutationBuffer.position(position); - ApproximateHistogram h0 = ApproximateHistogram.fromBytesDense(mutationBuffer); + ApproximateHistogram h0 = ApproximateHistogram.fromBytesDense( + (new PositionalMemoryRegion(mutationBuffer)).getRemainingMemory()); h0.setLowerLimit(lowerLimit); h0.setUpperLimit(upperLimit); ApproximateHistogram hNext = selector.get(); @@ -82,7 +84,7 @@ public Object get(ByteBuffer buf, int position) { ByteBuffer mutationBuffer = buf.asReadOnlyBuffer(); mutationBuffer.position(position); - return ApproximateHistogram.fromBytesDense(mutationBuffer); + return ApproximateHistogram.fromBytesDense((new PositionalMemoryRegion(mutationBuffer)).getRemainingMemory()); } @Override diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java index 20c9ea691eb1..1b370f1f5011 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java @@ -20,7 +20,10 @@ package io.druid.query.aggregation.histogram; import com.google.common.collect.Ordering; +import com.yahoo.memory.Memory; +import com.yahoo.memory.NativeMemory; import io.druid.data.input.InputRow; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -90,11 +93,9 @@ public ApproximateHistogram extractValue(InputRow inputRow, String metricName) } @Override - public void deserializeColumn( - ByteBuffer byteBuffer, ColumnBuilder columnBuilder - ) + public void deserializeColumn(PositionalMemoryRegion pMemory, ColumnBuilder columnBuilder) { - final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy()); + final GenericIndexed column = GenericIndexed.read(pMemory, getObjectStrategy()); columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); } @@ -113,7 +114,12 @@ public ApproximateHistogram fromByteBuffer(ByteBuffer buffer, int numBytes) { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); - return ApproximateHistogram.fromBytes(readOnlyBuffer); + return ApproximateHistogram.fromBytes(new NativeMemory(readOnlyBuffer)); + } + + @Override public ApproximateHistogram fromMemory(Memory memory) + { + return ApproximateHistogram.fromBytes(memory); } @Override diff --git a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java index 4bb83ada71ae..fd78720b2b14 100644 --- a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java +++ b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java @@ -20,7 +20,9 @@ package io.druid.query.aggregation.variance; import com.google.common.collect.Ordering; +import com.yahoo.memory.Memory; import io.druid.data.input.InputRow; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -79,10 +81,10 @@ public VarianceAggregatorCollector extractValue(InputRow inputRow, String metric @Override public void deserializeColumn( - ByteBuffer byteBuffer, ColumnBuilder columnBuilder + PositionalMemoryRegion pMemory, ColumnBuilder columnBuilder ) { - final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy()); + final GenericIndexed column = GenericIndexed.read(pMemory, getObjectStrategy()); columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); } @@ -105,6 +107,11 @@ public VarianceAggregatorCollector fromByteBuffer(ByteBuffer buffer, int numByte return VarianceAggregatorCollector.from(readOnlyBuffer); } + @Override public VarianceAggregatorCollector fromMemory(Memory memory) + { + return null; + } + @Override public byte[] toBytes(VarianceAggregatorCollector collector) { diff --git a/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceSerdeTest.java b/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceSerdeTest.java index 47f5b0008dfe..4f8b737bd8a8 100644 --- a/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceSerdeTest.java +++ b/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceSerdeTest.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation.variance; +import com.yahoo.memory.NativeMemory; import io.druid.segment.data.ObjectStrategy; import org.junit.Assert; import org.junit.Test; @@ -39,7 +40,7 @@ public void testSerde() for (int i = 0; i < 100; i++) { byte[] array = strategy.toBytes(holder); Assert.assertArrayEquals(array, holder.toByteArray()); - Assert.assertEquals(holder, strategy.fromByteBuffer(ByteBuffer.wrap(array), array.length)); + Assert.assertEquals(holder, strategy.fromMemory(new NativeMemory(array))); holder.add(r.nextFloat()); } } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index 937c0db262db..1eceec9d34f7 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -214,7 +214,7 @@ public static final InputRow fromBytes(byte[] data, AggregatorFactory[] aggs) } else { ComplexMetricSerde serde = getComplexMetricSerde(type); byte[] value = readBytes(in); - event.put(metric, serde.fromBytes(value, 0, value.length)); + event.put(metric, serde.fromBytes(value)); } } diff --git a/java-util/pom.xml b/java-util/pom.xml index b38d6ccc2ee5..368b5be36a77 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -85,6 +85,11 @@ + + com.yahoo.datasketches + memory + 0.8.4 + diff --git a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java index e340c2a087fd..ceb685d0e55a 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java @@ -21,6 +21,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; +import com.yahoo.memory.Memory; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; @@ -54,6 +55,25 @@ public static String fromUtf8(final ByteBuffer buffer, final int numBytes) return fromUtf8(bytes); } + public static String fromUtf8(final Memory memory) + { + final byte[] bytes = new byte[(int)memory.getCapacity()]; + memory.getByteArray(0, bytes, 0, (int)memory.getCapacity()); + return fromUtf8(bytes); + } + +// public static byte[] toBigEndianInt(final Memory memory, int numBytes) +// { +// byte[] bytes = new byte[numBytes]; +// int b = 0; +// while(b < numBytes){ +// bytes[numBytes-1-b] = memory.getByte(b); +// b++; +// } +// +// return bytes; +// } + public static String fromUtf8(final ByteBuffer buffer) { return fromUtf8(buffer, buffer.remaining()); diff --git a/java-util/src/main/java/io/druid/java/util/common/io/smoosh/PositionalMemoryRegion.java b/java-util/src/main/java/io/druid/java/util/common/io/smoosh/PositionalMemoryRegion.java new file mode 100644 index 000000000000..2f7cb25c6959 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/common/io/smoosh/PositionalMemoryRegion.java @@ -0,0 +1,238 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.java.util.common.io.smoosh; + +import java.nio.ByteBuffer; + +import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; +import com.yahoo.memory.NativeMemory; + +import java.nio.ByteOrder; + +/** + * Created by niketh on 1/25/17. + */ +public class PositionalMemoryRegion +{ + private final Memory memory; + long offSet = 0; + long sizeOffset = 0; + + public Memory getReadOnlyMemory() + { + return memory.asReadOnlyMemory(); + } + + public Memory getMemory() + { + return memory; + } + + public ByteBuffer getByteBuffer() + { + return memory.byteBuffer(); + } + + public Memory getRemainingMemory() + { + return new MemoryRegion(memory, offSet, sizeOffset - offSet); + } + + public PositionalMemoryRegion(byte[] bytes) + { + this.memory = new NativeMemory(bytes); + this.offSet = 0; + this.sizeOffset = memory.getCapacity(); + } + + public PositionalMemoryRegion(int a) + { + this.memory = new NativeMemory(ByteBuffer.allocate(Ints.BYTES).order(ByteOrder.nativeOrder()).putInt(a)); + this.offSet = 0; + this.sizeOffset = memory.getCapacity(); + } + + public PositionalMemoryRegion(MemoryRegion memory) + { + this.memory = memory; + this.offSet = 0; + this.sizeOffset = memory.getCapacity(); + } + + public PositionalMemoryRegion(ByteBuffer byteBuffer) + { + Memory mem = new NativeMemory(byteBuffer); + this.memory = new MemoryRegion(mem, byteBuffer.position(), byteBuffer.capacity() - byteBuffer.position()); + this.offSet = 0; + this.sizeOffset = memory.getCapacity(); + } + + private PositionalMemoryRegion(PositionalMemoryRegion pMemory) + { + this.memory = pMemory.memory; + this.offSet = pMemory.position(); + this.sizeOffset = pMemory.getCapacity(); + } + + public PositionalMemoryRegion(PositionalMemoryRegion memory, long position, long capacity) + { + this.memory = new MemoryRegion(memory.getMemory(), position, capacity); + this.offSet = 0; + this.sizeOffset = this.memory.getCapacity(); + } + + public int getInt(){ + long offSet = this.offSet; + this.offSet += Integer.BYTES; + return memory.getInt(offSet); + } + + public long getLong(){ + long offSet = this.offSet; + this.offSet += Long.BYTES; + return memory.getLong(offSet); + } + + public void getFloatArray(long offSet, float[] dst, int dstStart, int dstCapacity) + { + memory.getFloatArray(offSet, dst, dstStart, dstCapacity); + } + + public void getLongArray(long offSet, long[] dst, int dstStart, int dstCapacity) + { + memory.getLongArray(offSet, dst, dstStart, dstCapacity); + } + + public float getFloat(){ + long offSet = this.offSet; + this.offSet += Float.BYTES; + return memory.getFloat(offSet); + } + + public short getShort(){ + long offSet = this.offSet; + this.offSet += Short.BYTES; + return memory.getShort(offSet); + } + + public short getShort(long offSet){ + return memory.getShort(offSet); + } + + + public int getInt(long offSet){ + return memory.getInt(offSet); + } + + public long getLong(long offSet) + { + return memory.getLong(offSet); + } + + public float getFloat(long offSet) + { + return memory.getFloat(offSet); + } + + public byte getByte(){ + long offSet = this.offSet; + this.offSet += Byte.BYTES; + return memory.getByte(offSet); + } + + public byte getByte(long offSet){ + return memory.getByte(offSet); + } + + public byte[] getBytes(int numBytes) + { + byte[] bytes = new byte[numBytes]; + long offSet = this.offSet; + this.offSet += numBytes; + memory.getByteArray(offSet, bytes, 0, numBytes); + return bytes; + } + + public void putByte(Byte b){ + long offSet = this.offSet; + this.offSet += Byte.BYTES; + memory.putByte(offSet, b); + } + + public void putInt(int b){ + long offSet = this.offSet; + this.offSet += Ints.BYTES; + memory.putInt(offSet, b); + } + + public void putByteArray(byte[] bytes){ + long offSet = this.offSet; + this.offSet += bytes.length; + memory.putByteArray(offSet, bytes, 0, bytes.length); + } + + public boolean hasRemaining() + { + return sizeOffset - offSet > 0 ? true : false; + } + + public long getCapacity() + { + return memory.getCapacity(); + } + public long position() + { + return offSet; + } + + public void position(long offSet) + { + this.offSet = offSet; + } + + public long remaining() + { + return sizeOffset - offSet; + } + + public long limit() + { + return sizeOffset; + } + + public void limit(long offSet){ + this.sizeOffset = offSet; + } + + public long capacity() + { + return memory.getCapacity(); + } + + public PositionalMemoryRegion duplicate() + { + return new PositionalMemoryRegion(this); + } + + public static void main(String[] args){ + + } +} diff --git a/java-util/src/main/java/io/druid/java/util/common/io/smoosh/SmooshedFileMapper.java b/java-util/src/main/java/io/druid/java/util/common/io/smoosh/SmooshedFileMapper.java index ba6da3b5fdc3..e3b5a75f1b1e 100644 --- a/java-util/src/main/java/io/druid/java/util/common/io/smoosh/SmooshedFileMapper.java +++ b/java-util/src/main/java/io/druid/java/util/common/io/smoosh/SmooshedFileMapper.java @@ -25,7 +25,9 @@ import com.google.common.collect.Maps; import com.google.common.io.Closeables; import com.google.common.io.Files; -import io.druid.java.util.common.ByteBufferUtils; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryMappedFile; +import com.yahoo.memory.MemoryRegion; import io.druid.java.util.common.ISE; import java.io.BufferedReader; @@ -96,6 +98,7 @@ public static SmooshedFileMapper load(File baseDir) throws IOException private final List outFiles; private final Map internalFiles; private final List buffersList = Lists.newArrayList(); + private final List memoryList = Lists.newArrayList(); private SmooshedFileMapper( List outFiles, @@ -134,13 +137,48 @@ public ByteBuffer mapFile(String name) throws IOException return retVal.slice(); } + public Memory mapFileToMemory(String name) throws IOException + { + final Metadata metadata = internalFiles.get(name); + if (metadata == null) { + return null; + } + + final int fileNum = metadata.getFileNum(); + while (memoryList.size() <= fileNum) { + buffersList.add(null); + memoryList.add(null); + } + + MappedByteBuffer mappedBuffer = buffersList.get(fileNum); + if (mappedBuffer == null) { + mappedBuffer = Files.map(outFiles.get(fileNum)); + System.out.println("Buffer Size=" + mappedBuffer.remaining() + " " + mappedBuffer.capacity()); + buffersList.set(fileNum, mappedBuffer); + } + + MemoryMappedFile mappedMemory = memoryList.get(fileNum); + if(mappedMemory == null) + { + try { + mappedMemory = new MemoryMappedFile(outFiles.get(fileNum), 0, outFiles.get(fileNum).length()); + } catch (Exception e) { + e.printStackTrace(); + } + memoryList.set(fileNum, mappedMemory); + } + + return new MemoryRegion(mappedMemory, metadata.getStartOffset(), + metadata.getEndOffset() - metadata.getStartOffset()); + } + @Override public void close() { Throwable thrown = null; - for (MappedByteBuffer mappedByteBuffer : buffersList) { + for (MemoryMappedFile mappedMemory : memoryList) { try { - ByteBufferUtils.unmap(mappedByteBuffer); + mappedMemory.freeMemory(); } catch (Throwable t) { if (thrown == null) { thrown = t; @@ -149,6 +187,17 @@ public void close() } } } +// for (MappedByteBuffer mappedByteBuffer : buffersList) { +// try { +// ByteBufferUtils.unmap(mappedByteBuffer); +// } catch (Throwable t) { +// if (thrown == null) { +// thrown = t; +// } else { +// thrown.addSuppressed(t); +// } +// } +// } Throwables.propagateIfPossible(thrown); } } diff --git a/java-util/src/test/java/io/druid/java/util/common/io/smoosh/SmooshedFileMapperTest.java b/java-util/src/test/java/io/druid/java/util/common/io/smoosh/SmooshedFileMapperTest.java index 04cf1fe85a01..386304f997ff 100644 --- a/java-util/src/test/java/io/druid/java/util/common/io/smoosh/SmooshedFileMapperTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/io/smoosh/SmooshedFileMapperTest.java @@ -21,6 +21,7 @@ import com.google.common.io.Files; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; import io.druid.java.util.common.BufferUtils; import io.druid.java.util.common.ISE; import junit.framework.Assert; @@ -211,5 +212,14 @@ private void validateOutput(File baseDir) throws IOException Assert.assertEquals(i, buf.getInt()); } } + + try (SmooshedFileMapper mapper = SmooshedFileMapper.load(baseDir)) { + for (int i = 0; i < 20; ++i) { + Memory mem = mapper.mapFileToMemory(String.format("%d", i)); + Assert.assertEquals(4, mem.getCapacity()); + // The entries are written in Big Endian in the tests + Assert.assertEquals(i, Integer.reverseBytes(mem.getInt(0))); + } + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java index 7e4272e63bba..138f5045b2a7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java @@ -23,6 +23,8 @@ import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogHash; import io.druid.hll.HyperLogLogCollector; +import com.yahoo.memory.Memory; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -95,11 +97,9 @@ public HyperLogLogCollector extractValue(InputRow inputRow, String metricName) } @Override - public void deserializeColumn( - ByteBuffer byteBuffer, ColumnBuilder columnBuilder - ) + public void deserializeColumn(PositionalMemoryRegion pMemory, ColumnBuilder columnBuilder) { - final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy()); + final GenericIndexed column = GenericIndexed.read(pMemory, getObjectStrategy()); columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); } @@ -114,12 +114,17 @@ public Class getClazz() return HyperLogLogCollector.class; } - @Override - public HyperLogLogCollector fromByteBuffer(ByteBuffer buffer, int numBytes) +// @Override +// public HyperLogLogCollector fromByteBuffer(ByteBuffer buffer, int numBytes) +// { +// final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); +// readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); +// return HyperLogLogCollector.makeCollector(readOnlyBuffer); +// } + + @Override public HyperLogLogCollector fromMemory(Memory memory) { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); - return HyperLogLogCollector.makeCollector(readOnlyBuffer); + return null; } @Override diff --git a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java index e2a47e36eb5c..821bfecdb6cf 100644 --- a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java @@ -21,6 +21,7 @@ import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier; import io.druid.segment.data.IndexedInts; @@ -77,20 +78,18 @@ public void writeToChannel(WritableByteChannel channel) throws IOException valueSupplier.writeToChannel(channel); } - public static CompressedVSizeIndexedSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order, SmooshedFileMapper fileMapper) + public static CompressedVSizeIndexedSupplier fromMemory(PositionalMemoryRegion pMemory, ByteOrder order) { - byte versionFromBuffer = buffer.get(); + byte versionFromBuffer = pMemory.getByte(); if (versionFromBuffer == version) { - CompressedVSizeIntsIndexedSupplier offsetSupplier = CompressedVSizeIntsIndexedSupplier.fromByteBuffer( - buffer, - order, - fileMapper + CompressedVSizeIntsIndexedSupplier offsetSupplier = CompressedVSizeIntsIndexedSupplier.fromMemory( + pMemory, + order ); - CompressedVSizeIntsIndexedSupplier valueSupplier = CompressedVSizeIntsIndexedSupplier.fromByteBuffer( - buffer, - order, - fileMapper + CompressedVSizeIntsIndexedSupplier valueSupplier = CompressedVSizeIntsIndexedSupplier.fromMemory( + pMemory, + order ); return new CompressedVSizeIndexedSupplier(offsetSupplier, valueSupplier); } diff --git a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedV3Supplier.java b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedV3Supplier.java index 26e10afa5543..87e17b70798d 100644 --- a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedV3Supplier.java +++ b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedV3Supplier.java @@ -20,7 +20,7 @@ package io.druid.segment; import io.druid.java.util.common.IAE; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.data.CompressedIntsIndexedSupplier; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier; @@ -59,24 +59,19 @@ public class CompressedVSizeIndexedV3Supplier implements WritableSupplier availableDimensions = GenericIndexed.read( - indexBuffer, - GenericIndexed.STRING_STRATEGY, - smooshedFiles + pIndexMemory.getByte(); // Skip the version byte + final GenericIndexed availableDimensions = GenericIndexed.read(pIndexMemory, GenericIndexed.STRING_STRATEGY ); - final GenericIndexed availableMetrics = GenericIndexed.read( - indexBuffer, - GenericIndexed.STRING_STRATEGY, - smooshedFiles + final GenericIndexed availableMetrics = GenericIndexed.read(pIndexMemory, GenericIndexed.STRING_STRATEGY ); - final Interval dataInterval = new Interval(serializerUtils.readString(indexBuffer)); + final Interval dataInterval = new Interval(serializerUtils.readString(pIndexMemory)); final BitmapSerdeFactory bitmapSerdeFactory = new BitmapSerde.LegacyBitmapSerdeFactory(); - CompressedLongsIndexedSupplier timestamps = CompressedLongsIndexedSupplier.fromByteBuffer( - smooshedFiles.mapFile(makeTimeFile(inDir, BYTE_ORDER).getName()), - BYTE_ORDER, - smooshedFiles + Memory timestampsMemory = smooshedFiles.mapFileToMemory(makeTimeFile(inDir, BYTE_ORDER).getName()); + PositionalMemoryRegion pTimestampsMemory = new PositionalMemoryRegion( + new MemoryRegion(timestampsMemory,0, timestampsMemory.getCapacity())); + CompressedLongsIndexedSupplier timestamps = CompressedLongsIndexedSupplier.fromMemory( + pTimestampsMemory, + BYTE_ORDER ); Map metrics = Maps.newLinkedHashMap(); for (String metric : availableMetrics) { final String metricFilename = makeMetricFile(inDir, metric, BYTE_ORDER).getName(); - final MetricHolder holder = MetricHolder.fromByteBuffer(smooshedFiles.mapFile(metricFilename), smooshedFiles); - + final MetricHolder holder = MetricHolder.fromMemory(smooshedFiles.mapFileToMemory(metricFilename)); if (!metric.equals(holder.getName())) { throw new ISE("Metric[%s] loaded up metric[%s] from disk. File names do matter.", metric, holder.getName()); } @@ -430,8 +429,9 @@ public MMappedIndex mapDir(File inDir) throws IOException Map> bitmaps = Maps.newHashMap(); for (String dimension : IndexedIterable.create(availableDimensions)) { - ByteBuffer dimBuffer = smooshedFiles.mapFile(makeDimFile(inDir, dimension).getName()); - String fileDimensionName = serializerUtils.readString(dimBuffer); + Memory dimMemory = smooshedFiles.mapFileToMemory(makeDimFile(inDir, dimension).getName()); + PositionalMemoryRegion pDimMemory = new PositionalMemoryRegion(new MemoryRegion(dimMemory, 0, dimMemory.getCapacity())); + String fileDimensionName = serializerUtils.readStringReverse(pDimMemory); Preconditions.checkState( dimension.equals(fileDimensionName), "Dimension file[%s] has dimension[%s] in it!?", @@ -439,25 +439,29 @@ public MMappedIndex mapDir(File inDir) throws IOException fileDimensionName ); - dimValueLookups.put(dimension, GenericIndexed.read(dimBuffer, GenericIndexed.STRING_STRATEGY)); - dimColumns.put(dimension, VSizeIndexed.readFromByteBuffer(dimBuffer)); + dimValueLookups.put(dimension, GenericIndexed.read(pDimMemory, GenericIndexed.STRING_STRATEGY)); + dimColumns.put(dimension, VSizeIndexed.readFromMemory(pDimMemory)); } - ByteBuffer invertedBuffer = smooshedFiles.mapFile("inverted.drd"); + Memory invertedDrdMemory = smooshedFiles.mapFileToMemory("inverted.drd"); + PositionalMemoryRegion pInvertedDrdMemory = new PositionalMemoryRegion( + new MemoryRegion(invertedDrdMemory, 0, invertedDrdMemory.getCapacity())); for (int i = 0; i < availableDimensions.size(); ++i) { bitmaps.put( - serializerUtils.readString(invertedBuffer), - GenericIndexed.read(invertedBuffer, bitmapSerdeFactory.getObjectStrategy()) + serializerUtils.readStringReverse(pInvertedDrdMemory), + GenericIndexed.read(pInvertedDrdMemory, bitmapSerdeFactory.getObjectStrategy()) ); } Map spatialIndexed = Maps.newHashMap(); - ByteBuffer spatialBuffer = smooshedFiles.mapFile("spatial.drd"); - while (spatialBuffer != null && spatialBuffer.hasRemaining()) { + Memory spatialMemory = smooshedFiles.mapFileToMemory("spatial.drd"); + PositionalMemoryRegion pSpatialMemory = new PositionalMemoryRegion(new MemoryRegion(spatialMemory, 0, + spatialMemory.getCapacity())); + while (pSpatialMemory != null && pSpatialMemory.hasRemaining()) { spatialIndexed.put( - serializerUtils.readString(spatialBuffer), - ByteBufferSerializer.read( - spatialBuffer, + serializerUtils.readStringReverse(pSpatialMemory), + MemorySerializer.read( + pSpatialMemory, new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()) ) ); @@ -508,24 +512,28 @@ public void convertV8toV9(File v8Dir, File v9Dir, IndexSpec indexSpec) ByteStreams.write(Ints.toByteArray(9), Files.newOutputStreamSupplier(new File(v9Dir, "version.bin"))); Map> bitmapIndexes = Maps.newHashMap(); - final ByteBuffer invertedBuffer = v8SmooshedFiles.mapFile("inverted.drd"); + final Memory invertedMemory = v8SmooshedFiles.mapFileToMemory("inverted.drd"); + PositionalMemoryRegion pInvertedMemory = new PositionalMemoryRegion( + new MemoryRegion(invertedMemory,0, invertedMemory.getCapacity())); BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); - while (invertedBuffer.hasRemaining()) { - final String dimName = serializerUtils.readString(invertedBuffer); + while (pInvertedMemory.hasRemaining()) { bitmapIndexes.put( - dimName, - GenericIndexed.read(invertedBuffer, bitmapSerdeFactory.getObjectStrategy(), v8SmooshedFiles) + serializerUtils.readStringReverse(pInvertedMemory), + GenericIndexed.read(pInvertedMemory, bitmapSerdeFactory.getObjectStrategy(), + v8SmooshedFiles) ); } Map spatialIndexes = Maps.newHashMap(); - final ByteBuffer spatialBuffer = v8SmooshedFiles.mapFile("spatial.drd"); - while (spatialBuffer != null && spatialBuffer.hasRemaining()) { + final Memory spatialMemory = v8SmooshedFiles.mapFileToMemory("spatial.drd"); + PositionalMemoryRegion pSpatialMemory = new PositionalMemoryRegion( + new MemoryRegion(spatialMemory, 0, spatialMemory.getCapacity())); + while (pSpatialMemory != null && pSpatialMemory.hasRemaining()) { spatialIndexes.put( - serializerUtils.readString(spatialBuffer), - ByteBufferSerializer.read( - spatialBuffer, new IndexedRTree.ImmutableRTreeObjectStrategy( + serializerUtils.readStringReverse(pSpatialMemory), + MemorySerializer.read( + pSpatialMemory, new IndexedRTree.ImmutableRTreeObjectStrategy( bitmapSerdeFactory.getBitmapFactory() ) ) @@ -542,8 +550,10 @@ public void convertV8toV9(File v8Dir, File v9Dir, IndexSpec indexSpec) final List outParts = Lists.newArrayList(); - ByteBuffer dimBuffer = v8SmooshedFiles.mapFile(filename); - String dimension = serializerUtils.readString(dimBuffer); + Memory dimBuffer = v8SmooshedFiles.mapFileToMemory(filename); + PositionalMemoryRegion pDimMem = new PositionalMemoryRegion(new MemoryRegion(dimBuffer, + 0 ,dimBuffer.getCapacity())); + String dimension = serializerUtils.readStringReverse(pDimMem); if (!filename.equals(String.format("dim_%s.drd", dimension))) { throw new ISE("loaded dimension[%s] from file[%s]", dimension, filename); } @@ -552,8 +562,7 @@ public void convertV8toV9(File v8Dir, File v9Dir, IndexSpec indexSpec) serializerUtils.writeString(nameBAOS, dimension); outParts.add(ByteBuffer.wrap(nameBAOS.toByteArray())); - GenericIndexed dictionary = GenericIndexed.read( - dimBuffer, GenericIndexed.STRING_STRATEGY + GenericIndexed dictionary = GenericIndexed.read(pDimMem, GenericIndexed.STRING_STRATEGY ); if (dictionary.size() == 0) { @@ -564,7 +573,7 @@ public void convertV8toV9(File v8Dir, File v9Dir, IndexSpec indexSpec) int emptyStrIdx = dictionary.indexOf(""); List singleValCol = null; - VSizeIndexed multiValCol = VSizeIndexed.readFromByteBuffer(dimBuffer.asReadOnlyBuffer()); + VSizeIndexed multiValCol = VSizeIndexed.readFromMemory(pDimMem); GenericIndexed bitmaps = bitmapIndexes.get(dimension); ImmutableRTree spatialIndex = spatialIndexes.get(dimension); @@ -708,7 +717,7 @@ public int size() continue; } - MetricHolder holder = MetricHolder.fromByteBuffer(v8SmooshedFiles.mapFile(filename), v8SmooshedFiles); + MetricHolder holder = MetricHolder.fromMemory(v8SmooshedFiles.mapFileToMemory(filename)); final String metric = holder.getName(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); @@ -762,10 +771,13 @@ public int size() serdeficator.write(channel, v9Smoosher); channel.close(); } else if (String.format("time_%s.drd", BYTE_ORDER).equals(filename)) { - CompressedLongsIndexedSupplier timestamps = CompressedLongsIndexedSupplier.fromByteBuffer( - v8SmooshedFiles.mapFile(filename), - BYTE_ORDER, - v8SmooshedFiles + + Memory timestampsMemory = v8SmooshedFiles.mapFileToMemory(filename); + PositionalMemoryRegion pTimestampsMemory = new PositionalMemoryRegion( + new MemoryRegion(timestampsMemory, 0, timestampsMemory.getCapacity())); + CompressedLongsIndexedSupplier timestamps = CompressedLongsIndexedSupplier.fromMemory( + pTimestampsMemory, + BYTE_ORDER ); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); @@ -793,12 +805,15 @@ public int size() } } - final ByteBuffer indexBuffer = v8SmooshedFiles.mapFile("index.drd"); + final Memory indexMemory = v8SmooshedFiles.mapFileToMemory("index.drd"); + PositionalMemoryRegion pIndexMemory = new PositionalMemoryRegion( + new MemoryRegion(indexMemory, 0, indexMemory.getCapacity())); - indexBuffer.get(); // Skip the version byte + pIndexMemory.getByte(); // Skip the version byte final GenericIndexed dims8 = GenericIndexed.read( - indexBuffer, GenericIndexed.STRING_STRATEGY + pIndexMemory, GenericIndexed.STRING_STRATEGY ); + final GenericIndexed dims9 = GenericIndexed.fromIterable( Iterables.filter( dims8, new Predicate() @@ -813,11 +828,12 @@ public boolean apply(String s) GenericIndexed.STRING_STRATEGY ); final GenericIndexed availableMetrics = GenericIndexed.read( - indexBuffer, GenericIndexed.STRING_STRATEGY + pIndexMemory, GenericIndexed.STRING_STRATEGY ); - final Interval dataInterval = new Interval(serializerUtils.readString(indexBuffer)); + + final Interval dataInterval = new Interval(serializerUtils.readStringReverse(pIndexMemory)); final BitmapSerdeFactory segmentBitmapSerdeFactory = mapper.readValue( - serializerUtils.readString(indexBuffer), + serializerUtils.readStringReverse(pIndexMemory), BitmapSerdeFactory.class ); @@ -838,9 +854,11 @@ public boolean apply(String s) serializerUtils.writeString(writer, segmentBitmapSerdeFactoryString); writer.close(); - final ByteBuffer metadataBuffer = v8SmooshedFiles.mapFile("metadata.drd"); - if (metadataBuffer != null) { - v9Smoosher.add("metadata.drd", metadataBuffer); + final Memory metadataMemory = v8SmooshedFiles.mapFileToMemory("metadata.drd"); + byte[] bytes = new byte[(int)metadataMemory.getCapacity()]; + metadataMemory.getByteArray(0, bytes, 0, bytes.length); + if (metadataMemory != null) { + v9Smoosher.add("metadata.drd", ByteBuffer.wrap(bytes)); } log.info("Skipped files[%s]", skippedFiles); @@ -986,22 +1004,20 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException SmooshedFileMapper smooshedFiles = Smoosh.map(inDir); - ByteBuffer indexBuffer = smooshedFiles.mapFile("index.drd"); + Memory memory = smooshedFiles.mapFileToMemory("index.drd"); + PositionalMemoryRegion pMemoryRegion = new PositionalMemoryRegion( + new MemoryRegion(memory, 0, memory.getCapacity())); /** * Index.drd should consist of the segment version, the columns and dimensions of the segment as generic * indexes, the interval start and end millis as longs (in 16 bytes), and a bitmap index type. */ - final GenericIndexed cols = GenericIndexed.read( - indexBuffer, - GenericIndexed.STRING_STRATEGY, - smooshedFiles - ); - final GenericIndexed dims = GenericIndexed.read( - indexBuffer, - GenericIndexed.STRING_STRATEGY, - smooshedFiles - ); - final Interval dataInterval = new Interval(indexBuffer.getLong(), indexBuffer.getLong()); + final GenericIndexed cols = GenericIndexed.read(pMemoryRegion, GenericIndexed.STRING_STRATEGY); + + final GenericIndexed dims = GenericIndexed.read(pMemoryRegion, GenericIndexed.STRING_STRATEGY); + + final Interval dataInterval = new Interval(Long.reverseBytes(pMemoryRegion.getLong()), + Long.reverseBytes(pMemoryRegion.getLong())); + final BitmapSerdeFactory segmentBitmapSerdeFactory; /** @@ -1009,18 +1025,18 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException * index to use. Since we cannot very cleanly build v9 segments directly, we are using a workaround where * this information is appended to the end of index.drd. */ - if (indexBuffer.hasRemaining()) { - segmentBitmapSerdeFactory = mapper.readValue(serializerUtils.readString(indexBuffer), BitmapSerdeFactory.class); + if (pMemoryRegion.hasRemaining()) { + segmentBitmapSerdeFactory = mapper.readValue(serializerUtils.readStringReverse(pMemoryRegion), BitmapSerdeFactory.class); } else { segmentBitmapSerdeFactory = new BitmapSerde.LegacyBitmapSerdeFactory(); } Metadata metadata = null; - ByteBuffer metadataBB = smooshedFiles.mapFile("metadata.drd"); + Memory metadataBB = smooshedFiles.mapFileToMemory("metadata.drd"); if (metadataBB != null) { try { metadata = mapper.readValue( - serializerUtils.readBytes(metadataBB, metadataBB.remaining()), + serializerUtils.readBytes(metadataBB, (int)metadataBB.getCapacity()), Metadata.class ); } @@ -1037,10 +1053,10 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException Map columns = Maps.newHashMap(); for (String columnName : cols) { - columns.put(columnName, deserializeColumn(mapper, smooshedFiles.mapFile(columnName), smooshedFiles)); + columns.put(columnName, deserializeColumn(mapper, smooshedFiles.mapFileToMemory(columnName), smooshedFiles)); } - columns.put(Column.TIME_COLUMN_NAME, deserializeColumn(mapper, smooshedFiles.mapFile("__time"), smooshedFiles)); + columns.put(Column.TIME_COLUMN_NAME, deserializeColumn(mapper, smooshedFiles.mapFileToMemory("__time"), smooshedFiles)); final QueryableIndex index = new SimpleQueryableIndex( dataInterval, cols, dims, segmentBitmapSerdeFactory.getBitmapFactory(), columns, smooshedFiles, metadata @@ -1051,13 +1067,15 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException return index; } - private Column deserializeColumn(ObjectMapper mapper, ByteBuffer byteBuffer, SmooshedFileMapper smooshedFiles) - throws IOException + private Column deserializeColumn(ObjectMapper mapper, Memory memory, SmooshedFileMapper smooshedFiles) throws IOException { + PositionalMemoryRegion pMemoryRegion = new PositionalMemoryRegion(new MemoryRegion(memory, 0, memory.getCapacity())); + ColumnDescriptor serde = mapper.readValue( - serializerUtils.readString(byteBuffer), ColumnDescriptor.class + serializerUtils.readStringReverse(pMemoryRegion), ColumnDescriptor.class ); - return serde.read(byteBuffer, columnConfig, smooshedFiles); + + return serde.read(pMemoryRegion.getRemainingMemory(), columnConfig, smooshedFiles); } } diff --git a/processing/src/main/java/io/druid/segment/MetricHolder.java b/processing/src/main/java/io/druid/segment/MetricHolder.java index 1782d8277361..3b590a8ca8d4 100644 --- a/processing/src/main/java/io/druid/segment/MetricHolder.java +++ b/processing/src/main/java/io/druid/segment/MetricHolder.java @@ -23,10 +23,12 @@ import com.google.common.io.ByteStreams; import com.google.common.io.InputSupplier; import com.google.common.io.OutputSupplier; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.data.CompressedFloatsIndexedSupplier; import io.druid.segment.data.CompressedLongsIndexedSupplier; import io.druid.segment.data.FloatSupplierSerializer; @@ -123,43 +125,43 @@ public static void writeToChannel(MetricHolder holder, WritableByteChannel out) } } - public static MetricHolder fromByteBuffer(ByteBuffer buf, SmooshedFileMapper mapper) throws IOException + public static MetricHolder fromMemory(Memory memory) throws IOException { - return fromByteBuffer(buf, null, mapper); + return fromMemory(memory, null); } - public static MetricHolder fromByteBuffer(ByteBuffer buf, ObjectStrategy strategy, SmooshedFileMapper mapper) - throws IOException + public static MetricHolder fromMemory(Memory memory, ObjectStrategy strategy) throws IOException { - final byte ver = buf.get(); + PositionalMemoryRegion pMemory = new PositionalMemoryRegion(new MemoryRegion(memory, 0, memory.getCapacity())); + final byte ver = pMemory.getByte(); if (version[0] != ver) { throw new ISE("Unknown version[%s] of MetricHolder", ver); } - final String metricName = serializerUtils.readString(buf); - final String typeName = serializerUtils.readString(buf); + final String metricName = serializerUtils.readStringReverse(pMemory); + final String typeName = serializerUtils.readStringReverse(pMemory); MetricHolder holder = new MetricHolder(metricName, typeName); switch (holder.type) { - case LONG: - holder.longType = CompressedLongsIndexedSupplier.fromByteBuffer(buf, ByteOrder.nativeOrder(), mapper); - break; - case FLOAT: - holder.floatType = CompressedFloatsIndexedSupplier.fromByteBuffer(buf, ByteOrder.nativeOrder(), mapper); - break; - case COMPLEX: - if (strategy != null) { - holder.complexType = GenericIndexed.read(buf, strategy, mapper); - } else { - final ComplexMetricSerde serdeForType = ComplexMetrics.getSerdeForType(holder.getTypeName()); + case LONG: + holder.longType = CompressedLongsIndexedSupplier.fromMemory(pMemory, ByteOrder.nativeOrder()); + break; + case FLOAT: + holder.floatType = CompressedFloatsIndexedSupplier.fromMemory(pMemory, ByteOrder.nativeOrder()); + break; + case COMPLEX: + if (strategy != null) { + holder.complexType = GenericIndexed.read(pMemory, strategy); + } else { + final ComplexMetricSerde serdeForType = ComplexMetrics.getSerdeForType(holder.getTypeName()); - if (serdeForType == null) { - throw new ISE("Unknown type[%s], cannot load.", holder.getTypeName()); - } - - holder.complexType = GenericIndexed.read(buf, serdeForType.getObjectStrategy()); + if (serdeForType == null) { + throw new ISE("Unknown type[%s], cannot load.", holder.getTypeName()); } - break; + + holder.complexType = GenericIndexed.read(pMemory, serdeForType.getObjectStrategy()); + } + break; } return holder; diff --git a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java index c017841f2e86..0b472df14c5b 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java @@ -168,6 +168,7 @@ public Closeable getSubColumn(Column column) @Override public int[] getEncodedKeyComponentFromColumn(Closeable column, int currRow) { + System.out.println("CurrRow=" + currRow); DictionaryEncodedColumn dict = (DictionaryEncodedColumn) column; int[] theVals; if (dict.hasMultipleValues()) { diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerLegacy.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerLegacy.java index 558016702268..b145454a0e00 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerLegacy.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerLegacy.java @@ -22,17 +22,18 @@ import com.google.common.io.ByteSink; import com.google.common.io.ByteStreams; import com.google.common.io.Closer; -import com.google.common.io.Files; import com.google.common.io.OutputSupplier; import com.google.common.primitives.Ints; +import com.yahoo.memory.MemoryMappedFile; +import com.yahoo.memory.MemoryRegion; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.spatial.ImmutableRTree; import io.druid.collections.spatial.RTree; import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; import io.druid.common.guava.FileOutputSupplier; import io.druid.common.utils.SerializerUtils; -import io.druid.java.util.common.ByteBufferUtils; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.java.util.common.logger.Logger; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.data.BitmapSerdeFactory; @@ -51,7 +52,6 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.IntBuffer; -import java.nio.MappedByteBuffer; import java.util.List; public class StringDimensionMergerLegacy extends StringDimensionMergerV9 implements DimensionMergerLegacy @@ -105,20 +105,28 @@ public void writeIndexes(List segmentRowNumConversions, Closer closer ); bitmapWriter.open(); - final MappedByteBuffer dimValsMapped = Files.map(dictionaryFile); + final MemoryMappedFile dimValsMapped; + try { + dimValsMapped = new MemoryMappedFile(dictionaryFile,0, dictionaryFile.length()); + } catch (Exception e) { + throw new IOException(); + } closer.register(new Closeable() { @Override public void close() throws IOException { - ByteBufferUtils.unmap(dimValsMapped); + dimValsMapped.freeMemory(); } }); - if (!dimensionName.equals(serializerUtils.readString(dimValsMapped))) { + PositionalMemoryRegion pMemory = new PositionalMemoryRegion( + new MemoryRegion(dimValsMapped, 0, dictionaryFile.length())); + String dimName = serializerUtils.readString(pMemory, Integer.reverseBytes(pMemory.getInt())); + if (!dimensionName.equals(dimName)) { throw new ISE("dimensions[%s] didn't equate!? This is a major WTF moment.", dimensionName); } - Indexed dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY); + Indexed dimVals = GenericIndexed.read(pMemory, GenericIndexed.STRING_STRATEGY); log.info("Starting dimension[%s] with cardinality[%,d]", dimensionName, dimVals.size()); diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index eee670ac65d7..0dc3459e5ca2 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -33,6 +33,7 @@ import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; import io.druid.java.util.common.ByteBufferUtils; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.java.util.common.logger.Logger; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; @@ -302,7 +303,7 @@ public void close() ByteBufferUtils.unmap(dimValsMapped); } }) { - Indexed dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY); + Indexed dimVals = GenericIndexed.read(new PositionalMemoryRegion(dimValsMapped), GenericIndexed.STRING_STRATEGY); BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory(); RTree tree = null; diff --git a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java index 7119012abb83..431c32df3356 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java @@ -23,13 +23,15 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.serde.ColumnPartSerde; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; import java.util.List; @@ -94,15 +96,16 @@ public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOE } } - public Column read(ByteBuffer buffer, ColumnConfig columnConfig, SmooshedFileMapper smooshedFiles) + public Column read(Memory memory, ColumnConfig columnConfig, SmooshedFileMapper smooshedFiles) { + PositionalMemoryRegion pMemoryRegion = new PositionalMemoryRegion(new MemoryRegion(memory, 0, memory.getCapacity())); final ColumnBuilder builder = new ColumnBuilder() .setType(valueType) .setHasMultipleValues(hasMultipleValues) .setFileMapper(smooshedFiles); for (ColumnPartSerde part : parts) { - part.getDeserializer().read(buffer, builder, columnConfig); + part.getDeserializer().read(pMemoryRegion, builder, columnConfig); } return builder.build(); diff --git a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedFloatSupplier.java b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedFloatSupplier.java index 08a0415a6423..5a1937d11e16 100644 --- a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedFloatSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedFloatSupplier.java @@ -21,9 +21,12 @@ import com.google.common.base.Supplier; import com.google.common.primitives.Floats; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; import io.druid.collections.ResourceHolder; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -36,23 +39,13 @@ public class BlockLayoutIndexedFloatSupplier implements Supplier private final int sizePer; public BlockLayoutIndexedFloatSupplier( - int totalSize, - int sizePer, - ByteBuffer fromBuffer, - ByteOrder order, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper mapper + int totalSize, int sizePer, Memory memory, ByteOrder order, + CompressedObjectStrategy.CompressionStrategy strategy ) { - baseFloatBuffers = GenericIndexed.read( - fromBuffer, - VSizeCompressedObjectStrategy.getBufferForOrder( - order, - strategy, - sizePer * Floats.BYTES - ), - mapper - ); + baseFloatBuffers = GenericIndexed.read(new PositionalMemoryRegion(new MemoryRegion(memory, 0, memory.getCapacity())), VSizeCompressedObjectStrategy.getBufferForOrder( + order, strategy, sizePer * Floats.BYTES + )); this.totalSize = totalSize; this.sizePer = sizePer; } diff --git a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedLongSupplier.java b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedLongSupplier.java index 729d75cc9a26..af6778c11f7e 100644 --- a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedLongSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedLongSupplier.java @@ -23,6 +23,7 @@ import io.druid.collections.ResourceHolder; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -37,24 +38,14 @@ public class BlockLayoutIndexedLongSupplier implements Supplier private final CompressionFactory.LongEncodingReader baseReader; public BlockLayoutIndexedLongSupplier( - int totalSize, - int sizePer, - ByteBuffer fromBuffer, - ByteOrder order, + int totalSize, int sizePer, PositionalMemoryRegion pMemory, ByteOrder order, CompressionFactory.LongEncodingReader reader, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper fileMapper + CompressedObjectStrategy.CompressionStrategy strategy ) { - baseLongBuffers = GenericIndexed.read( - fromBuffer, - VSizeCompressedObjectStrategy.getBufferForOrder( - order, - strategy, - reader.getNumBytes(sizePer) - ), - fileMapper - ); + baseLongBuffers = GenericIndexed.read(pMemory, VSizeCompressedObjectStrategy.getBufferForOrder( + order, strategy, reader.getNumBytes(sizePer) + )); this.totalSize = totalSize; this.sizePer = sizePer; this.baseReader = reader; diff --git a/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java index 839149f5423b..e12408daee94 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java @@ -21,8 +21,9 @@ import com.google.common.base.Supplier; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; import io.druid.java.util.common.IAE; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.io.IOException; import java.nio.ByteBuffer; @@ -36,21 +37,21 @@ public class CompressedFloatsIndexedSupplier implements Supplier private final int totalSize; private final int sizePer; - private final ByteBuffer buffer; + private final Memory memory; private final Supplier supplier; private final CompressedObjectStrategy.CompressionStrategy compression; CompressedFloatsIndexedSupplier( int totalSize, int sizePer, - ByteBuffer buffer, + Memory memory, Supplier supplier, CompressedObjectStrategy.CompressionStrategy compression ) { this.totalSize = totalSize; this.sizePer = sizePer; - this.buffer = buffer; + this.memory = memory; this.supplier = supplier; this.compression = compression; } @@ -68,7 +69,7 @@ public IndexedFloats get() public long getSerializedSize() { - return buffer.remaining() + 1 + 4 + 4 + 1; + return memory.getCapacity() + 1 + 4 + 4 + 1; } public void writeToChannel(WritableByteChannel channel) throws IOException @@ -77,37 +78,45 @@ public void writeToChannel(WritableByteChannel channel) throws IOException channel.write(ByteBuffer.wrap(Ints.toByteArray(totalSize))); channel.write(ByteBuffer.wrap(Ints.toByteArray(sizePer))); channel.write(ByteBuffer.wrap(new byte[]{compression.getId()})); - channel.write(buffer.asReadOnlyBuffer()); + long size = memory.getCapacity(); + long bytesOut = 0; + while(size > bytesOut){ + int bytes = (int) size ; + if(size > Integer.MAX_VALUE){ + bytes = Integer.MAX_VALUE; + } + byte[] byteArray = new byte[bytes]; + memory.getByteArray(bytesOut, byteArray, 0, byteArray.length); + ByteBuffer bb = ByteBuffer.allocate(bytes).put(byteArray); + bb.flip(); + channel.write(bb); + bytesOut += bytes; + } } - public static CompressedFloatsIndexedSupplier fromByteBuffer( - ByteBuffer buffer, - ByteOrder order, - SmooshedFileMapper mapper - ) + public static CompressedFloatsIndexedSupplier fromMemory(PositionalMemoryRegion pMemory, ByteOrder order) { - byte versionFromBuffer = buffer.get(); + byte versionFromBuffer = pMemory.getByte(); if (versionFromBuffer == LZF_VERSION || versionFromBuffer == version) { - final int totalSize = buffer.getInt(); - final int sizePer = buffer.getInt(); + final int totalSize = Integer.reverseBytes(pMemory.getInt()); + final int sizePer = Integer.reverseBytes(pMemory.getInt()); CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.LZF; if (versionFromBuffer == version) { - byte compressionId = buffer.get(); + byte compressionId = pMemory.getByte(); compression = CompressedObjectStrategy.CompressionStrategy.forId(compressionId); } Supplier supplier = CompressionFactory.getFloatSupplier( totalSize, sizePer, - buffer.asReadOnlyBuffer(), + pMemory.getRemainingMemory(), order, - compression, - mapper + compression ); return new CompressedFloatsIndexedSupplier( totalSize, sizePer, - buffer, + pMemory.getRemainingMemory(), supplier, compression ); diff --git a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java index 4e56126354b0..c000cb562519 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java @@ -26,7 +26,7 @@ import io.druid.collections.StupidResourceHolder; import io.druid.java.util.common.IAE; import io.druid.java.util.common.guava.CloseQuietly; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.CompressedPools; import it.unimi.dsi.fastutil.ints.IntIterator; @@ -130,11 +130,26 @@ GenericIndexed> getBaseIntBuffers() return baseIntBuffers; } - public static CompressedIntsIndexedSupplier fromByteBuffer( - ByteBuffer buffer, - ByteOrder order, - SmooshedFileMapper fileMapper - ) + public static CompressedIntsIndexedSupplier fromMemory(PositionalMemoryRegion pMemory, ByteOrder order) + { + byte versionFromBuffer = pMemory.getByte(); + + if (versionFromBuffer == VERSION) { + final int totalSize = Integer.reverseBytes(pMemory.getInt()); + final int sizePer = Integer.reverseBytes(pMemory.getInt()); + final CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.forId(pMemory.getByte()); + return new CompressedIntsIndexedSupplier( + totalSize, + sizePer, + GenericIndexed.read(pMemory, CompressedIntBufferObjectStrategy.getBufferForOrder(order, compression, sizePer)), + compression + ); + } + + throw new IAE("Unknown version[%s]", versionFromBuffer); + } + + public static CompressedIntsIndexedSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); @@ -147,11 +162,7 @@ public static CompressedIntsIndexedSupplier fromByteBuffer( return new CompressedIntsIndexedSupplier( totalSize, sizePer, - GenericIndexed.read( - buffer, - CompressedIntBufferObjectStrategy.getBufferForOrder(order, compression, sizePer), - fileMapper - ), + GenericIndexed.read(new PositionalMemoryRegion(buffer), CompressedIntBufferObjectStrategy.getBufferForOrder(order, compression, sizePer)), compression ); } diff --git a/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java index 1d701922fcbc..55253397ea32 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java @@ -21,8 +21,10 @@ import com.google.common.base.Supplier; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.io.IOException; import java.nio.ByteBuffer; @@ -39,7 +41,7 @@ public class CompressedLongsIndexedSupplier implements Supplier private final int totalSize; private final int sizePer; - private final ByteBuffer buffer; + private final Memory memory; private final Supplier supplier; private final CompressedObjectStrategy.CompressionStrategy compression; private final CompressionFactory.LongEncodingFormat encoding; @@ -47,7 +49,7 @@ public class CompressedLongsIndexedSupplier implements Supplier CompressedLongsIndexedSupplier( int totalSize, int sizePer, - ByteBuffer buffer, + Memory memory, Supplier supplier, CompressedObjectStrategy.CompressionStrategy compression, CompressionFactory.LongEncodingFormat encoding @@ -55,7 +57,7 @@ public class CompressedLongsIndexedSupplier implements Supplier { this.totalSize = totalSize; this.sizePer = sizePer; - this.buffer = buffer; + this.memory = memory; this.supplier = supplier; this.compression = compression; this.encoding = encoding; @@ -74,7 +76,7 @@ public IndexedLongs get() public long getSerializedSize() { - return buffer.remaining() + 1 + 4 + 4 + 1 + (encoding == CompressionFactory.LEGACY_LONG_ENCODING_FORMAT ? 0 : 1); + return memory.getCapacity() + 1 + 4 + 4 + 1 + (encoding == CompressionFactory.LEGACY_LONG_ENCODING_FORMAT ? 0 : 1); } public void writeToChannel(WritableByteChannel channel) throws IOException @@ -88,26 +90,35 @@ public void writeToChannel(WritableByteChannel channel) throws IOException channel.write(ByteBuffer.wrap(new byte[]{CompressionFactory.setEncodingFlag(compression.getId())})); channel.write(ByteBuffer.wrap(new byte[]{encoding.getId()})); } - channel.write(buffer.asReadOnlyBuffer()); + long size = memory.getCapacity(); + long bytesOut = 0; + while(size > bytesOut){ + int bytes = (int) size ; + if(size > Integer.MAX_VALUE){ + bytes = Integer.MAX_VALUE; + } + byte[] byteArray = new byte[bytes]; + memory.getByteArray(bytesOut, byteArray, 0, byteArray.length); + ByteBuffer bb = ByteBuffer.allocate(bytes).put(byteArray); + bb.flip(); + channel.write(bb); + bytesOut += bytes; + } } - public static CompressedLongsIndexedSupplier fromByteBuffer( - ByteBuffer buffer, - ByteOrder order, - SmooshedFileMapper fileMapper - ) + public static CompressedLongsIndexedSupplier fromMemory(PositionalMemoryRegion pMemory, ByteOrder order) { - byte versionFromBuffer = buffer.get(); + byte versionFromBuffer = pMemory.getByte(); if (versionFromBuffer == LZF_VERSION || versionFromBuffer == version) { - final int totalSize = buffer.getInt(); - final int sizePer = buffer.getInt(); + final int totalSize = Integer.reverseBytes(pMemory.getInt()); + final int sizePer = Integer.reverseBytes(pMemory.getInt()); CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.LZF; CompressionFactory.LongEncodingFormat encoding = CompressionFactory.LEGACY_LONG_ENCODING_FORMAT; if (versionFromBuffer == version) { - byte compressionId = buffer.get(); + byte compressionId = pMemory.getByte(); if (CompressionFactory.hasEncodingFlag(compressionId)) { - encoding = CompressionFactory.LongEncodingFormat.forId(buffer.get()); + encoding = CompressionFactory.LongEncodingFormat.forId(pMemory.getByte()); compressionId = CompressionFactory.clearEncodingFlag(compressionId); } compression = CompressedObjectStrategy.CompressionStrategy.forId(compressionId); @@ -115,16 +126,15 @@ public static CompressedLongsIndexedSupplier fromByteBuffer( Supplier supplier = CompressionFactory.getLongSupplier( totalSize, sizePer, - buffer.asReadOnlyBuffer(), + pMemory.duplicate(), order, encoding, - compression, - fileMapper + compression ); return new CompressedLongsIndexedSupplier( totalSize, sizePer, - buffer, + pMemory.getRemainingMemory(), supplier, compression, encoding @@ -133,4 +143,42 @@ public static CompressedLongsIndexedSupplier fromByteBuffer( throw new IAE("Unknown version[%s]", versionFromBuffer); } + +// public static CompressedLongsIndexedSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) +// { +// byte versionFromBuffer = buffer.get(); +// +// if (versionFromBuffer == LZF_VERSION || versionFromBuffer == version) { +// final int totalSize = buffer.getInt(); +// final int sizePer = buffer.getInt(); +// CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.LZF; +// CompressionFactory.LongEncodingFormat encoding = CompressionFactory.LEGACY_LONG_ENCODING_FORMAT; +// if (versionFromBuffer == version) { +// byte compressionId = buffer.get(); +// if (CompressionFactory.hasEncodingFlag(compressionId)) { +// encoding = CompressionFactory.LongEncodingFormat.forId(buffer.get()); +// compressionId = CompressionFactory.clearEncodingFlag(compressionId); +// } +// compression = CompressedObjectStrategy.CompressionStrategy.forId(compressionId); +// } +// Supplier supplier = CompressionFactory.getLongSupplier( +// totalSize, +// sizePer, +// buffer.asReadOnlyBuffer(), +// order, +// encoding, +// compression +// ); +// return new CompressedLongsIndexedSupplier( +// totalSize, +// sizePer, +// buffer, +// supplier, +// compression, +// encoding +// ); +// } +// +// throw new IAE("Unknown version[%s]", versionFromBuffer); +// } } diff --git a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java index 764d0e850dbb..c99c98fdd313 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java @@ -25,6 +25,7 @@ import com.ning.compress.BufferRecycler; import com.ning.compress.lzf.LZFDecoder; import com.ning.compress.lzf.LZFEncoder; +import com.yahoo.memory.Memory; import io.druid.collections.ResourceHolder; import io.druid.java.util.common.logger.Logger; import io.druid.segment.CompressedPools; @@ -328,14 +329,18 @@ public Class> getClazz() } @Override - public ResourceHolder fromByteBuffer(ByteBuffer buffer, int numBytes) + public ResourceHolder fromMemory(Memory memory) { + byte[] bytes = new byte[(int)memory.getCapacity()]; + memory.getByteArray(0, bytes, 0, (int)memory.getCapacity()); + ByteBuffer buffer = ByteBuffer.wrap(bytes); + final ResourceHolder bufHolder = CompressedPools.getByteBuf(order); final ByteBuffer buf = bufHolder.get(); buf.position(0); buf.limit(buf.capacity()); - decompress(buffer, numBytes, buf); + decompress(buffer, (int)memory.getCapacity(), buf); return new ResourceHolder() { @Override @@ -352,6 +357,31 @@ public void close() }; } +// @Override +// public ResourceHolder fromByteBuffer(ByteBuffer buffer, int numBytes) +// { +// final ResourceHolder bufHolder = CompressedPools.getByteBuf(order); +// final ByteBuffer buf = bufHolder.get(); +// buf.position(0); +// buf.limit(buf.capacity()); +// +// decompress(buffer, numBytes, buf); +// return new ResourceHolder() +// { +// @Override +// public T get() +// { +// return converter.convert(buf); +// } +// +// @Override +// public void close() +// { +// bufHolder.close(); +// } +// }; +// } + protected void decompress( ByteBuffer buffer, int numBytes, diff --git a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java index dd678c44c581..4c8f67e927c1 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java @@ -27,7 +27,7 @@ import io.druid.collections.StupidResourceHolder; import io.druid.java.util.common.IAE; import io.druid.java.util.common.guava.CloseQuietly; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.CompressedPools; import it.unimi.dsi.fastutil.ints.IntIterator; @@ -149,11 +149,37 @@ GenericIndexed> getBaseBuffers() return baseBuffers; } - public static CompressedVSizeIntsIndexedSupplier fromByteBuffer( - ByteBuffer buffer, - ByteOrder order, - SmooshedFileMapper fileMapper - ) + public static CompressedVSizeIntsIndexedSupplier fromMemory(PositionalMemoryRegion pMemory, ByteOrder order) + { + byte versionFromBuffer = pMemory.getByte(); + + if (versionFromBuffer == VERSION) { + final int numBytes = pMemory.getByte(); + final int totalSize = Integer.reverseBytes(pMemory.getInt()); + final int sizePer = Integer.reverseBytes(pMemory.getInt()); + final int chunkBytes = sizePer * numBytes + bufferPadding(numBytes); + + final CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.forId( + pMemory.getByte() + ); + + return new CompressedVSizeIntsIndexedSupplier( + totalSize, + sizePer, + numBytes, + GenericIndexed.read( + pMemory, + CompressedByteBufferObjectStrategy.getBufferForOrder(order, compression, chunkBytes) + ), + compression + ); + + } + + throw new IAE("Unknown version[%s]", versionFromBuffer); + } + + public static CompressedVSizeIntsIndexedSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); @@ -172,9 +198,8 @@ public static CompressedVSizeIntsIndexedSupplier fromByteBuffer( sizePer, numBytes, GenericIndexed.read( - buffer, - CompressedByteBufferObjectStrategy.getBufferForOrder(order, compression, chunkBytes), - fileMapper + new PositionalMemoryRegion(buffer), + CompressedByteBufferObjectStrategy.getBufferForOrder(order, compression, chunkBytes) ), compression ); diff --git a/processing/src/main/java/io/druid/segment/data/CompressionFactory.java b/processing/src/main/java/io/druid/segment/data/CompressionFactory.java index 09c4da117e22..ec29ff14e0b5 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressionFactory.java +++ b/processing/src/main/java/io/druid/segment/data/CompressionFactory.java @@ -23,8 +23,10 @@ import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Supplier; import com.google.common.collect.Maps; +import com.yahoo.memory.Memory; import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.io.IOException; import java.io.OutputStream; @@ -157,9 +159,9 @@ public enum LongEncodingFormat */ DELTA((byte) 0x0) { @Override - public LongEncodingReader getReader(ByteBuffer buffer, ByteOrder order) + public LongEncodingReader getReader(PositionalMemoryRegion memory, ByteOrder order) { - return new DeltaLongEncodingReader(buffer); + return new DeltaLongEncodingReader(memory); } }, /** @@ -169,9 +171,9 @@ public LongEncodingReader getReader(ByteBuffer buffer, ByteOrder order) */ TABLE((byte) 0x1) { @Override - public LongEncodingReader getReader(ByteBuffer buffer, ByteOrder order) + public LongEncodingReader getReader(PositionalMemoryRegion memory, ByteOrder order) { - return new TableLongEncodingReader(buffer); + return new TableLongEncodingReader(memory); } }, /** @@ -179,9 +181,9 @@ public LongEncodingReader getReader(ByteBuffer buffer, ByteOrder order) */ LONGS((byte) 0xFF) { @Override - public LongEncodingReader getReader(ByteBuffer buffer, ByteOrder order) + public LongEncodingReader getReader(PositionalMemoryRegion memory, ByteOrder order) { - return new LongsLongEncodingReader(buffer, order); + return new LongsLongEncodingReader(memory, order); } }; @@ -205,7 +207,7 @@ public byte getId() } } - public abstract LongEncodingReader getReader(ByteBuffer buffer, ByteOrder order); + public abstract LongEncodingReader getReader(PositionalMemoryRegion memory, ByteOrder order); public static LongEncodingFormat forId(byte id) { @@ -264,26 +266,16 @@ public interface LongEncodingReader } public static Supplier getLongSupplier( - int totalSize, - int sizePer, - ByteBuffer fromBuffer, - ByteOrder order, + int totalSize, int sizePer, PositionalMemoryRegion pMemory, ByteOrder order, LongEncodingFormat encodingFormat, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper fileMapper + CompressedObjectStrategy.CompressionStrategy strategy ) { if (strategy == CompressedObjectStrategy.CompressionStrategy.NONE) { - return new EntireLayoutIndexedLongSupplier(totalSize, encodingFormat.getReader(fromBuffer, order)); + return new EntireLayoutIndexedLongSupplier(totalSize, encodingFormat.getReader(pMemory, order)); } else { - return new BlockLayoutIndexedLongSupplier( - totalSize, - sizePer, - fromBuffer, - order, - encodingFormat.getReader(fromBuffer, order), - strategy, - fileMapper + return new BlockLayoutIndexedLongSupplier(totalSize, sizePer, pMemory, order, + encodingFormat.getReader(pMemory, order), strategy ); } } @@ -314,18 +306,14 @@ ioPeon, filenameBase, order, new LongsLongEncodingWriter(order), compressionStra // Float currently does not support any encoding types, and stores values as 4 byte float public static Supplier getFloatSupplier( - int totalSize, - int sizePer, - ByteBuffer fromBuffer, - ByteOrder order, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper fileMapper + int totalSize, int sizePer, Memory memory, ByteOrder order, + CompressedObjectStrategy.CompressionStrategy strategy ) { if (strategy == CompressedObjectStrategy.CompressionStrategy.NONE) { - return new EntireLayoutIndexedFloatSupplier(totalSize, fromBuffer, order); + return new EntireLayoutIndexedFloatSupplier(totalSize, memory, order); } else { - return new BlockLayoutIndexedFloatSupplier(totalSize, sizePer, fromBuffer, order, strategy, fileMapper); + return new BlockLayoutIndexedFloatSupplier(totalSize, sizePer, memory, order, strategy); } } diff --git a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java index 7cdcf6937e39..e761259a2ac4 100644 --- a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java @@ -19,10 +19,9 @@ package io.druid.segment.data; -import java.nio.ByteBuffer; - import com.google.common.collect.Ordering; +import com.yahoo.memory.Memory; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; @@ -78,11 +77,9 @@ public Class getClazz() } @Override - public WrappedImmutableConciseBitmap fromByteBuffer(ByteBuffer buffer, int numBytes) + public WrappedImmutableConciseBitmap fromMemory(Memory memory) { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); - return new WrappedImmutableConciseBitmap(new ImmutableConciseSet(readOnlyBuffer)); + return new WrappedImmutableConciseBitmap(new ImmutableConciseSet(memory)); } @Override diff --git a/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingReader.java b/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingReader.java index 5c21615c9141..0d82c6946748 100644 --- a/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingReader.java +++ b/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingReader.java @@ -20,43 +20,44 @@ package io.druid.segment.data; import io.druid.java.util.common.IAE; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.nio.ByteBuffer; public class DeltaLongEncodingReader implements CompressionFactory.LongEncodingReader { - private ByteBuffer buffer; + private PositionalMemoryRegion pMemory; private final long base; private final int bitsPerValue; private VSizeLongSerde.LongDeserializer deserializer; - public DeltaLongEncodingReader(ByteBuffer fromBuffer) + public DeltaLongEncodingReader(PositionalMemoryRegion fromMemory) { - this.buffer = fromBuffer.asReadOnlyBuffer(); - byte version = buffer.get(); + this.pMemory = fromMemory.duplicate(); + byte version = pMemory.getByte(); if (version == CompressionFactory.DELTA_ENCODING_VERSION) { - base = buffer.getLong(); - bitsPerValue = buffer.getInt(); - fromBuffer.position(buffer.position()); - deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, buffer, buffer.position()); + base = pMemory.getLong(); + bitsPerValue = Integer.reverseBytes(pMemory.getInt()); + fromMemory.position(pMemory.position()); + deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, pMemory.getRemainingMemory(), 0); } else { throw new IAE("Unknown version[%s]", version); } } - private DeltaLongEncodingReader(ByteBuffer buffer, long base, int bitsPerValue) + private DeltaLongEncodingReader(PositionalMemoryRegion memory, long base, int bitsPerValue) { - this.buffer = buffer; + this.pMemory = memory; this.base = base; this.bitsPerValue = bitsPerValue; - deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, buffer, buffer.position()); + deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, pMemory.getRemainingMemory(), 0); } @Override public void setBuffer(ByteBuffer buffer) { - deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, buffer, buffer.position()); + deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, (new PositionalMemoryRegion(buffer)).getRemainingMemory(), 0); } @Override @@ -74,6 +75,6 @@ public int getNumBytes(int values) @Override public CompressionFactory.LongEncodingReader duplicate() { - return new DeltaLongEncodingReader(buffer.duplicate(), base, bitsPerValue); + return new DeltaLongEncodingReader(pMemory.duplicate(), base, bitsPerValue); } } diff --git a/processing/src/main/java/io/druid/segment/data/EntireLayoutIndexedFloatSupplier.java b/processing/src/main/java/io/druid/segment/data/EntireLayoutIndexedFloatSupplier.java index 14b806fee2a8..63f2d7b78a53 100644 --- a/processing/src/main/java/io/druid/segment/data/EntireLayoutIndexedFloatSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/EntireLayoutIndexedFloatSupplier.java @@ -20,20 +20,23 @@ package io.druid.segment.data; import com.google.common.base.Supplier; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; -import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.FloatBuffer; public class EntireLayoutIndexedFloatSupplier implements Supplier { private final int totalSize; - private FloatBuffer buffer; + private PositionalMemoryRegion pMemory; + private ByteOrder order; - public EntireLayoutIndexedFloatSupplier(int totalSize, ByteBuffer fromBuffer, ByteOrder order) + public EntireLayoutIndexedFloatSupplier(int totalSize, Memory memory, ByteOrder order) { this.totalSize = totalSize; - this.buffer = fromBuffer.asReadOnlyBuffer().order(order).asFloatBuffer(); + this.pMemory = new PositionalMemoryRegion(new MemoryRegion(memory, 0, memory.getCapacity())); + this.order = order; } @Override @@ -54,7 +57,12 @@ public int size() @Override public float get(int index) { - return buffer.get(buffer.position() + index); + if(order == ByteOrder.BIG_ENDIAN) { + return Float.intBitsToFloat(Integer.reverseBytes( + Float.floatToRawIntBits(pMemory.getFloat(pMemory.position() + index * Float.BYTES)))); + } else { + return pMemory.getFloat(pMemory.position() + index * Float.BYTES); + } } @Override diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 6e278ea79b1c..09d1dc772ca5 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -22,10 +22,14 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; +import com.yahoo.memory.NativeMemory; import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.IAE; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import it.unimi.dsi.fastutil.bytes.ByteArrays; @@ -85,10 +89,9 @@ public Class getClazz() return String.class; } - @Override - public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) + @Override public String fromMemory(Memory memory) { - return StringUtils.fromUtf8(buffer, numBytes); + return StringUtils.fromUtf8(memory); } @Override @@ -111,32 +114,30 @@ public int compare(String o1, String o2) private final boolean allowReverseLookup; private final int size; private final BufferIndexed bufferIndexed; - private final List valueBuffers; - private final ByteBuffer headerBuffer; + private final List valueBuffers; + private final Memory headerBuffer; private int logBaseTwoOfElementsPerValueFile; - private ByteBuffer theBuffer; + private Memory memory; // used for single file version, v1 GenericIndexed( - ByteBuffer buffer, + Memory memory, ObjectStrategy strategy, boolean allowReverseLookup ) { - this.theBuffer = buffer; + this.memory = memory; this.strategy = strategy; this.allowReverseLookup = allowReverseLookup; - size = theBuffer.getInt(); + size = Integer.reverseBytes(memory.getInt(0)); - int indexOffset = theBuffer.position(); - int valuesOffset = theBuffer.position() + size * Ints.BYTES; + int indexOffset = Ints.BYTES; + int valuesOffset = Ints.BYTES + size * Ints.BYTES; - buffer.position(valuesOffset); - valueBuffers = Lists.newArrayList(buffer.slice()); - buffer.position(indexOffset); - headerBuffer = buffer.slice(); - final ByteBuffer valueBuffer = valueBuffers.get(0); + valueBuffers = Lists.newArrayList(); + valueBuffers.add(new MemoryRegion(memory, valuesOffset, memory.getCapacity() - valuesOffset)); + headerBuffer = new MemoryRegion(memory, indexOffset, valuesOffset - indexOffset); bufferIndexed = new BufferIndexed() { @Override @@ -149,21 +150,21 @@ public T get(int index) if (index == 0) { startOffset = 4; - endOffset = headerBuffer.getInt(0); + endOffset = Integer.reverseBytes(headerBuffer.getInt(0)); } else { int headerPosition = (index - 1) * Ints.BYTES; - startOffset = headerBuffer.getInt(headerPosition) + Ints.BYTES; - endOffset = headerBuffer.getInt(headerPosition + Ints.BYTES); + startOffset = Integer.reverseBytes(headerBuffer.getInt(headerPosition)) + Ints.BYTES; + endOffset = Integer.reverseBytes(headerBuffer.getInt(headerPosition + Ints.BYTES)); } - return _get(valueBuffer.asReadOnlyBuffer(), startOffset, endOffset); + return _get(new MemoryRegion(valueBuffers.get(0), startOffset, endOffset - startOffset)); } }; } // used for multiple file version, v2. GenericIndexed( - List valueBuffs, - ByteBuffer headerBuff, + List valueBuffs, + Memory headerBuff, ObjectStrategy strategy, boolean allowReverseLookup, int logBaseTwoOfElementsPerValueFile, @@ -176,14 +177,13 @@ public T get(int index) this.headerBuffer = headerBuff; this.size = numWritten; this.logBaseTwoOfElementsPerValueFile = logBaseTwoOfElementsPerValueFile; - headerBuffer.order(ByteOrder.nativeOrder()); bufferIndexed = new BufferIndexed() { @Override public T get(int index) { int fileNum = index >> GenericIndexed.this.logBaseTwoOfElementsPerValueFile; - final ByteBuffer copyBuffer = valueBuffers.get(fileNum).asReadOnlyBuffer(); + final Memory copyBuffer = valueBuffers.get(fileNum); checkIndex(index, size); @@ -199,7 +199,7 @@ public T get(int index) startOffset = headerBuffer.getInt(headerPosition) + 4; endOffset = headerBuffer.getInt(headerPosition + 4); } - return _get(copyBuffer, startOffset, endOffset); + return _get(new MemoryRegion(copyBuffer, startOffset, endOffset - startOffset)); } }; } @@ -241,9 +241,7 @@ public static GenericIndexed fromIterable(Iterable objectsIterable, Ob { Iterator objects = objectsIterable.iterator(); if (!objects.hasNext()) { - final ByteBuffer buffer = ByteBuffer.allocate(4).putInt(0); - buffer.flip(); - return new GenericIndexed(buffer, strategy, true); + return new GenericIndexed(new NativeMemory(new byte[]{0,0,0,0}), strategy, true); } boolean allowReverseLookup = true; @@ -287,42 +285,43 @@ public static GenericIndexed fromIterable(Iterable objectsIterable, Ob theBuffer.put(valueBytes.toByteArray()); theBuffer.flip(); - return new GenericIndexed(theBuffer.asReadOnlyBuffer(), strategy, allowReverseLookup); + return new GenericIndexed(new NativeMemory(theBuffer), strategy, allowReverseLookup); } - public static GenericIndexed read(ByteBuffer buffer, ObjectStrategy strategy) + public static GenericIndexed read(PositionalMemoryRegion pMemory, ObjectStrategy strategy) { - byte versionFromBuffer = buffer.get(); + byte versionFromBuffer = pMemory.getByte(); if (VERSION_ONE == versionFromBuffer) { - return createVersionOneGenericIndexed(buffer, strategy); + return createVersionOneGenericIndexed(pMemory, strategy); } else if (VERSION_TWO == versionFromBuffer) { throw new IAE( "use read(ByteBuffer buffer, ObjectStrategy strategy, SmooshedFileMapper fileMapper)" - + " to read version 2 indexed.", + + " to read version 2 indexed.", versionFromBuffer ); } throw new IAE("Unknown version[%s]", versionFromBuffer); } - private static GenericIndexed createVersionOneGenericIndexed(ByteBuffer byteBuffer, ObjectStrategy strategy) + private static GenericIndexed createVersionOneGenericIndexed(PositionalMemoryRegion pMemory, + ObjectStrategy strategy) { - boolean allowReverseLookup = byteBuffer.get() == REVERSE_LOOKUP_ALLOWED; - int size = byteBuffer.getInt(); - ByteBuffer bufferToUse = byteBuffer.asReadOnlyBuffer(); - bufferToUse.limit(bufferToUse.position() + size); - byteBuffer.position(bufferToUse.limit()); + boolean allowReverseLookup = pMemory.getByte() == REVERSE_LOOKUP_ALLOWED; + int size = Integer.reverseBytes(pMemory.getInt()); + PositionalMemoryRegion memoryToUse = pMemory.duplicate(); + memoryToUse.limit(memoryToUse.position() + size); + pMemory.position(memoryToUse.limit()); return new GenericIndexed( - bufferToUse, + memoryToUse.getRemainingMemory(), strategy, allowReverseLookup ); } private static GenericIndexed createVersionTwoGenericIndexed( - ByteBuffer byteBuffer, + PositionalMemoryRegion pMemory, ObjectStrategy strategy, SmooshedFileMapper fileMapper ) @@ -330,25 +329,24 @@ private static GenericIndexed createVersionTwoGenericIndexed( if (fileMapper == null) { throw new IAE("SmooshedFileMapper can not be null for version 2."); } - boolean allowReverseLookup = byteBuffer.get() == REVERSE_LOOKUP_ALLOWED; - int logBaseTwoOfElementsPerValueFile = byteBuffer.getInt(); - int numElements = byteBuffer.getInt(); + boolean allowReverseLookup = pMemory.getByte() == REVERSE_LOOKUP_ALLOWED; + int logBaseTwoOfElementsPerValueFile = pMemory.getInt(); + int numElements = pMemory.getInt(); String columnName; - List valueBuffersToUse; - ByteBuffer headerBuffer; + List valueBuffersToUse; + Memory headerBuffer; try { - columnName = SERIALIZER_UTILS.readString(byteBuffer); + columnName = SERIALIZER_UTILS.readString(pMemory); valueBuffersToUse = Lists.newArrayList(); int elementsPerValueFile = 1 << logBaseTwoOfElementsPerValueFile; int numberOfFilesRequired = getNumberOfFilesRequired(elementsPerValueFile, numElements); for (int i = 0; i < numberOfFilesRequired; i++) { valueBuffersToUse.add( - fileMapper.mapFile(GenericIndexedWriter.generateValueFileName(columnName, i)) - .asReadOnlyBuffer() + fileMapper.mapFileToMemory(GenericIndexedWriter.generateValueFileName(columnName, i)) ); } - headerBuffer = fileMapper.mapFile(GenericIndexedWriter.generateHeaderFileName(columnName)); + headerBuffer = fileMapper.mapFileToMemory(GenericIndexedWriter.generateHeaderFileName(columnName)); } catch (IOException e) { throw new RuntimeException("File mapping failed.", e); @@ -364,14 +362,15 @@ private static GenericIndexed createVersionTwoGenericIndexed( ); } - public static GenericIndexed read(ByteBuffer buffer, ObjectStrategy strategy, SmooshedFileMapper fileMapper) + public static GenericIndexed read(PositionalMemoryRegion pMemory, + ObjectStrategy strategy, SmooshedFileMapper fileMapper) { - byte versionFromBuffer = buffer.get(); + byte versionFromBuffer = pMemory.getByte(); if (VERSION_ONE == versionFromBuffer) { - return createVersionOneGenericIndexed(buffer, strategy); + return createVersionOneGenericIndexed(pMemory, strategy); } else if (VERSION_TWO == versionFromBuffer) { - return createVersionTwoGenericIndexed(buffer, strategy, fileMapper); + return createVersionTwoGenericIndexed(pMemory, strategy, fileMapper); } throw new IAE("Unknown version [%s]", versionFromBuffer); @@ -421,11 +420,9 @@ public long getSerializedSize() if (valueBuffers.size() != 1) { throw new UnsupportedOperationException("Method not supported for version 2 GenericIndexed."); } - return theBuffer.remaining() - + 2 - + Ints.BYTES - + Ints.BYTES; //2 Bytes for version and sorted flag. 4 bytes to store numbers - // of bytes and next 4 bytes to store number of elements. + return memory.getCapacity() + + 2 + + Ints.BYTES; //4 bytes to store numbers of bytes } public void writeToChannel(WritableByteChannel channel) throws IOException @@ -433,9 +430,21 @@ public void writeToChannel(WritableByteChannel channel) throws IOException //version 2 will always have more than one buffer in valueBuffers. if (valueBuffers.size() == 1) { channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE, allowReverseLookup ? (byte) 0x1 : (byte) 0x0})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(theBuffer.remaining() + 4))); // 4 Bytes to store size. - channel.write(ByteBuffer.wrap(Ints.toByteArray(size))); - channel.write(theBuffer.asReadOnlyBuffer()); + channel.write(ByteBuffer.wrap(Ints.toByteArray((int)memory.getCapacity()))); // 4 Bytes to store size. + + long size = memory.getCapacity(); + while(size > 0){ + int bytes = (int) size; + if(size > Integer.MAX_VALUE){ + bytes = Integer.MAX_VALUE; + } + byte[] byteArray = new byte[bytes]; + memory.getByteArray(0, byteArray, 0, byteArray.length); + ByteBuffer bb = ByteBuffer.allocate(bytes).put(byteArray); + bb.flip(); + channel.write(bb); + size -= bytes; + } } else { throw new UnsupportedOperationException( "GenericIndexed serialization for V2 is unsupported. Use GenericIndexedWriter instead."); @@ -450,7 +459,7 @@ public void writeToChannel(WritableByteChannel channel) throws IOException public GenericIndexed.BufferIndexed singleThreaded() { if (valueBuffers.size() == 1) { - final ByteBuffer copyBuffer = valueBuffers.get(0).asReadOnlyBuffer(); + final Memory copyBuffer = valueBuffers.get(0); return new BufferIndexed() { @Override @@ -463,29 +472,23 @@ public T get(final int index) if (index == 0) { startOffset = 4; - endOffset = headerBuffer.getInt(0); + endOffset = Integer.reverseBytes(headerBuffer.getInt(0)); } else { int headerPosition = (index - 1) * Ints.BYTES; - startOffset = headerBuffer.getInt(headerPosition) + 4; - endOffset = headerBuffer.getInt(headerPosition + 4); + startOffset = Integer.reverseBytes(headerBuffer.getInt(headerPosition)) + 4; + endOffset = Integer.reverseBytes(headerBuffer.getInt(headerPosition + 4)); } - return _get(copyBuffer, startOffset, endOffset); + return _get(new MemoryRegion(copyBuffer, startOffset, endOffset - startOffset)); } }; } else { - - final List copyValueBuffers = new ArrayList<>(); - for (ByteBuffer buffer : valueBuffers) { - copyValueBuffers.add(buffer.asReadOnlyBuffer()); - } - return new BufferIndexed() { @Override public T get(final int index) { int fileNum = index >> logBaseTwoOfElementsPerValueFile; - final ByteBuffer copyBuffer = copyValueBuffers.get(fileNum); + final Memory copyBuffer = valueBuffers.get(fileNum); checkIndex(index, size); final int startOffset; @@ -502,7 +505,7 @@ public T get(final int index) endOffset = headerBuffer.getInt(headerPosition + 4); } - return _get(copyBuffer, startOffset, endOffset); + return _get(new MemoryRegion(memory, startOffset, endOffset - startOffset)); } }; } @@ -525,16 +528,15 @@ public int size() return size; } - protected T _get(ByteBuffer copyValueBuffer, int startOffset, int endOffset) + protected T _get(Memory memory) { - final int size = endOffset - startOffset; - if (startOffset == endOffset) { + final int size = (int)memory.getCapacity(); + if (size == 0) { return null; } - copyValueBuffer.position(startOffset); lastReadSize = size; // fromByteBuffer must not modify the buffer limit - return strategy.fromByteBuffer(copyValueBuffer, size); + return strategy.fromMemory(memory); } /** diff --git a/processing/src/main/java/io/druid/segment/data/IndexedRTree.java b/processing/src/main/java/io/druid/segment/data/IndexedRTree.java index ba1ea9123aa0..0cff3f43176a 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedRTree.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedRTree.java @@ -20,11 +20,10 @@ package io.druid.segment.data; import com.google.common.collect.Ordering; +import com.yahoo.memory.Memory; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.spatial.ImmutableRTree; -import java.nio.ByteBuffer; - /** */ public class IndexedRTree implements Comparable @@ -83,12 +82,9 @@ public Class getClazz() return ImmutableRTree.class; } - @Override - public ImmutableRTree fromByteBuffer(ByteBuffer buffer, int numBytes) + @Override public ImmutableRTree fromMemory(Memory memory) { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); - return new ImmutableRTree(readOnlyBuffer, bitmapFactory); + return new ImmutableRTree(memory, bitmapFactory); } @Override diff --git a/processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java b/processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java index e8712bcd3ca6..a98c6bbf1e3f 100644 --- a/processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java @@ -21,12 +21,14 @@ import com.google.common.collect.Ordering; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; +import com.yahoo.memory.NativeMemory; import io.druid.collections.IntList; import it.unimi.dsi.fastutil.ints.IntIterator; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.IntBuffer; +import java.nio.ByteOrder; /** */ @@ -37,52 +39,62 @@ public class IntBufferIndexedInts implements IndexedInts, Comparable getClazz() } @Override - public IntBufferIndexedInts fromByteBuffer(ByteBuffer buffer, int numBytes) + public IntBufferIndexedInts fromMemory(Memory memory) { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); - return new IntBufferIndexedInts(readOnlyBuffer); + return new IntBufferIndexedInts(memory); } @Override public byte[] toBytes(IntBufferIndexedInts val) { - ByteBuffer buffer = val.getBuffer(); - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes); + Memory memory = val.getBuffer(); + byte[] bytes = new byte[(int)memory.getCapacity()]; + memory.getByteArray(0, bytes, 0, bytes.length); return bytes; } diff --git a/processing/src/main/java/io/druid/segment/data/LongsLongEncodingReader.java b/processing/src/main/java/io/druid/segment/data/LongsLongEncodingReader.java index 9cbb4f96a70b..45227f26e507 100644 --- a/processing/src/main/java/io/druid/segment/data/LongsLongEncodingReader.java +++ b/processing/src/main/java/io/druid/segment/data/LongsLongEncodingReader.java @@ -20,35 +20,41 @@ package io.druid.segment.data; import com.google.common.primitives.Longs; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.LongBuffer; public class LongsLongEncodingReader implements CompressionFactory.LongEncodingReader { - private LongBuffer buffer; + private PositionalMemoryRegion pMemory; + private ByteOrder order; - public LongsLongEncodingReader(ByteBuffer fromBuffer, ByteOrder order) + public LongsLongEncodingReader(PositionalMemoryRegion pMemory, ByteOrder order) { - this.buffer = fromBuffer.asReadOnlyBuffer().order(order).asLongBuffer(); + this.pMemory = pMemory.duplicate(); + this.order = order; } - private LongsLongEncodingReader(LongBuffer buffer) + private LongsLongEncodingReader(PositionalMemoryRegion memory) { - this.buffer = buffer; + this.pMemory = memory; } @Override public void setBuffer(ByteBuffer buffer) { - this.buffer = buffer.asLongBuffer(); + this.pMemory = new PositionalMemoryRegion(buffer); } @Override public long read(int index) { - return buffer.get(buffer.position() + index); + if(order == ByteOrder.BIG_ENDIAN) { + return Long.reverseBytes(pMemory.getLong(pMemory.position() + index * Long.BYTES)); + } else { + return pMemory.getLong(pMemory.position() + index * Long.BYTES); + } } @Override @@ -60,6 +66,6 @@ public int getNumBytes(int values) @Override public CompressionFactory.LongEncodingReader duplicate() { - return new LongsLongEncodingReader(buffer.duplicate()); + return new LongsLongEncodingReader(pMemory.duplicate()); } } diff --git a/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java b/processing/src/main/java/io/druid/segment/data/MemorySerializer.java similarity index 71% rename from processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java rename to processing/src/main/java/io/druid/segment/data/MemorySerializer.java index e0035aeaecdf..e459100b49a1 100644 --- a/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/MemorySerializer.java @@ -20,6 +20,8 @@ package io.druid.segment.data; import com.google.common.primitives.Ints; +import com.yahoo.memory.MemoryRegion; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.io.IOException; import java.nio.ByteBuffer; @@ -27,16 +29,16 @@ /** */ -public class ByteBufferSerializer +public class MemorySerializer { - public static T read(ByteBuffer buffer, ObjectStrategy strategy) + public static T read(PositionalMemoryRegion pMemory, ObjectStrategy strategy) { - int size = buffer.getInt(); - ByteBuffer bufferToUse = buffer.asReadOnlyBuffer(); - bufferToUse.limit(bufferToUse.position() + size); - buffer.position(bufferToUse.limit()); + int size = Integer.reverseBytes(pMemory.getInt()); + PositionalMemoryRegion memoryToUse = pMemory.duplicate(); + memoryToUse.limit(memoryToUse.position() + size); + pMemory.position(memoryToUse.limit()); - return strategy.fromByteBuffer(bufferToUse, size); + return strategy.fromMemory(new MemoryRegion(pMemory.getRemainingMemory(), 0, size)); } public static void writeToChannel(T obj, ObjectStrategy strategy, WritableByteChannel channel) diff --git a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java index 5778a82e89e7..b6120cae9ee5 100644 --- a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java @@ -19,23 +19,26 @@ package io.druid.segment.data; -import java.nio.ByteBuffer; +import com.yahoo.memory.Memory; + import java.util.Comparator; public interface ObjectStrategy extends Comparator { public Class getClazz(); - /** - * Convert values from their underlying byte representation. - * - * Implementations of this method must not change the given buffer mark, or limit, but may modify its position. - * Use buffer.asReadOnlyBuffer() or buffer.duplicate() if mark or limit need to be set. - * - * @param buffer buffer to read value from - * @param numBytes number of bytes used to store the value, starting at buffer.position() - * @return an object created from the given byte buffer representation - */ - public T fromByteBuffer(ByteBuffer buffer, int numBytes); +// /** +// * Convert values from their underlying byte representation. +// * +// * Implementations of this method must not change the given buffer mark, or limit, but may modify its position. +// * Use buffer.asReadOnlyBuffer() or buffer.duplicate() if mark or limit need to be set. +// * +// * @param buffer buffer to read value from +// * @param numBytes number of bytes used to store the value, starting at buffer.position() +// * @return an object created from the given byte buffer representation +// */ + + public T fromMemory(Memory memory); + public byte[] toBytes(T val); } diff --git a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java index 9c2ae1d21a85..2ee373c2e94d 100644 --- a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Ordering; +import com.yahoo.memory.Memory; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.RoaringBitmapFactory; @@ -99,12 +100,21 @@ public Class getClazz() return ImmutableBitmap.class; } - @Override - public ImmutableBitmap fromByteBuffer(ByteBuffer buffer, int numBytes) +// @Override +// public ImmutableBitmap fromByteBuffer(ByteBuffer buffer, int numBytes) +// { +// final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); +// readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); +// return new WrappedImmutableRoaringBitmap(new ImmutableRoaringBitmap(readOnlyBuffer)); +// } + + //TODO See how we can Copy/Paste Roaring bitmap to Druid + @Override public ImmutableBitmap fromMemory(Memory memory) { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); - return new WrappedImmutableRoaringBitmap(new ImmutableRoaringBitmap(readOnlyBuffer)); + byte[] bytes = new byte[(int)memory.getCapacity()]; + memory.getByteArray(0, bytes, 0, (int)memory.getCapacity()); + + return new WrappedImmutableRoaringBitmap(new ImmutableRoaringBitmap(ByteBuffer.wrap(bytes))); } @Override diff --git a/processing/src/main/java/io/druid/segment/data/TableLongEncodingReader.java b/processing/src/main/java/io/druid/segment/data/TableLongEncodingReader.java index db79c23229b3..536c4e73edaf 100644 --- a/processing/src/main/java/io/druid/segment/data/TableLongEncodingReader.java +++ b/processing/src/main/java/io/druid/segment/data/TableLongEncodingReader.java @@ -20,6 +20,7 @@ package io.druid.segment.data; import io.druid.java.util.common.IAE; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.nio.ByteBuffer; @@ -27,42 +28,43 @@ public class TableLongEncodingReader implements CompressionFactory.LongEncodingR { private final long table[]; private final int bitsPerValue; - private final ByteBuffer buffer; + private final PositionalMemoryRegion memory; private VSizeLongSerde.LongDeserializer deserializer; - public TableLongEncodingReader(ByteBuffer fromBuffer) + public TableLongEncodingReader(PositionalMemoryRegion fromMemory) { - this.buffer = fromBuffer.asReadOnlyBuffer(); - byte version = buffer.get(); + + this.memory = fromMemory.duplicate(); + byte version = memory.getByte(); if (version == CompressionFactory.TABLE_ENCODING_VERSION) { - int tableSize = buffer.getInt(); + int tableSize = Integer.reverseBytes(memory.getInt()); if (tableSize < 0 || tableSize > CompressionFactory.MAX_TABLE_SIZE) { throw new IAE("Invalid table size[%s]", tableSize); } bitsPerValue = VSizeLongSerde.getBitsForMax(tableSize); table = new long[tableSize]; for (int i = 0; i < tableSize; i++) { - table[i] = buffer.getLong(); + table[i] = Long.reverseBytes(memory.getLong()); } - fromBuffer.position(buffer.position()); - deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, buffer, buffer.position()); + fromMemory.position(memory.position()); + deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, memory.getRemainingMemory(), 0); } else { throw new IAE("Unknown version[%s]", version); } } - private TableLongEncodingReader(ByteBuffer buffer, long table[], int bitsPerValue) + private TableLongEncodingReader(PositionalMemoryRegion memory, long table[], int bitsPerValue) { - this.buffer = buffer; + this.memory = memory; this.table = table; this.bitsPerValue = bitsPerValue; - deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, buffer, buffer.position()); + deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, memory.getRemainingMemory(), 0); } @Override public void setBuffer(ByteBuffer buffer) { - deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, buffer, buffer.position()); + deserializer = VSizeLongSerde.getDeserializer(bitsPerValue, (new PositionalMemoryRegion(buffer)).getRemainingMemory(), 0); } @Override @@ -80,6 +82,7 @@ public int getNumBytes(int values) @Override public CompressionFactory.LongEncodingReader duplicate() { - return new TableLongEncodingReader(buffer.duplicate(), table, bitsPerValue); + return new TableLongEncodingReader( + new PositionalMemoryRegion(memory, memory.position(), memory.remaining()), table, bitsPerValue); } } diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java index 4696cc97befb..816476fb17e8 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java @@ -20,8 +20,12 @@ package io.druid.segment.data; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; +import com.yahoo.memory.NativeMemory; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -41,7 +45,7 @@ public static VSizeIndexed fromIterable(Iterable objectsIterab if (!objects.hasNext()) { final ByteBuffer buffer = ByteBuffer.allocate(4).putInt(0); buffer.flip(); - return new VSizeIndexed(buffer, 4); + return new VSizeIndexed(new NativeMemory(buffer), 4); } int numBytes = -1; @@ -82,10 +86,10 @@ public static VSizeIndexed fromIterable(Iterable objectsIterab theBuffer.put(valueBytes.toByteArray()); theBuffer.flip(); - return new VSizeIndexed(theBuffer.asReadOnlyBuffer(), numBytes); + return new VSizeIndexed(new NativeMemory(theBuffer), numBytes); } - private final ByteBuffer theBuffer; + private final Memory memory; private final int numBytes; private final int size; @@ -93,15 +97,15 @@ public static VSizeIndexed fromIterable(Iterable objectsIterab private final int bufferBytes; VSizeIndexed( - ByteBuffer buffer, + Memory memory, int numBytes ) { - this.theBuffer = buffer; + this.memory = memory; this.numBytes = numBytes; - size = theBuffer.getInt(); - valuesOffset = theBuffer.position() + (size << 2); + size = Integer.reverseBytes(this.memory.getInt(0)); + valuesOffset = Integer.BYTES + (size << 2); bufferBytes = 4 - numBytes; } @@ -124,21 +128,20 @@ public VSizeIndexedInts get(int index) throw new IllegalArgumentException(String.format("Index[%s] >= size[%s]", index, size)); } - ByteBuffer myBuffer = theBuffer.asReadOnlyBuffer(); +// ByteBuffer myBuffer = theBuffer.asReadOnlyBuffer(); int startOffset = 0; int endOffset; if (index == 0) { - endOffset = myBuffer.getInt(); + endOffset = Integer.reverseBytes(memory.getInt(Integer.BYTES)); } else { - myBuffer.position(myBuffer.position() + ((index - 1) * Ints.BYTES)); - startOffset = myBuffer.getInt(); - endOffset = myBuffer.getInt(); + startOffset = Integer.reverseBytes(memory.getInt((index ) * Ints.BYTES)); + endOffset = Integer.reverseBytes(memory.getInt((index + 1) * Ints.BYTES)); } - myBuffer.position(valuesOffset + startOffset); - myBuffer.limit(myBuffer.position() + (endOffset - startOffset) + bufferBytes); - return myBuffer.hasRemaining() ? new VSizeIndexedInts(myBuffer, numBytes) : null; + int position = valuesOffset + startOffset; + int limit = position + (endOffset - startOffset) + bufferBytes; + return memory.getCapacity() > position ? new VSizeIndexedInts(new MemoryRegion(memory, position, limit - position), numBytes) : null; } @Override @@ -149,29 +152,32 @@ public int indexOf(IndexedInts value) public int getSerializedSize() { - return theBuffer.remaining() + 4 + 4 + 2; + return (int)memory.getCapacity() + 4 + 2; } public void writeToChannel(WritableByteChannel channel) throws IOException { channel.write(ByteBuffer.wrap(new byte[]{version, (byte) numBytes})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(theBuffer.remaining() + 4))); + channel.write(ByteBuffer.wrap(Ints.toByteArray((int)memory.getCapacity()))); channel.write(ByteBuffer.wrap(Ints.toByteArray(size))); - channel.write(theBuffer.asReadOnlyBuffer()); + + byte[] bytes = new byte[(int)memory.getCapacity()-4]; + memory.getByteArray(4, bytes, 0, bytes.length); + channel.write(ByteBuffer.wrap(bytes)); } - public static VSizeIndexed readFromByteBuffer(ByteBuffer buffer) + public static VSizeIndexed readFromMemory(PositionalMemoryRegion fromMemory) { - byte versionFromBuffer = buffer.get(); + byte versionFromBuffer = fromMemory.getByte(); if (version == versionFromBuffer) { - int numBytes = buffer.get(); - int size = buffer.getInt(); - ByteBuffer bufferToUse = buffer.asReadOnlyBuffer(); - bufferToUse.limit(bufferToUse.position() + size); - buffer.position(bufferToUse.limit()); + int numBytes = fromMemory.getByte(); + int size = Integer.reverseBytes(fromMemory.getInt()); + PositionalMemoryRegion memoryToUse = fromMemory.duplicate(); + memoryToUse.limit(memoryToUse.position() + size); + fromMemory.position(memoryToUse.limit()); - return new VSizeIndexed(bufferToUse, numBytes); + return new VSizeIndexed(memoryToUse.getRemainingMemory(), numBytes); } throw new IAE("Unknown version[%s]", versionFromBuffer); diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java index f8f9c9fdcf19..f9f77d4d0d15 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java @@ -21,7 +21,10 @@ import com.google.common.collect.Lists; import com.google.common.primitives.Ints; +import com.yahoo.memory.Memory; +import com.yahoo.memory.NativeMemory; import io.druid.java.util.common.IAE; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import it.unimi.dsi.fastutil.ints.IntIterator; import java.io.IOException; @@ -70,7 +73,7 @@ public static VSizeIndexedInts fromList(List list, int maxValue) final ByteBuffer buffer = ByteBuffer.allocate((list.size() * numBytes) + (4 - numBytes)); writeToBuffer(buffer, list, numBytes, maxValue); - return new VSizeIndexedInts(buffer.asReadOnlyBuffer(), numBytes); + return new VSizeIndexedInts(new NativeMemory(buffer), numBytes); } private static void writeToBuffer(ByteBuffer buffer, List list, int numBytes, int maxValue) @@ -110,21 +113,21 @@ else if (maxValue <= 0xFFFFFF) { return numBytes; } - private final ByteBuffer buffer; + private final Memory memory; private final int numBytes; private final int bitsToShift; private final int size; - public VSizeIndexedInts(ByteBuffer buffer, int numBytes) + public VSizeIndexedInts(Memory memory, int numBytes) { - this.buffer = buffer; + this.memory = memory; this.numBytes = numBytes; bitsToShift = 32 - (numBytes << 3); // numBytes * 8 int numBufferBytes = 4 - numBytes; - size = (buffer.remaining() - numBufferBytes) / numBytes; + size = ((int)memory.getCapacity() - numBufferBytes) / numBytes; } @Override @@ -136,31 +139,33 @@ public int size() @Override public int get(int index) { - return buffer.getInt(buffer.position() + (index * numBytes)) >>> bitsToShift; + return Integer.reverseBytes(memory.getInt( index * numBytes)) >>> bitsToShift; } public byte[] getBytesNoPadding() { - int bytesToTake = buffer.remaining() - (4 - numBytes); + int bytesToTake = (int)memory.getCapacity() - (4 - numBytes); byte[] bytes = new byte[bytesToTake]; - buffer.asReadOnlyBuffer().get(bytes); + memory.getByteArray(0, bytes, 0, bytesToTake); return bytes; } public byte[] getBytes() { - byte[] bytes = new byte[buffer.remaining()]; - buffer.asReadOnlyBuffer().get(bytes); + byte[] bytes = new byte[(int)memory.getCapacity()]; + memory.getByteArray(0, bytes, 0, (int)memory.getCapacity()); return bytes; } @Override public int compareTo(VSizeIndexedInts o) { + //TODO Need to implement this int retVal = Ints.compare(numBytes, o.numBytes); if (retVal == 0) { - retVal = buffer.compareTo(o.buffer); +// retVal = buffer.compareTo(o.buffer); + } return retVal; @@ -174,7 +179,7 @@ public int getNumBytes() public long getSerializedSize() { // version, numBytes, size, remaining - return 1 + 1 + 4 + buffer.remaining(); + return 1 + 1 + 4 + memory.getCapacity(); } @Override @@ -186,23 +191,26 @@ public IntIterator iterator() public void writeToChannel(WritableByteChannel channel) throws IOException { channel.write(ByteBuffer.wrap(new byte[]{VERSION, (byte) numBytes})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(buffer.remaining()))); - channel.write(buffer.asReadOnlyBuffer()); + channel.write(ByteBuffer.wrap(Ints.toByteArray((int)memory.getCapacity()))); + + byte[] bytes = new byte[(int)memory.getCapacity()]; + memory.getByteArray(0, bytes, 0, bytes.length); + channel.write(ByteBuffer.wrap(bytes)); } - public static VSizeIndexedInts readFromByteBuffer(ByteBuffer buffer) + public static VSizeIndexedInts readFromMemory(PositionalMemoryRegion pMemory) { - byte versionFromBuffer = buffer.get(); + byte versionFromBuffer = pMemory.getByte(0); if (VERSION == versionFromBuffer) { - int numBytes = buffer.get(); - int size = buffer.getInt(); - ByteBuffer bufferToUse = buffer.asReadOnlyBuffer(); - bufferToUse.limit(bufferToUse.position() + size); - buffer.position(bufferToUse.limit()); + int numBytes = pMemory.getByte(); + int size = Integer.reverseBytes(pMemory.getInt()); + PositionalMemoryRegion memoryToUse = pMemory.duplicate(); + memoryToUse.limit(memoryToUse.position() + size); + pMemory.limit(memoryToUse.limit()); return new VSizeIndexedInts( - bufferToUse, + memoryToUse.getRemainingMemory(), numBytes ); } diff --git a/processing/src/main/java/io/druid/segment/data/VSizeLongSerde.java b/processing/src/main/java/io/druid/segment/data/VSizeLongSerde.java index 2ebcf666aab6..ea82ef838b77 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeLongSerde.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeLongSerde.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import com.yahoo.memory.Memory; import io.druid.java.util.common.IAE; import java.io.Closeable; @@ -144,37 +145,37 @@ public static LongSerializer getSerializer(int longSize, ByteBuffer buffer, int // LongDeserializers were adapted from Apache Lucene DirectReader, see: // https://github.com/apache/lucene-solr/blob/master/lucene/core/src/java/org/apache/lucene/util/packed/DirectReader.java - public static LongDeserializer getDeserializer(int longSize, ByteBuffer fromBuffer, int bufferOffset) + public static LongDeserializer getDeserializer(int longSize, Memory fromMemory, int bufferOffset) { // The buffer needs to be duplicated since the byte order is changed - ByteBuffer buffer = fromBuffer.duplicate().order(ByteOrder.BIG_ENDIAN); +// ByteBuffer buffer = fromBuffer.duplicate().order(ByteOrder.BIG_ENDIAN); switch (longSize) { case 1: - return new Size1Des(buffer, bufferOffset); + return new Size1Des(fromMemory, bufferOffset); case 2: - return new Size2Des(buffer, bufferOffset); + return new Size2Des(fromMemory, bufferOffset); case 4: - return new Size4Des(buffer, bufferOffset); + return new Size4Des(fromMemory, bufferOffset); case 8: - return new Size8Des(buffer, bufferOffset); + return new Size8Des(fromMemory, bufferOffset); case 12: - return new Size12Des(buffer, bufferOffset); + return new Size12Des(fromMemory, bufferOffset); case 16: - return new Size16Des(buffer, bufferOffset); + return new Size16Des(fromMemory, bufferOffset); case 20: - return new Size20Des(buffer, bufferOffset); + return new Size20Des(fromMemory, bufferOffset); case 24: - return new Size24Des(buffer, bufferOffset); + return new Size24Des(fromMemory, bufferOffset); case 32: - return new Size32Des(buffer, bufferOffset); + return new Size32Des(fromMemory, bufferOffset); case 40: - return new Size40Des(buffer, bufferOffset); + return new Size40Des(fromMemory, bufferOffset); case 48: - return new Size48Des(buffer, bufferOffset); + return new Size48Des(fromMemory, bufferOffset); case 56: - return new Size56Des(buffer, bufferOffset); + return new Size56Des(fromMemory, bufferOffset); case 64: - return new Size64Des(buffer, bufferOffset); + return new Size64Des(fromMemory, bufferOffset); default: throw new IAE("Unsupported size %s", longSize); } @@ -393,12 +394,12 @@ public interface LongDeserializer private static final class Size1Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size1Des(ByteBuffer buffer, int bufferOffset) + public Size1Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @@ -406,18 +407,18 @@ public Size1Des(ByteBuffer buffer, int bufferOffset) public long get(int index) { int shift = 7 - (index & 7); - return (buffer.get(offset + (index >> 3)) >> shift) & 1; + return (memory.getByte(offset + (index >> 3)) >> shift) & 1; } } private static final class Size2Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size2Des(ByteBuffer buffer, int bufferOffset) + public Size2Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @@ -425,18 +426,18 @@ public Size2Des(ByteBuffer buffer, int bufferOffset) public long get(int index) { int shift = 6 - ((index & 3) << 1); - return (buffer.get(offset + (index >> 2)) >> shift) & 3; + return (memory.getByte(offset + (index >> 2)) >> shift) & 3; } } private static final class Size4Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size4Des(ByteBuffer buffer, int bufferOffset) + public Size4Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @@ -444,36 +445,36 @@ public Size4Des(ByteBuffer buffer, int bufferOffset) public long get(int index) { int shift = ((index + 1) & 1) << 2; - return (buffer.get(offset + (index >> 1)) >> shift) & 0xF; + return (memory.getByte(offset + (index >> 1)) >> shift) & 0xF; } } private static final class Size8Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size8Des(ByteBuffer buffer, int bufferOffset) + public Size8Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @Override public long get(int index) { - return buffer.get(offset + index) & 0xFF; + return memory.getByte(offset + index) & 0xFF; } } private static final class Size12Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size12Des(ByteBuffer buffer, int bufferOffset) + public Size12Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @@ -482,35 +483,35 @@ public long get(int index) { int shift = ((index + 1) & 1) << 2; int offset = (index * 3) >> 1; - return (buffer.getShort(this.offset + offset) >> shift) & 0xFFF; + return (Short.reverseBytes(memory.getShort(this.offset + offset)) >> shift) & 0xFFF; } } private static final class Size16Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size16Des(ByteBuffer buffer, int bufferOffset) + public Size16Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } public long get(int index) { - return buffer.getShort(offset + (index << 1)) & 0xFFFF; + return Short.reverseBytes(memory.getShort(offset + (index << 1))) & 0xFFFF; } } private static final class Size20Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size20Des(ByteBuffer buffer, int bufferOffset) + public Size20Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @@ -519,116 +520,129 @@ public long get(int index) { int shift = (((index + 1) & 1) << 2) + 8; int offset = (index * 5) >> 1; - return (buffer.getInt(this.offset + offset) >> shift) & 0xFFFFF; + return (Integer.reverseBytes(memory.getInt(this.offset + offset)) >> shift) & 0xFFFFF; } } private static final class Size24Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size24Des(ByteBuffer buffer, int bufferOffset) + public Size24Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @Override public long get(int index) { - return buffer.getInt(offset + index * 3) >>> 8; + return Integer.reverseBytes(memory.getInt(offset + index * 3)) >>> 8; } } private static final class Size32Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size32Des(ByteBuffer buffer, int bufferOffset) + public Size32Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @Override public long get(int index) { - return buffer.getInt(offset + (index << 2)) & 0xFFFFFFFFL; + return Integer.reverseBytes(memory.getInt(offset + (index << 2))) & 0xFFFFFFFFL; } } private static final class Size40Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size40Des(ByteBuffer buffer, int bufferOffset) + public Size40Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @Override public long get(int index) { - return buffer.getLong(offset + index * 5) >>> 24; + return Long.reverseBytes(memory.getLong(offset + index * 5)) >>> 24; } } private static final class Size48Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size48Des(ByteBuffer buffer, int bufferOffset) + public Size48Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @Override public long get(int index) { - return buffer.getLong(offset + index * 6) >>> 16; + return Long.reverseBytes(memory.getLong(offset + index * 6)) >>> 16; } } private static final class Size56Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size56Des(ByteBuffer buffer, int bufferOffset) + public Size56Des(Memory buffer, int bufferOffset) { - this.buffer = buffer; + this.memory = buffer; this.offset = bufferOffset; } @Override public long get(int index) { - return buffer.getLong(offset + index * 7) >>> 8; + return Long.reverseBytes(memory.getLong(offset + index * 7)) >>> 8; } } private static final class Size64Des implements LongDeserializer { - final ByteBuffer buffer; + final Memory memory; final int offset; - public Size64Des(ByteBuffer buffer, int bufferOffset) + public Size64Des(Memory memory, int bufferOffset) { - this.buffer = buffer; + this.memory = memory; this.offset = bufferOffset; } @Override public long get(int index) { - return buffer.getLong(offset + (index << 3)); + return Long.reverseBytes(memory.getLong(offset + (index << 3))); } } + public static void main(String[] args){ + ByteBuffer bb = ByteBuffer.allocate(7).order(ByteOrder.nativeOrder()); + bb.put(new Byte("0")); + bb.put(new Byte("0")); + bb.put(new Byte("0")); + bb.put(new Byte("0")); + bb.put(new Byte("0")); + bb.put(new Byte("1")); + bb.put(new Byte("0")); + + System.out.println("Long=" + bb.getLong()); + } + } diff --git a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java index 2900c36ad023..fe1305196ba2 100644 --- a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java @@ -22,11 +22,11 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; /** @@ -53,6 +53,6 @@ public interface Serializer public interface Deserializer { - public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig); + public void read(PositionalMemoryRegion pMemory, ColumnBuilder builder, ColumnConfig columnConfig); } } diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java index 74f9f131b87e..f62a6fb5cc3a 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java @@ -22,12 +22,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; import io.druid.segment.data.GenericIndexed; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; /** @@ -80,10 +80,10 @@ public Deserializer getDeserializer() return new Deserializer() { @Override - public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) + public void read(PositionalMemoryRegion pMemory, ColumnBuilder builder, ColumnConfig columnConfig) { if (serde != null) { - serde.deserializeColumn(buffer, builder); + serde.deserializeColumn(pMemory, builder); } } }; diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index 5a0d3418ac40..be2072d2a2f5 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -21,12 +21,12 @@ import com.google.common.base.Function; import io.druid.segment.GenericColumnSerializer; +import com.yahoo.memory.NativeMemory; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; -import java.nio.ByteBuffer; - /** */ public abstract class ComplexMetricSerde @@ -39,10 +39,9 @@ public abstract class ComplexMetricSerde * Deserializes a ByteBuffer and adds it to the ColumnBuilder. This method allows for the ComplexMetricSerde * to implement it's own versioning scheme to allow for changes of binary format in a forward-compatible manner. * - * @param buffer the buffer to deserialize * @param builder ColumnBuilder to add the column to */ - public abstract void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder); + public abstract void deserializeColumn(PositionalMemoryRegion pMemory, ColumnBuilder builder); /** * This is deprecated because its usage is going to be removed from the code. @@ -87,18 +86,12 @@ public byte[] toBytes(Object val) * Converts byte[] to intermediate representation of the aggregate. * * @param data array - * @param start offset in the byte array where to start reading - * @param numBytes number of bytes to read in given array * * @return intermediate representation of the aggregate */ - public Object fromBytes(byte[] data, int start, int numBytes) + public Object fromBytes(byte[] data) { - ByteBuffer bb = ByteBuffer.wrap(data); - if (start > 0) { - bb.position(start); - } - return getObjectStrategy().fromByteBuffer(bb, numBytes); + return getObjectStrategy().fromMemory(new NativeMemory(data)); } /** diff --git a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 67375f9539a4..35d1f786c09b 100644 --- a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -27,7 +27,7 @@ import io.druid.collections.spatial.ImmutableRTree; import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.FileSmoosher; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.CompressedVSizeIndexedSupplier; import io.druid.segment.CompressedVSizeIndexedV3Supplier; import io.druid.segment.column.ColumnBuilder; @@ -35,7 +35,6 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; -import io.druid.segment.data.ByteBufferSerializer; import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier; import io.druid.segment.data.GenericIndexed; @@ -44,6 +43,7 @@ import io.druid.segment.data.IndexedIntsWriter; import io.druid.segment.data.IndexedMultivalue; import io.druid.segment.data.IndexedRTree; +import io.druid.segment.data.MemorySerializer; import io.druid.segment.data.VSizeIndexed; import io.druid.segment.data.VSizeIndexedInts; import io.druid.segment.data.WritableSupplier; @@ -397,7 +397,7 @@ public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOE } if (spatialIndex != null) { - ByteBufferSerializer.writeToChannel( + MemorySerializer.writeToChannel( spatialIndex, new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()), channel @@ -421,13 +421,13 @@ public Deserializer getDeserializer() return new Deserializer() { @Override - public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) + public void read(PositionalMemoryRegion pMemory, ColumnBuilder builder, ColumnConfig columnConfig) { - final VERSION rVersion = VERSION.fromByte(buffer.get()); + final VERSION rVersion = VERSION.fromByte(pMemory.getByte()); final int rFlags; if (rVersion.compareTo(VERSION.COMPRESSED) >= 0) { - rFlags = buffer.getInt(); + rFlags = Integer.reverseBytes(pMemory.getInt()); } else { rFlags = rVersion.equals(VERSION.UNCOMPRESSED_MULTI_VALUE) ? Feature.MULTI_VALUE.getMask() @@ -436,21 +436,17 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo final boolean hasMultipleValues = Feature.MULTI_VALUE.isSet(rFlags) || Feature.MULTI_VALUE_V3.isSet(rFlags); - final GenericIndexed rDictionary = GenericIndexed.read( - buffer, - GenericIndexed.STRING_STRATEGY, - builder.getFileMapper() - ); + final GenericIndexed rDictionary = GenericIndexed.read(pMemory, GenericIndexed.STRING_STRATEGY); builder.setType(ValueType.STRING); final WritableSupplier rSingleValuedColumn; final WritableSupplier> rMultiValuedColumn; if (hasMultipleValues) { - rMultiValuedColumn = readMultiValuedColum(rVersion, buffer, rFlags, builder.getFileMapper()); + rMultiValuedColumn = readMultiValuedColum(rVersion, pMemory, rFlags); rSingleValuedColumn = null; } else { - rSingleValuedColumn = readSingleValuedColumn(rVersion, buffer, builder.getFileMapper()); + rSingleValuedColumn = readSingleValuedColumn(rVersion, pMemory); rMultiValuedColumn = null; } @@ -465,7 +461,7 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo ); GenericIndexed rBitmaps = GenericIndexed.read( - buffer, bitmapSerdeFactory.getObjectStrategy(), builder.getFileMapper() + pMemory, bitmapSerdeFactory.getObjectStrategy() ); builder.setBitmapIndex( new BitmapIndexColumnPartSupplier( @@ -476,42 +472,38 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo ); ImmutableRTree rSpatialIndex = null; - if (buffer.hasRemaining()) { - rSpatialIndex = ByteBufferSerializer.read( - buffer, new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()) + if (pMemory.hasRemaining()) { + rSpatialIndex = MemorySerializer.read( + pMemory, new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()) ); builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(rSpatialIndex)); } } - private WritableSupplier readSingleValuedColumn( - VERSION version, - ByteBuffer buffer, - SmooshedFileMapper fileMapper - ) + private WritableSupplier readSingleValuedColumn(VERSION version, PositionalMemoryRegion pMemory) { switch (version) { case UNCOMPRESSED_SINGLE_VALUE: - return VSizeIndexedInts.readFromByteBuffer(buffer).asWritableSupplier(); + return VSizeIndexedInts.readFromMemory(pMemory).asWritableSupplier(); case COMPRESSED: - return CompressedVSizeIntsIndexedSupplier.fromByteBuffer(buffer, byteOrder, fileMapper); + return CompressedVSizeIntsIndexedSupplier.fromMemory(pMemory, byteOrder); } throw new IAE("Unsupported single-value version[%s]", version); } private WritableSupplier> readMultiValuedColum( - VERSION version, ByteBuffer buffer, int flags, SmooshedFileMapper fileMapper + VERSION version, PositionalMemoryRegion pMemory, int flags ) { switch (version) { case UNCOMPRESSED_MULTI_VALUE: - return VSizeIndexed.readFromByteBuffer(buffer).asWritableSupplier(); + return VSizeIndexed.readFromMemory(pMemory).asWritableSupplier(); case COMPRESSED: if (Feature.MULTI_VALUE.isSet(flags)) { - return CompressedVSizeIndexedSupplier.fromByteBuffer(buffer, byteOrder, fileMapper); + return CompressedVSizeIndexedSupplier.fromMemory(pMemory, byteOrder); } else if (Feature.MULTI_VALUE_V3.isSet(flags)) { - return CompressedVSizeIndexedV3Supplier.fromByteBuffer(buffer, byteOrder, fileMapper); + return CompressedVSizeIndexedV3Supplier.fromMemory(pMemory, byteOrder); } else { throw new IAE("Unrecognized multi-value flag[%d]", flags); } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java index 4e693dc671e8..631153dda79c 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.FloatColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; @@ -30,7 +31,6 @@ import io.druid.segment.data.CompressedFloatsIndexedSupplier; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -159,12 +159,11 @@ public Deserializer getDeserializer() return new Deserializer() { @Override - public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) + public void read(PositionalMemoryRegion pMemory, ColumnBuilder builder, ColumnConfig columnConfig) { - final CompressedFloatsIndexedSupplier column = CompressedFloatsIndexedSupplier.fromByteBuffer( - buffer, - byteOrder, - builder.getFileMapper() + final CompressedFloatsIndexedSupplier column = CompressedFloatsIndexedSupplier.fromMemory( + pMemory, + byteOrder ); builder.setType(ValueType.FLOAT) .setHasMultipleValues(false) diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java index d8eeff9ae738..0e38ad3fd318 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.LongColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; @@ -30,7 +30,6 @@ import io.druid.segment.data.CompressedLongsIndexedSupplier; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -159,12 +158,11 @@ public Deserializer getDeserializer() return new Deserializer() { @Override - public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) + public void read(PositionalMemoryRegion pMemory, ColumnBuilder builder, ColumnConfig columnConfig) { - final CompressedLongsIndexedSupplier column = CompressedLongsIndexedSupplier.fromByteBuffer( - buffer, - byteOrder, - builder.getFileMapper() + final CompressedLongsIndexedSupplier column = CompressedLongsIndexedSupplier.fromMemory( + pMemory, + byteOrder ); builder.setType(ValueType.LONG) .setHasMultipleValues(false) diff --git a/processing/src/test/java/io/druid/segment/CompressedVSizeIndexedV3SupplierTest.java b/processing/src/test/java/io/druid/segment/CompressedVSizeIndexedV3SupplierTest.java index a09dd68e0884..1b13d84a57aa 100644 --- a/processing/src/test/java/io/druid/segment/CompressedVSizeIndexedV3SupplierTest.java +++ b/processing/src/test/java/io/druid/segment/CompressedVSizeIndexedV3SupplierTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressedVSizeIndexedSupplierTest; import io.druid.segment.data.IndexedInts; @@ -30,7 +31,6 @@ import org.junit.After; import org.junit.Before; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; @@ -68,12 +68,10 @@ public void teardown(){ } @Override - protected WritableSupplier> fromByteBuffer(ByteBuffer buffer, ByteOrder order) + protected WritableSupplier> fromMemory(PositionalMemoryRegion memory, ByteOrder order) { - return CompressedVSizeIndexedV3Supplier.fromByteBuffer( - buffer, - ByteOrder.nativeOrder(), - null + return CompressedVSizeIndexedV3Supplier.fromMemory( + memory, ByteOrder.nativeOrder() ); } } diff --git a/processing/src/test/java/io/druid/segment/data/CompressedFloatsSerdeTest.java b/processing/src/test/java/io/druid/segment/data/CompressedFloatsSerdeTest.java index d47b060b1f0a..b3675b741d4a 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedFloatsSerdeTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedFloatsSerdeTest.java @@ -23,6 +23,7 @@ import com.google.common.io.ByteSink; import com.google.common.primitives.Floats; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,7 +32,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.Channels; import java.util.ArrayList; @@ -129,7 +129,7 @@ public OutputStream openStream() throws IOException ); Assert.assertEquals(baos.size(), serializer.getSerializedSize()); CompressedFloatsIndexedSupplier supplier = CompressedFloatsIndexedSupplier - .fromByteBuffer(ByteBuffer.wrap(baos.toByteArray()), order, null); + .fromMemory(new PositionalMemoryRegion(baos.toByteArray()), order); IndexedFloats floats = supplier.get(); assertIndexMatchesVals(floats, values); @@ -182,8 +182,8 @@ private void testSupplierSerde(CompressedFloatsIndexedSupplier supplier, float[] final byte[] bytes = baos.toByteArray(); Assert.assertEquals(supplier.getSerializedSize(), bytes.length); - CompressedFloatsIndexedSupplier anotherSupplier = CompressedFloatsIndexedSupplier.fromByteBuffer( - ByteBuffer.wrap(bytes), order, null + CompressedFloatsIndexedSupplier anotherSupplier = CompressedFloatsIndexedSupplier.fromMemory( + new PositionalMemoryRegion(bytes), order ); IndexedFloats indexed = anotherSupplier.get(); assertIndexMatchesVals(indexed, vals); diff --git a/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedSupplierTest.java b/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedSupplierTest.java index 66b6dfa94353..fb55b419740b 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedSupplierTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedSupplierTest.java @@ -22,6 +22,7 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.CompressedPools; import org.junit.After; import org.junit.Assert; @@ -103,7 +104,7 @@ private void makeWithSerde(final int chunkSize) throws IOException final byte[] bytes = baos.toByteArray(); Assert.assertEquals(theSupplier.getSerializedSize(), bytes.length); - supplier = CompressedIntsIndexedSupplier.fromByteBuffer(ByteBuffer.wrap(bytes), ByteOrder.nativeOrder(), null); + supplier = CompressedIntsIndexedSupplier.fromMemory(new PositionalMemoryRegion(bytes), ByteOrder.nativeOrder()); indexed = supplier.get(); } diff --git a/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedWriterTest.java b/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedWriterTest.java index 60ce1fa027c8..282f0db02250 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedWriterTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedWriterTest.java @@ -26,6 +26,7 @@ import com.google.common.primitives.Longs; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.java.util.common.io.smoosh.Smoosh; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; @@ -133,10 +134,9 @@ private void checkSerializedSizeAndData(int chunkFactor) throws Exception assertEquals(writtenLength, supplierFromList.getSerializedSize()); // read from ByteBuffer and check values - CompressedIntsIndexedSupplier supplierFromByteBuffer = CompressedIntsIndexedSupplier.fromByteBuffer( - ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), - byteOrder, - null + CompressedIntsIndexedSupplier supplierFromByteBuffer = CompressedIntsIndexedSupplier.fromMemory( + new PositionalMemoryRegion(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), + byteOrder ); IndexedInts indexedInts = supplierFromByteBuffer.get(); assertEquals(vals.length, indexedInts.size()); @@ -215,10 +215,9 @@ private void checkV2SerializedSizeAndData(int chunkFactor) throws Exception SmooshedFileMapper mapper = Smoosh.map(tmpDirectory); // read from ByteBuffer and check values - CompressedIntsIndexedSupplier supplierFromByteBuffer = CompressedIntsIndexedSupplier.fromByteBuffer( - mapper.mapFile("test"), - byteOrder, - mapper + CompressedIntsIndexedSupplier supplierFromByteBuffer = CompressedIntsIndexedSupplier.fromMemory( + new PositionalMemoryRegion(mapper.mapFile("test")), + byteOrder ); IndexedInts indexedInts = supplierFromByteBuffer.get(); assertEquals(vals.length, indexedInts.size()); diff --git a/processing/src/test/java/io/druid/segment/data/CompressedLongsSerdeTest.java b/processing/src/test/java/io/druid/segment/data/CompressedLongsSerdeTest.java index ffae76ff6535..a9e8e7000f09 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedLongsSerdeTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedLongsSerdeTest.java @@ -23,6 +23,7 @@ import com.google.common.io.ByteSink; import com.google.common.primitives.Longs; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,7 +32,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.Channels; import java.util.ArrayList; @@ -151,7 +151,7 @@ public OutputStream openStream() throws IOException ); Assert.assertEquals(baos.size(), serializer.getSerializedSize()); CompressedLongsIndexedSupplier supplier = CompressedLongsIndexedSupplier - .fromByteBuffer(ByteBuffer.wrap(baos.toByteArray()), order, null); + .fromMemory(new PositionalMemoryRegion(baos.toByteArray()), order); IndexedLongs longs = supplier.get(); assertIndexMatchesVals(longs, values); @@ -204,10 +204,8 @@ private void testSupplierSerde(CompressedLongsIndexedSupplier supplier, long[] v final byte[] bytes = baos.toByteArray(); Assert.assertEquals(supplier.getSerializedSize(), bytes.length); - CompressedLongsIndexedSupplier anotherSupplier = CompressedLongsIndexedSupplier.fromByteBuffer( - ByteBuffer.wrap(bytes), - order, - null + CompressedLongsIndexedSupplier anotherSupplier = CompressedLongsIndexedSupplier.fromMemory( + new PositionalMemoryRegion(bytes), order ); IndexedLongs indexed = anotherSupplier.get(); assertIndexMatchesVals(indexed, vals); diff --git a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedSupplierTest.java b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedSupplierTest.java index dd9554da4857..db1a7b338adc 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedSupplierTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedSupplierTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.CompressedVSizeIndexedSupplier; import org.junit.After; import org.junit.Assert; @@ -91,8 +92,8 @@ public void testSerde() throws IOException final byte[] bytes = baos.toByteArray(); Assert.assertEquals(indexedSupplier.getSerializedSize(), bytes.length); - WritableSupplier> deserializedIndexed = fromByteBuffer( - ByteBuffer.wrap(bytes), + WritableSupplier> deserializedIndexed = fromMemory( + new PositionalMemoryRegion(ByteBuffer.wrap(bytes)), ByteOrder.nativeOrder() ); @@ -139,12 +140,10 @@ private void assertSame(List someInts, IndexedMultivalue ind } } - protected WritableSupplier> fromByteBuffer(ByteBuffer buffer, ByteOrder order) + protected WritableSupplier> fromMemory(PositionalMemoryRegion memory, ByteOrder order) { - return CompressedVSizeIndexedSupplier.fromByteBuffer( - buffer, - ByteOrder.nativeOrder(), - null + return CompressedVSizeIndexedSupplier.fromMemory( + memory, ByteOrder.nativeOrder() ); } } diff --git a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedV3WriterTest.java b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedV3WriterTest.java index c14942367aef..3f21873eafe2 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedV3WriterTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedV3WriterTest.java @@ -25,11 +25,14 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; +import com.yahoo.memory.Memory; +import com.yahoo.memory.MemoryRegion; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.java.util.common.io.smoosh.Smoosh; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.CompressedVSizeIndexedV3Supplier; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -42,7 +45,6 @@ import javax.annotation.Nullable; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; @@ -154,10 +156,9 @@ public IndexedInts apply(@Nullable final int[] input) assertEquals(writtenLength, supplierFromIterable.getSerializedSize()); // read from ByteBuffer and check values - CompressedVSizeIndexedV3Supplier supplierFromByteBuffer = CompressedVSizeIndexedV3Supplier.fromByteBuffer( - ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), - byteOrder, - null + CompressedVSizeIndexedV3Supplier supplierFromByteBuffer = CompressedVSizeIndexedV3Supplier.fromMemory( + new PositionalMemoryRegion(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), + byteOrder ); indexedMultivalue = supplierFromByteBuffer.get(); assertEquals(indexedMultivalue.size(), vals.size()); @@ -299,10 +300,10 @@ private void checkV2SerializedSizeAndData(int offsetChunkFactor, int valueChunkF smoosher.close(); SmooshedFileMapper mapper = Smoosh.map(tmpDirectory); - CompressedVSizeIndexedV3Supplier supplierFromByteBuffer = CompressedVSizeIndexedV3Supplier.fromByteBuffer( - mapper.mapFile("test"), - byteOrder, - mapper + Memory memory = mapper.mapFileToMemory("test"); + CompressedVSizeIndexedV3Supplier supplierFromByteBuffer = CompressedVSizeIndexedV3Supplier.fromMemory( + new PositionalMemoryRegion(new MemoryRegion(memory, 0, memory.getCapacity())), + byteOrder ); IndexedMultivalue indexedMultivalue = supplierFromByteBuffer.get(); assertEquals(indexedMultivalue.size(), vals.size()); diff --git a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplierTest.java b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplierTest.java index bb29c04513ca..269e1c021b7e 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplierTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplierTest.java @@ -25,6 +25,7 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.CompressedPools; import org.junit.After; import org.junit.Assert; @@ -148,7 +149,7 @@ private void makeWithSerde(final int chunkSize) throws IOException final byte[] bytes = baos.toByteArray(); Assert.assertEquals(theSupplier.getSerializedSize(), bytes.length); - supplier = CompressedVSizeIntsIndexedSupplier.fromByteBuffer(ByteBuffer.wrap(bytes), byteOrder, null); + supplier = CompressedVSizeIntsIndexedSupplier.fromMemory(new PositionalMemoryRegion(bytes), byteOrder); indexed = supplier.get(); } diff --git a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriterTest.java b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriterTest.java index 14afd1b1648f..f4b1981efd08 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriterTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriterTest.java @@ -26,6 +26,7 @@ import com.google.common.primitives.Longs; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.java.util.common.io.smoosh.Smoosh; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; @@ -131,10 +132,9 @@ private void checkSerializedSizeAndData(int chunkSize) throws Exception assertEquals(writtenLength, supplierFromList.getSerializedSize()); // read from ByteBuffer and check values - CompressedVSizeIntsIndexedSupplier supplierFromByteBuffer = CompressedVSizeIntsIndexedSupplier.fromByteBuffer( - ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), - byteOrder, - null + CompressedVSizeIntsIndexedSupplier supplierFromByteBuffer = CompressedVSizeIntsIndexedSupplier.fromMemory( + new PositionalMemoryRegion(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), + byteOrder ); IndexedInts indexedInts = supplierFromByteBuffer.get(); for (int i = 0; i < vals.length; ++i) { @@ -209,10 +209,9 @@ private void checkV2SerializedSizeAndData(int chunkSize) throws Exception SmooshedFileMapper mapper = Smoosh.map(tmpDirectory); - CompressedVSizeIntsIndexedSupplier supplierFromByteBuffer = CompressedVSizeIntsIndexedSupplier.fromByteBuffer( - mapper.mapFile("test"), - byteOrder, - mapper + CompressedVSizeIntsIndexedSupplier supplierFromByteBuffer = CompressedVSizeIntsIndexedSupplier.fromMemory( + new PositionalMemoryRegion(mapper.mapFile("test")), + byteOrder ); IndexedInts indexedInts = supplierFromByteBuffer.get(); diff --git a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java index f4bfd70bfffb..e108e3a0a4b3 100644 --- a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java @@ -20,12 +20,16 @@ package io.druid.segment.data; import com.google.common.collect.Maps; +import com.yahoo.memory.MemoryRegion; +import com.yahoo.memory.NativeMemory; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import org.junit.Assert; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; import java.util.HashMap; @@ -128,11 +132,15 @@ private GenericIndexed serializeAndDeserialize(GenericIndexed in indexed.writeToChannel(channel); channel.close(); - final ByteBuffer byteBuffer = ByteBuffer.wrap(baos.toByteArray()); + byte[] bytes = baos.toByteArray(); + final ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length) + .order(ByteOrder.nativeOrder()).put(baos.toByteArray()); + byteBuffer.flip(); Assert.assertEquals(indexed.getSerializedSize(), byteBuffer.remaining()); GenericIndexed deserialized = GenericIndexed.read( - byteBuffer, GenericIndexed.STRING_STRATEGY + new PositionalMemoryRegion(byteBuffer), GenericIndexed.STRING_STRATEGY ); + byteBuffer.position(byteBuffer.position() + (int)deserialized.getSerializedSize()); Assert.assertEquals(0, byteBuffer.remaining()); return deserialized; } diff --git a/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java index 3894f0e8a490..ec7730c3f11c 100644 --- a/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java +++ b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java @@ -20,11 +20,11 @@ package io.druid.segment.data; import com.google.common.primitives.Ints; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import org.junit.Assert; import org.junit.Test; import java.io.ByteArrayOutputStream; -import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.util.List; @@ -56,7 +56,7 @@ public void testSerialization() throws Exception final byte[] bytes = baos.toByteArray(); Assert.assertEquals(ints.getSerializedSize(), bytes.length); - VSizeIndexedInts deserialized = VSizeIndexedInts.readFromByteBuffer(ByteBuffer.wrap(bytes)); + VSizeIndexedInts deserialized = VSizeIndexedInts.readFromMemory(new PositionalMemoryRegion(bytes)); Assert.assertEquals(1, deserialized.getNumBytes()); Assert.assertEquals(array.length, deserialized.size()); diff --git a/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsWriterTest.java b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsWriterTest.java index 4a0a4a2926ad..f8d369aff898 100644 --- a/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsWriterTest.java +++ b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsWriterTest.java @@ -20,13 +20,13 @@ package io.druid.segment.data; import com.google.common.primitives.Ints; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; import java.util.Random; @@ -84,8 +84,8 @@ private void checkSerializedSizeAndData() throws Exception assertEquals(writtenLength, intsFromList.getSerializedSize()); // read from ByteBuffer and check values - VSizeIndexedInts intsFromByteBuffer = VSizeIndexedInts.readFromByteBuffer( - ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))) + VSizeIndexedInts intsFromByteBuffer = VSizeIndexedInts.readFromMemory( + new PositionalMemoryRegion(IOUtils.toByteArray(ioPeon.makeInputStream("output"))) ); assertEquals(vals.length, intsFromByteBuffer.size()); for (int i = 0; i < vals.length; ++i) { diff --git a/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java b/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java index ee46c55bde6e..a8253d6b0d19 100644 --- a/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import org.junit.Assert; import org.junit.Test; @@ -64,7 +65,7 @@ public VSizeIndexedInts apply(int[] input) final byte[] bytes = baos.toByteArray(); Assert.assertEquals(indexed.getSerializedSize(), bytes.length); - VSizeIndexed deserializedIndexed = VSizeIndexed.readFromByteBuffer(ByteBuffer.wrap(bytes)); + VSizeIndexed deserializedIndexed = VSizeIndexed.readFromMemory(new PositionalMemoryRegion(ByteBuffer.wrap(bytes))); assertSame(someInts, deserializedIndexed); } diff --git a/processing/src/test/java/io/druid/segment/data/VSizeLongSerdeTest.java b/processing/src/test/java/io/druid/segment/data/VSizeLongSerdeTest.java index fea87e9956ec..8aee13a9d349 100644 --- a/processing/src/test/java/io/druid/segment/data/VSizeLongSerdeTest.java +++ b/processing/src/test/java/io/druid/segment/data/VSizeLongSerdeTest.java @@ -19,7 +19,8 @@ package io.druid.segment.data; - +import com.yahoo.memory.NativeMemory; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -115,9 +116,10 @@ public void testSerde(int longSize, long[] values) throws IOException buffer = ByteBuffer.wrap(outStream.toByteArray()); Assert.assertEquals(VSizeLongSerde.getSerializedSize(longSize, values.length), buffer.capacity()); Assert.assertEquals(VSizeLongSerde.getSerializedSize(longSize, values.length), outBuffer.position()); - VSizeLongSerde.LongDeserializer streamDes = VSizeLongSerde.getDeserializer(longSize, buffer, 0); - VSizeLongSerde.LongDeserializer bufferDes = VSizeLongSerde.getDeserializer(longSize, outBuffer, 0); + VSizeLongSerde.LongDeserializer streamDes = VSizeLongSerde.getDeserializer(longSize, new NativeMemory(buffer), 0); + VSizeLongSerde.LongDeserializer bufferDes = VSizeLongSerde.getDeserializer(longSize, new NativeMemory(outBuffer), 0); for (int i = 0; i < values.length; i++) { + System.out.println("i=" + i + " longsize=" + longSize); Assert.assertEquals(values[i], streamDes.get(i)); Assert.assertEquals(values[i], bufferDes.get(i)); } @@ -139,13 +141,14 @@ public void testSerdeIncLoop(int longSize, long start, long end) throws IOExcept buffer = ByteBuffer.wrap(outStream.toByteArray()); Assert.assertEquals(VSizeLongSerde.getSerializedSize(longSize, (int) (end - start)), buffer.capacity()); Assert.assertEquals(VSizeLongSerde.getSerializedSize(longSize, (int) (end - start)), outBuffer.position()); - VSizeLongSerde.LongDeserializer streamDes = VSizeLongSerde.getDeserializer(longSize, buffer, 0); - VSizeLongSerde.LongDeserializer bufferDes = VSizeLongSerde.getDeserializer(longSize, outBuffer, 0); + VSizeLongSerde.LongDeserializer streamDes = VSizeLongSerde.getDeserializer(longSize, + (new PositionalMemoryRegion(buffer)).getRemainingMemory(), 0); + VSizeLongSerde.LongDeserializer bufferDes = VSizeLongSerde.getDeserializer(longSize, + new NativeMemory(outBuffer), 0); for (int i = 0; i < end - start; i++) { + System.out.println("i=" + i + " start=" + start + " longsize=" + longSize); Assert.assertEquals(start + i, streamDes.get(i)); Assert.assertEquals(start + i, bufferDes.get(i)); } } - - } diff --git a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java index 3f2df176c16b..850d3d0dc4a5 100644 --- a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java +++ b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java @@ -22,8 +22,10 @@ import com.google.common.collect.Ordering; import com.google.common.hash.HashFunction; import com.metamx.common.StringUtils; +import com.yahoo.memory.Memory; import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogCollector; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; @@ -97,16 +99,13 @@ public HyperLogLogCollector extractValue(InputRow inputRow, String metricName) }; } - @Override - public void deserializeColumn( - ByteBuffer byteBuffer, ColumnBuilder columnBuilder - ) + @Override public void deserializeColumn(PositionalMemoryRegion pMemory, ColumnBuilder columnBuilder) { final GenericIndexed column; if (columnBuilder.getFileMapper() == null) { - column = GenericIndexed.read(byteBuffer, getObjectStrategy()); + column = GenericIndexed.read(pMemory, getObjectStrategy()); } else { - column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); + column = GenericIndexed.read(pMemory, getObjectStrategy(), columnBuilder.getFileMapper()); } columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); @@ -123,12 +122,11 @@ public Class getClazz() return HyperLogLogCollector.class; } - @Override - public HyperLogLogCollector fromByteBuffer(ByteBuffer buffer, int numBytes) + @Override public HyperLogLogCollector fromMemory(Memory memory) { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); - return HyperLogLogCollector.makeCollector(readOnlyBuffer); + byte[] bytes = new byte[(int)memory.getCapacity()]; + memory.getByteArray(0, bytes, 0, bytes.length); + return HyperLogLogCollector.makeCollector(ByteBuffer.wrap(bytes)); } @Override diff --git a/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java b/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java index bba412f58bc8..482609552aaa 100644 --- a/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java +++ b/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java @@ -24,6 +24,7 @@ import com.google.common.primitives.Longs; import io.druid.hll.HyperLogLogCollector; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.java.util.common.io.smoosh.PositionalMemoryRegion; import io.druid.java.util.common.io.smoosh.Smoosh; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; @@ -91,7 +92,7 @@ public void testSanity() throws IOException .setType(ValueType.COMPLEX) .setHasMultipleValues(false) .setFileMapper(mapper); - serde.deserializeColumn(mapper.mapFile("test"), builder); + serde.deserializeColumn(new PositionalMemoryRegion(mapper.mapFile("test")), builder); Column column = builder.build(); ComplexColumn complexColumn = column.getComplexColumn();