diff --git a/bytebuffer-collections/pom.xml b/bytebuffer-collections/pom.xml deleted file mode 100755 index 435e82b9759f..000000000000 --- a/bytebuffer-collections/pom.xml +++ /dev/null @@ -1,137 +0,0 @@ - - - - - 4.0.0 - - - io.druid - druid - 0.12.0-rc2-SNAPSHOT - - - bytebuffer-collections - bytebuffer-collections - ByteBuffer Collections - - - - io.druid - extendedset - ${project.parent.version} - - - com.google.guava - guava - 16.0.1 - - - com.fasterxml.jackson.core - jackson-annotations - 2.4.6 - - - com.fasterxml.jackson.core - jackson-core - 2.4.6 - - - com.fasterxml.jackson.core - jackson-databind - 2.4.6 - - - org.roaringbitmap - RoaringBitmap - - - - - junit - junit - test - - - org.easymock - easymock - 3.0 - test - - - com.carrotsearch - junit-benchmarks - 0.7.2 - test - - - com.h2database - h2 - 1.4.182 - test - - - com.google.guava - guava-testlib - test - - - - - - - org.apache.maven.plugins - maven-jar-plugin - 2.4 - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - io.druid.test.annotation.Benchmark - - - - - - - - benchmark - - - - maven-surefire-plugin - - -server -Xms3G -Xmx3G -Djub.consumers=CONSOLE,H2 -Djub.db.file=benchmarks/benchmarks - io.druid.test.annotation.Benchmark - io.druid.test.annotation.Dummy - - - - - - - diff --git a/pom.xml b/pom.xml index a2cf140d708a..5d187eb938ef 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,6 @@ benchmarks aws-common java-util - bytebuffer-collections extendedset hll diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html b/processing/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html rename to processing/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp b/processing/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp rename to processing/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html b/processing/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html rename to processing/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp b/processing/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp rename to processing/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp diff --git a/processing/pom.xml b/processing/pom.xml index b2ff197662bd..d964346cce94 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -47,9 +47,25 @@ io.druid - bytebuffer-collections + extendedset ${project.parent.version} + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + org.roaringbitmap + RoaringBitmap + it.unimi.dsi fastutil @@ -125,6 +141,17 @@ JUnitParams test + + com.h2database + h2 + 1.4.182 + test + + + com.google.guava + guava-testlib + test + @@ -156,7 +183,32 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + io.druid.collections.test.annotation.Benchmark + + + + + benchmark + + + + maven-surefire-plugin + + -server -Xms3G -Xmx3G -Djub.consumers=CONSOLE,H2 -Djub.db.file=benchmarks/benchmarks + io.druid.collections.test.annotation.Benchmark + io.druid.collections.test.annotation.Dummy + + + + + + + diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/IntegerSet.java b/processing/src/main/java/io/druid/collections/IntegerSet.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/IntegerSet.java rename to processing/src/main/java/io/druid/collections/IntegerSet.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java b/processing/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java rename to processing/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitmapFactory.java b/processing/src/main/java/io/druid/collections/bitmap/BitmapFactory.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitmapFactory.java rename to processing/src/main/java/io/druid/collections/bitmap/BitmapFactory.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java b/processing/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java rename to processing/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/MutableBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/MutableBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/MutableBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/MutableBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java b/processing/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java rename to processing/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java old mode 100755 new mode 100644 similarity index 94% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java index 01768cfecf97..349fb493743e --- a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java +++ b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java @@ -64,11 +64,6 @@ public byte[] toBytes() return bitmap.toBytes(); } - public int compareTo(ImmutableBitmap other) - { - return bitmap.compareTo(((WrappedImmutableConciseBitmap) other).getBitmap()); - } - @Override public String toString() { diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java b/processing/src/main/java/io/druid/collections/spatial/ImmutableNode.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java rename to processing/src/main/java/io/druid/collections/spatial/ImmutableNode.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java b/processing/src/main/java/io/druid/collections/spatial/ImmutablePoint.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java rename to processing/src/main/java/io/druid/collections/spatial/ImmutablePoint.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java b/processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java old mode 100755 new mode 100644 similarity index 79% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java rename to processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java index 2cb3e3d4acf0..9b442d3c5bc8 --- a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java +++ b/processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java @@ -27,24 +27,36 @@ import io.druid.collections.spatial.search.Bound; import io.druid.collections.spatial.search.GutmanSearchStrategy; import io.druid.collections.spatial.search.SearchStrategy; +import io.druid.io.Channels; +import io.druid.segment.writeout.WriteOutBytes; +import it.unimi.dsi.fastutil.bytes.ByteArrays; +import java.io.IOException; import java.nio.ByteBuffer; /** * An immutable representation of an {@link RTree} for spatial indexing. */ -public class ImmutableRTree +public final class ImmutableRTree { - private static byte VERSION = 0x0; + private static final byte VERSION = 0x0; + + private static final ImmutableRTree EMPTY = new ImmutableRTree(); + + public static ImmutableRTree empty() + { + return EMPTY; + } + private final int numDims; private final ImmutableNode root; private final ByteBuffer data; private final SearchStrategy defaultSearchStrategy = new GutmanSearchStrategy(); - public ImmutableRTree() + private ImmutableRTree() { this.numDims = 0; - this.data = ByteBuffer.wrap(new byte[]{}); + this.data = ByteBuffer.wrap(ByteArrays.EMPTY_ARRAY); this.root = null; } @@ -52,7 +64,7 @@ public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory) { data = data.asReadOnlyBuffer(); final int initPosition = data.position(); - Preconditions.checkArgument(data.get(0) == VERSION, "Mismatching versions"); + Preconditions.checkArgument(data.get(initPosition) == VERSION, "Mismatching versions"); this.numDims = data.getInt(1 + initPosition) & 0x7FFF; this.data = data; this.root = new ImmutableNode(numDims, initPosition, 1 + Ints.BYTES, data, bitmapFactory); @@ -61,7 +73,7 @@ public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory) public static ImmutableRTree newImmutableFromMutable(RTree rTree) { if (rTree.getSize() == 0) { - return new ImmutableRTree(); + return empty(); } ByteBuffer buffer = ByteBuffer.wrap(new byte[calcNumBytes(rTree)]); @@ -103,7 +115,7 @@ private static int calcNodeBytes(Node node) public int size() { - return data.capacity(); + return data.remaining(); } public Iterable search(Bound bound) @@ -123,9 +135,19 @@ public Iterable search(SearchStrategy strategy, Bound bound) public byte[] toBytes() { - ByteBuffer buf = ByteBuffer.allocate(data.capacity()); - buf.put(data.asReadOnlyBuffer()); - return buf.array(); + if (size() == 0) { + return ByteArrays.EMPTY_ARRAY; + } + byte[] res = new byte[data.remaining()]; + data.duplicate().get(res); + return res; + } + + public void writeTo(WriteOutBytes out) throws IOException + { + if (size() != 0) { + Channels.writeFully(out, data.duplicate()); + } } public int compareTo(ImmutableRTree other) diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Node.java b/processing/src/main/java/io/druid/collections/spatial/Node.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/Node.java rename to processing/src/main/java/io/druid/collections/spatial/Node.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Point.java b/processing/src/main/java/io/druid/collections/spatial/Point.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/Point.java rename to processing/src/main/java/io/druid/collections/spatial/Point.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTree.java b/processing/src/main/java/io/druid/collections/spatial/RTree.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTree.java rename to processing/src/main/java/io/druid/collections/spatial/RTree.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTreeUtils.java b/processing/src/main/java/io/druid/collections/spatial/RTreeUtils.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTreeUtils.java rename to processing/src/main/java/io/druid/collections/spatial/RTreeUtils.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/Bound.java b/processing/src/main/java/io/druid/collections/spatial/search/Bound.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/Bound.java rename to processing/src/main/java/io/druid/collections/spatial/search/Bound.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java b/processing/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/PolygonBound.java b/processing/src/main/java/io/druid/collections/spatial/search/PolygonBound.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/PolygonBound.java rename to processing/src/main/java/io/druid/collections/spatial/search/PolygonBound.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RadiusBound.java b/processing/src/main/java/io/druid/collections/spatial/search/RadiusBound.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RadiusBound.java rename to processing/src/main/java/io/druid/collections/spatial/search/RadiusBound.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RectangularBound.java b/processing/src/main/java/io/druid/collections/spatial/search/RectangularBound.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RectangularBound.java rename to processing/src/main/java/io/druid/collections/spatial/search/RectangularBound.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java b/processing/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java b/processing/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java b/processing/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java b/processing/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index ce9dbbf2727b..3ad6574a08dc 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -229,12 +229,12 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) public ImmutableRTree getSpatialIndex(String dimension) { if (isVirtualColumn(dimension)) { - return new ImmutableRTree(); + return ImmutableRTree.empty(); } final Column column = index.getColumn(dimension); if (column == null || !column.getCapabilities().hasSpatialIndexes()) { - return new ImmutableRTree(); + return ImmutableRTree.empty(); } return column.getSpatialIndex().getRTree(); diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 2bbfa9ef6568..136658bf2a62 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -498,7 +498,9 @@ private static GenericIndexed fromIterableVersionOne( // for compatibility with the format, but this field is unused valuesOut.writeInt(0); - strategy.writeTo(next, valuesOut); + if (next != null) { + strategy.writeTo(next, valuesOut); + } headerOut.writeInt(Ints.checkedCast(valuesOut.size())); if (prevVal instanceof Closeable) { diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java index 31822e13cc51..341026a1c579 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java @@ -217,7 +217,9 @@ public void write(T objectToWrite) throws IOException // for compatibility with the format (see GenericIndexed javadoc for description of the format), but this field is // unused. valuesOut.writeInt(0); - strategy.writeTo(objectToWrite, valuesOut); + if (objectToWrite != null) { + strategy.writeTo(objectToWrite, valuesOut); + } if (!requireMultipleFiles) { headerOut.writeInt(Ints.checkedCast(valuesOut.size())); diff --git a/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java index 14f38f4ef49e..6e49c8626b7a 100644 --- a/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java @@ -22,8 +22,9 @@ import com.google.common.collect.Ordering; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.spatial.ImmutableRTree; -import it.unimi.dsi.fastutil.bytes.ByteArrays; +import io.druid.segment.writeout.WriteOutBytes; +import java.io.IOException; import java.nio.ByteBuffer; public class ImmutableRTreeObjectStrategy implements ObjectStrategy @@ -71,12 +72,15 @@ public ImmutableRTree fromByteBuffer(ByteBuffer buffer, int numBytes) @Override public byte[] toBytes(ImmutableRTree val) { - if (val == null || val.size() == 0) { - return ByteArrays.EMPTY_ARRAY; - } return val.toBytes(); } + @Override + public void writeTo(ImmutableRTree val, WriteOutBytes out) throws IOException + { + val.writeTo(out); + } + @Override public int compare(ImmutableRTree o1, ImmutableRTree o2) { diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index dc3c6647f3a1..8698b36e1f53 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -25,7 +25,9 @@ import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.ObjectStrategy; +import it.unimi.dsi.fastutil.bytes.ByteArrays; +import javax.annotation.Nullable; import java.nio.ByteBuffer; /** @@ -80,9 +82,9 @@ public Function inputSizeFn() * * @return serialized intermediate representation of aggregate in byte[] */ - public byte[] toBytes(Object val) + public byte[] toBytes(@Nullable Object val) { - return getObjectStrategy().toBytes(val); + return val != null ? getObjectStrategy().toBytes(val) : ByteArrays.EMPTY_ARRAY; } /** diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/IntSetTestUtility.java b/processing/src/test/java/io/druid/collections/IntSetTestUtility.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/IntSetTestUtility.java rename to processing/src/test/java/io/druid/collections/IntSetTestUtility.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/TestIntegerSet.java b/processing/src/test/java/io/druid/collections/TestIntegerSet.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/TestIntegerSet.java rename to processing/src/test/java/io/druid/collections/TestIntegerSet.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java b/processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java old mode 100755 new mode 100644 similarity index 99% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java rename to processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java index 93ee6eb9c909..26a19fe8e230 --- a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java +++ b/processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java @@ -39,7 +39,9 @@ import java.util.Locale; import java.util.Random; - +/** + * TODO rewrite this benchmark to JMH + */ @BenchmarkOptions(clock = Clock.NANO_TIME, benchmarkRounds = 50) public class BitmapBenchmark { diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapIterationTest.java b/processing/src/test/java/io/druid/collections/bitmap/BitmapIterationTest.java similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapIterationTest.java rename to processing/src/test/java/io/druid/collections/bitmap/BitmapIterationTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java b/processing/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java rename to processing/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java b/processing/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java old mode 100755 new mode 100644 similarity index 96% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java rename to processing/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java index 4c5d3769c9ab..74999ef80469 --- a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java +++ b/processing/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java @@ -24,13 +24,16 @@ import io.druid.extendedset.intset.ConciseSet; import io.druid.extendedset.intset.ImmutableConciseSet; import io.druid.java.util.common.StringUtils; -import io.druid.test.annotation.Benchmark; +import io.druid.collections.test.annotation.Benchmark; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; import org.roaringbitmap.buffer.MutableRoaringBitmap; import java.util.BitSet; +/** + * TODO rewrite this benchmark to JMH + */ @Category({Benchmark.class}) @BenchmarkHistoryChart(labelWith = LabelType.CUSTOM_KEY, maxRuns = 20) public class RangeBitmapBenchmarkTest extends BitmapBenchmark diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java b/processing/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java rename to processing/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java b/processing/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java old mode 100755 new mode 100644 similarity index 96% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java rename to processing/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java index 6a0438178891..eb6039528d92 --- a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java +++ b/processing/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java @@ -24,13 +24,16 @@ import io.druid.extendedset.intset.ConciseSet; import io.druid.extendedset.intset.ImmutableConciseSet; import io.druid.java.util.common.StringUtils; -import io.druid.test.annotation.Benchmark; +import io.druid.collections.test.annotation.Benchmark; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; import org.roaringbitmap.buffer.MutableRoaringBitmap; import java.util.BitSet; +/** + * TODO rewrite this benchmark to JMH + */ @Category({Benchmark.class}) @BenchmarkHistoryChart(labelWith = LabelType.CUSTOM_KEY, maxRuns = 20) public class UniformBitmapBenchmarkTest extends BitmapBenchmark diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java b/processing/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java rename to processing/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java b/processing/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java rename to processing/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java b/processing/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java old mode 100755 new mode 100644 similarity index 95% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java rename to processing/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java index 3c3bb9c9eba4..20e9d4d76be1 --- a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java +++ b/processing/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java @@ -23,6 +23,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import com.google.common.primitives.Bytes; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; @@ -31,11 +32,14 @@ import io.druid.collections.spatial.search.RadiusBound; import io.druid.collections.spatial.search.RectangularBound; import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.ImmutableRTreeObjectStrategy; import junit.framework.Assert; import org.junit.Test; import org.roaringbitmap.IntIterator; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Locale; import java.util.Random; import java.util.Set; @@ -648,4 +652,24 @@ public void showBenchmarksBoundWithLimits() } } } + + @Test + public void testToBytes() + { + BitmapFactory bf = new RoaringBitmapFactory(); + ImmutableRTreeObjectStrategy rTreeObjectStrategy = new ImmutableRTreeObjectStrategy(bf); + RTree rTree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + rTree.insert(new float[]{0, 0}, 1); + ImmutableRTree immutableRTree = ImmutableRTree.newImmutableFromMutable(rTree); + byte[] bytes1 = immutableRTree.toBytes(); + + GenericIndexed genericIndexed = GenericIndexed.fromIterable( + Arrays.asList(immutableRTree, immutableRTree), + rTreeObjectStrategy + ); + + ImmutableRTree deserializedTree = genericIndexed.get(0); + byte[] bytes2 = deserializedTree.toBytes(); + org.junit.Assert.assertEquals(Bytes.asList(bytes1), Bytes.asList(bytes2)); + } } diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/RTreeTest.java b/processing/src/test/java/io/druid/collections/spatial/RTreeTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/RTreeTest.java rename to processing/src/test/java/io/druid/collections/spatial/RTreeTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java b/processing/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java rename to processing/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java b/processing/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java rename to processing/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java b/processing/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java rename to processing/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java b/processing/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java rename to processing/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Benchmark.java b/processing/src/test/java/io/druid/collections/test/annotation/Benchmark.java old mode 100755 new mode 100644 similarity index 94% rename from bytebuffer-collections/src/test/java/io/druid/test/annotation/Benchmark.java rename to processing/src/test/java/io/druid/collections/test/annotation/Benchmark.java index 2fa616eef401..99aa5517a48c --- a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Benchmark.java +++ b/processing/src/test/java/io/druid/collections/test/annotation/Benchmark.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.test.annotation; +package io.druid.collections.test.annotation; public interface Benchmark { diff --git a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Dummy.java b/processing/src/test/java/io/druid/collections/test/annotation/Dummy.java old mode 100755 new mode 100644 similarity index 94% rename from bytebuffer-collections/src/test/java/io/druid/test/annotation/Dummy.java rename to processing/src/test/java/io/druid/collections/test/annotation/Dummy.java index ae1d36869320..75b0fa2925c8 --- a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Dummy.java +++ b/processing/src/test/java/io/druid/collections/test/annotation/Dummy.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.test.annotation; +package io.druid.collections.test.annotation; public interface Dummy { diff --git a/server/src/test/java/io/druid/client/cache/CacheConfigTest.java b/server/src/test/java/io/druid/client/cache/CacheConfigTest.java index 096887d81b52..7643a618405a 100644 --- a/server/src/test/java/io/druid/client/cache/CacheConfigTest.java +++ b/server/src/test/java/io/druid/client/cache/CacheConfigTest.java @@ -45,7 +45,7 @@ public class CacheConfigTest static Injector injector; static JsonConfigurator configurator; JsonConfigProvider configProvider; - private static final String propertyPrefix = "io.druid.test.cache"; + private static final String propertyPrefix = "io.druid.collections.test.cache"; @BeforeClass public static void populateStatics()