From 42c080ba8aad9b1085f491061012aaf95c6fb29e Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 20 Feb 2017 13:44:52 -0600 Subject: [PATCH 01/30] Monomorphic processing of topN queries with simple double aggregators and historical segments --- .../DoubleMaxBufferAggregator.java | 16 +- .../DoubleMinBufferAggregator.java | 16 +- .../DoubleSumBufferAggregator.java | 12 +- ...java => SimpleDoubleBufferAggregator.java} | 30 +- .../RowBasedColumnSelectorFactory.java | 9 +- .../SpecializationService.java | 2 +- .../topn/Historical1AggPooledTopNScanner.java | 48 ++ ...leDoubleAggPooledTopNScannerPrototype.java | 74 +++ ...leDoubleAggPooledTopNScannerPrototype.java | 68 +++ .../druid/query/topn/PooledTopNAlgorithm.java | 91 +++- .../java/io/druid/query/topn/TopNUtils.java | 45 ++ .../ColumnSelectorBitmapIndexSelector.java | 7 + .../CompressedVSizeIndexedSupplier.java | 13 + .../java/io/druid/segment/FilteredOffset.java | 197 ++++++++ .../java/io/druid/segment/IndexMerger.java | 45 -- .../druid/segment/NullDimensionSelector.java | 21 +- .../QueryableIndexIndexableAdapter.java | 7 + .../segment/QueryableIndexStorageAdapter.java | 455 +++--------------- .../segment/SingleScanTimeDimSelector.java | 8 +- .../segment/SingleValueDimensionSelector.java | 31 ++ .../druid/segment/StringDimensionIndexer.java | 5 + .../segment/StringDimensionMergerV9.java | 5 +- .../segment/ZeroFloatColumnSelector.java | 9 +- .../column/DictionaryEncodedColumn.java | 5 + .../column/SimpleDictionaryEncodedColumn.java | 180 +++++++ .../segment/data/ArrayBasedIndexedInts.java | 5 + .../io/druid/segment/data/ArrayIndexed.java | 7 + .../data/BitmapCompressedIndexedInts.java | 6 + .../io/druid/segment/data/CachingIndexed.java | 9 +- .../data/CompressedIntsIndexedSupplier.java | 9 + .../CompressedVSizeIntsIndexedSupplier.java | 31 +- .../druid/segment/data/EmptyIndexedInts.java | 5 + .../io/druid/segment/data/GenericIndexed.java | 17 + .../java/io/druid/segment/data/Indexed.java | 6 +- .../io/druid/segment/data/IndexedInts.java | 6 +- .../druid/segment/data/IndexedMultivalue.java | 2 + .../segment/data/IntBufferIndexedInts.java | 138 ------ .../io/druid/segment/data/ListIndexed.java | 8 + .../druid/segment/data/RangeIndexedInts.java | 5 + .../druid/segment/data/SingleIndexedInt.java | 6 + .../io/druid/segment/data/VSizeIndexed.java | 7 + .../druid/segment/data/VSizeIndexedInts.java | 6 + .../druid/segment/data/ZeroIndexedInts.java | 5 + .../segment/historical/HistoricalCursor.java | 26 + .../HistoricalDimensionSelector.java} | 45 +- .../HistoricalFloatColumnSelector.java | 27 ++ .../segment/historical/OffsetHolder.java | 27 ++ ...ingleValueHistoricalDimensionSelector.java | 28 ++ .../incremental/IncrementalIndexAdapter.java | 7 + .../BaseSingleValueDimensionSelector.java | 9 +- .../CardinalityAggregatorTest.java | 4 + .../druid/segment/data/IndexedIntsTest.java | 2 +- .../segment/virtual/VirtualColumnsTest.java | 37 +- 53 files changed, 1197 insertions(+), 692 deletions(-) rename processing/src/main/java/io/druid/query/aggregation/{DoubleBufferAggregator.java => SimpleDoubleBufferAggregator.java} (64%) create mode 100644 processing/src/main/java/io/druid/query/topn/Historical1AggPooledTopNScanner.java create mode 100644 processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java create mode 100644 processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java create mode 100644 processing/src/main/java/io/druid/query/topn/TopNUtils.java create mode 100644 processing/src/main/java/io/druid/segment/FilteredOffset.java create mode 100644 processing/src/main/java/io/druid/segment/SingleValueDimensionSelector.java delete mode 100644 processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java create mode 100644 processing/src/main/java/io/druid/segment/historical/HistoricalCursor.java rename processing/src/main/java/io/druid/segment/{data/ListBasedIndexedInts.java => historical/HistoricalDimensionSelector.java} (52%) create mode 100644 processing/src/main/java/io/druid/segment/historical/HistoricalFloatColumnSelector.java create mode 100644 processing/src/main/java/io/druid/segment/historical/OffsetHolder.java create mode 100644 processing/src/main/java/io/druid/segment/historical/SingleValueHistoricalDimensionSelector.java diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxBufferAggregator.java index 7a08e5a0434a..ed12dce4e304 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxBufferAggregator.java @@ -25,7 +25,7 @@ /** */ -public class DoubleMaxBufferAggregator extends DoubleBufferAggregator +public class DoubleMaxBufferAggregator extends SimpleDoubleBufferAggregator { DoubleMaxBufferAggregator(FloatColumnSelector selector) @@ -40,8 +40,18 @@ public void init(ByteBuffer buf, int position) } @Override - public void aggregate(ByteBuffer buf, int position) + public void putFirst(ByteBuffer buf, int position, double value) { - buf.putDouble(position, Math.max(buf.getDouble(position), (double) selector.get())); + if (!Double.isNaN(value)) { + buf.putDouble(position, value); + } else { + init(buf, position); + } + } + + @Override + public void aggregate(ByteBuffer buf, int position, double value) + { + buf.putDouble(position, Math.max(buf.getDouble(position), value)); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinBufferAggregator.java index b11712818d29..427d3290b066 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinBufferAggregator.java @@ -25,7 +25,7 @@ /** */ -public class DoubleMinBufferAggregator extends DoubleBufferAggregator +public class DoubleMinBufferAggregator extends SimpleDoubleBufferAggregator { DoubleMinBufferAggregator(FloatColumnSelector selector) @@ -40,8 +40,18 @@ public void init(ByteBuffer buf, int position) } @Override - public void aggregate(ByteBuffer buf, int position) + public void putFirst(ByteBuffer buf, int position, double value) { - buf.putDouble(position, Math.min(buf.getDouble(position), (double) selector.get())); + if (!Double.isNaN(value)) { + buf.putDouble(position, value); + } else { + init(buf, position); + } + } + + @Override + public void aggregate(ByteBuffer buf, int position, double value) + { + buf.putDouble(position, Math.min(buf.getDouble(position), value)); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java index 2c065ce49a7f..600fa646ee23 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java @@ -25,7 +25,7 @@ /** */ -public class DoubleSumBufferAggregator extends DoubleBufferAggregator +public class DoubleSumBufferAggregator extends SimpleDoubleBufferAggregator { DoubleSumBufferAggregator(FloatColumnSelector selector) @@ -40,8 +40,14 @@ public void init(ByteBuffer buf, int position) } @Override - public void aggregate(ByteBuffer buf, int position) + public void putFirst(ByteBuffer buf, int position, double value) { - buf.putDouble(position, buf.getDouble(position) + (double) selector.get()); + buf.putDouble(position, value); + } + + @Override + public void aggregate(ByteBuffer buf, int position, double value) + { + buf.putDouble(position, buf.getDouble(position) + value); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleBufferAggregator.java similarity index 64% rename from processing/src/main/java/io/druid/query/aggregation/DoubleBufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/SimpleDoubleBufferAggregator.java index 0e868674816c..a8fb46c7c7dd 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleBufferAggregator.java @@ -24,29 +24,49 @@ import java.nio.ByteBuffer; -public abstract class DoubleBufferAggregator implements BufferAggregator +public abstract class SimpleDoubleBufferAggregator implements BufferAggregator { protected final FloatColumnSelector selector; - DoubleBufferAggregator(FloatColumnSelector selector) + SimpleDoubleBufferAggregator(FloatColumnSelector selector) { this.selector = selector; } + public FloatColumnSelector getSelector() + { + return selector; + } + + /** + * Faster equivalent to + * aggregator.init(buf, position); + * aggregator.aggregate(buf, position, value); + */ + public abstract void putFirst(ByteBuffer buf, int position, double value); + + public abstract void aggregate(ByteBuffer buf, int position, double value); + + @Override + public final void aggregate(ByteBuffer buf, int position) + { + aggregate(buf, position, (double) selector.get()); + } + @Override - public Object get(ByteBuffer buf, int position) + public final Object get(ByteBuffer buf, int position) { return buf.getDouble(position); } @Override - public float getFloat(ByteBuffer buf, int position) + public final float getFloat(ByteBuffer buf, int position) { return (float) buf.getDouble(position); } @Override - public long getLong(ByteBuffer buf, int position) + public final long getLong(ByteBuffer buf, int position) { return (long) buf.getDouble(position); } diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java index d8d15070a8bd..4148a29e7a14 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java @@ -34,6 +34,7 @@ import io.druid.segment.IdLookup; import io.druid.segment.LongColumnSelector; import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.SingleValueDimensionSelector; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; @@ -105,7 +106,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi throw new UnsupportedOperationException("time dimension must provide an extraction function"); } - return new DimensionSelector() + return new SingleValueDimensionSelector() { @Override public IndexedInts getRow() @@ -113,6 +114,12 @@ public IndexedInts getRow() return ZeroIndexedInts.instance(); } + @Override + public int getRowValue() + { + return 0; + } + @Override public ValueMatcher makeValueMatcher(final String value) { diff --git a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java index e2d00b05031f..2eb625459d20 100644 --- a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java +++ b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java @@ -124,7 +124,7 @@ public static SpecializationState getSpecializationState( String runtimeShape ) { - return getSpecializationState(prototypeClass, runtimeShape, ImmutableMap., Class>of()); + return getSpecializationState(prototypeClass, runtimeShape, ImmutableMap.of()); } /** diff --git a/processing/src/main/java/io/druid/query/topn/Historical1AggPooledTopNScanner.java b/processing/src/main/java/io/druid/query/topn/Historical1AggPooledTopNScanner.java new file mode 100644 index 000000000000..fc1691a10786 --- /dev/null +++ b/processing/src/main/java/io/druid/query/topn/Historical1AggPooledTopNScanner.java @@ -0,0 +1,48 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.topn; + +import io.druid.query.aggregation.BufferAggregator; +import io.druid.segment.ColumnValueSelector; +import io.druid.segment.historical.HistoricalCursor; +import io.druid.segment.historical.HistoricalDimensionSelector; + +import java.nio.ByteBuffer; + +public interface Historical1AggPooledTopNScanner< + DimensionSelectorType extends HistoricalDimensionSelector, + MetricSelectorType extends ColumnValueSelector, + BufferAggregatorType extends BufferAggregator> +{ + /** + * @param aggregatorSize number of bytes required by aggregator for a single aggregation + * @param positions a cache for positions in resultsBuffer, where specific (indexed) dimension values are aggregated + * @return number of scanned rows, i. e. number of steps made with the given cursor + */ + long scanAndAggregate( + DimensionSelectorType dimensionSelector, + MetricSelectorType metricSelector, + BufferAggregatorType aggregator, + int aggregatorSize, + HistoricalCursor cursor, + int[] positions, + ByteBuffer resultsBuffer + ); +} diff --git a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java new file mode 100644 index 000000000000..81cb86cc0a81 --- /dev/null +++ b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -0,0 +1,74 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.topn; + +import io.druid.query.aggregation.SimpleDoubleBufferAggregator; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.Offset; +import io.druid.segment.historical.HistoricalCursor; +import io.druid.segment.historical.HistoricalDimensionSelector; +import io.druid.segment.historical.HistoricalFloatColumnSelector; + +import java.nio.ByteBuffer; + +public class Historical1SimpleDoubleAggPooledTopNScannerPrototype + implements Historical1AggPooledTopNScanner< + HistoricalDimensionSelector, + HistoricalFloatColumnSelector, + SimpleDoubleBufferAggregator + > +{ + @Override + public long scanAndAggregate( + HistoricalDimensionSelector dimensionSelector, + HistoricalFloatColumnSelector metricSelector, + SimpleDoubleBufferAggregator aggregator, + int aggregatorSize, + HistoricalCursor cursor, + int[] positions, + ByteBuffer resultsBuffer + ) + { + // See TopNUtils.copyOffset() for explanation + Offset offset = (Offset) TopNUtils.copyOffset(cursor); + long scannedRows = 0; + int positionToAllocate = 0; + while (offset.withinBounds() && !Thread.currentThread().isInterrupted()) { + int rowNum = offset.getOffset(); + double metric = metricSelector.get(rowNum); + final IndexedInts dimValues = dimensionSelector.getRow(rowNum); + final int dimSize = dimValues.size(); + for (int i = 0; i < dimSize; i++) { + int dimIndex = dimValues.get(i); + int position = positions[dimIndex]; + if (position >= 0) { + aggregator.aggregate(resultsBuffer, position, metric); + } else if (position == TopNAlgorithm.INIT_POSITION_VALUE) { + positions[dimIndex] = positionToAllocate; + aggregator.putFirst(resultsBuffer, position, metric); + positionToAllocate += aggregatorSize; + } + } + scannedRows++; + offset.increment(); + } + return scannedRows; + } +} diff --git a/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java new file mode 100644 index 000000000000..91f77c4a8091 --- /dev/null +++ b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -0,0 +1,68 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.topn; + +import io.druid.query.aggregation.SimpleDoubleBufferAggregator; +import io.druid.segment.data.Offset; +import io.druid.segment.historical.HistoricalCursor; +import io.druid.segment.historical.HistoricalFloatColumnSelector; +import io.druid.segment.historical.SingleValueHistoricalDimensionSelector; + +import java.nio.ByteBuffer; + +public class HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype + implements Historical1AggPooledTopNScanner< + SingleValueHistoricalDimensionSelector, + HistoricalFloatColumnSelector, + SimpleDoubleBufferAggregator + > +{ + @Override + public long scanAndAggregate( + SingleValueHistoricalDimensionSelector dimensionSelector, + HistoricalFloatColumnSelector metricSelector, + SimpleDoubleBufferAggregator aggregator, + int aggregatorSize, + HistoricalCursor cursor, + int[] positions, + ByteBuffer resultsBuffer + ) + { + // See TopNUtils.copyOffset() for explanation + Offset offset = (Offset) TopNUtils.copyOffset(cursor); + long scannedRows = 0; + int positionToAllocate = 0; + while (offset.withinBounds() && !Thread.currentThread().isInterrupted()) { + int rowNum = offset.getOffset(); + int dimIndex = dimensionSelector.getRowValue(rowNum); + int position = positions[dimIndex]; + if (position >= 0) { + aggregator.aggregate(resultsBuffer, position, metricSelector.get(rowNum)); + } else if (position == TopNAlgorithm.INIT_POSITION_VALUE) { + positions[dimIndex] = positionToAllocate; + aggregator.putFirst(resultsBuffer, positionToAllocate, metricSelector.get(rowNum)); + positionToAllocate += aggregatorSize; + } + scannedRows++; + offset.increment(); + } + return scannedRows; + } +} diff --git a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java index b14cfd8fb7f9..c81b0fb410eb 100644 --- a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; +import com.google.common.collect.ImmutableMap; import io.druid.collections.ResourceHolder; import io.druid.collections.StupidPool; import io.druid.java.util.common.Pair; @@ -28,6 +29,7 @@ import io.druid.query.BaseQuery; import io.druid.query.ColumnSelectorPlus; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.SimpleDoubleBufferAggregator; import io.druid.query.monomorphicprocessing.SpecializationService; import io.druid.query.monomorphicprocessing.SpecializationState; import io.druid.query.monomorphicprocessing.StringRuntimeShape; @@ -36,6 +38,11 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.Offset; +import io.druid.segment.historical.HistoricalCursor; +import io.druid.segment.historical.HistoricalDimensionSelector; +import io.druid.segment.historical.HistoricalFloatColumnSelector; +import io.druid.segment.historical.SingleValueHistoricalDimensionSelector; import java.nio.ByteBuffer; import java.util.Arrays; @@ -52,11 +59,20 @@ public class PooledTopNAlgorithm @VisibleForTesting static boolean specializeGeneric2AggPooledTopN = !Boolean.getBoolean("dontSpecializeGeneric2AggPooledTopN"); + private static final boolean specializeHistorical1SimpleDoubleAggPooledTopN = + !Boolean.getBoolean("dontSpecializeHistorical1SimpleDoubleAggPooledTopN"); + private static final boolean specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN = + !Boolean.getBoolean("dontSpecializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN"); private static final Generic1AggPooledTopNScanner defaultGeneric1AggScanner = new Generic1AggPooledTopNScannerPrototype(); private static final Generic2AggPooledTopNScanner defaultGeneric2AggScanner = new Generic2AggPooledTopNScannerPrototype(); + private static final Historical1AggPooledTopNScanner defaultHistorical1SimpleDoubleAggScanner = + new Historical1SimpleDoubleAggPooledTopNScannerPrototype(); + private static final + Historical1AggPooledTopNScanner defaultHistoricalSingleValueDimSelector1SimpleDoubleAggScanner = + new HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype(); private final Capabilities capabilities; private final TopNQuery query; @@ -194,16 +210,81 @@ protected void scanAndAggregate( ) { final Cursor cursor = params.getCursor(); - if (specializeGeneric1AggPooledTopN && theAggregators.length == 1) { - scanAndAggregateGeneric1Agg(params, positions, theAggregators[0], cursor); - } else if (specializeGeneric2AggPooledTopN && theAggregators.length == 2) { + if (theAggregators.length == 1) { + BufferAggregator aggregator = theAggregators[0]; + if (cursor instanceof HistoricalCursor && aggregator instanceof SimpleDoubleBufferAggregator) { + if (specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN && + params.getDimSelector() instanceof SingleValueHistoricalDimensionSelector && + ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalFloatColumnSelector) { + scanAndAggregateHistorical1SimpleDoubleAgg( + params, + positions, + (SimpleDoubleBufferAggregator) aggregator, + cursor, + defaultHistoricalSingleValueDimSelector1SimpleDoubleAggScanner + ); + BaseQuery.checkInterrupted(); + return; + } + if (specializeHistorical1SimpleDoubleAggPooledTopN && + params.getDimSelector() instanceof HistoricalDimensionSelector && + ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalFloatColumnSelector) { + scanAndAggregateHistorical1SimpleDoubleAgg( + params, + positions, + (SimpleDoubleBufferAggregator) aggregator, + cursor, + defaultHistorical1SimpleDoubleAggScanner + ); + BaseQuery.checkInterrupted(); + return; + } + } + if (specializeGeneric1AggPooledTopN) { + scanAndAggregateGeneric1Agg(params, positions, aggregator, cursor); + BaseQuery.checkInterrupted(); + return; + } + } + if (specializeGeneric2AggPooledTopN && theAggregators.length == 2) { scanAndAggregateGeneric2Agg(params, positions, theAggregators, cursor); - } else { - scanAndAggregateDefault(params, positions, theAggregators); + BaseQuery.checkInterrupted(); + return; } + scanAndAggregateDefault(params, positions, theAggregators); BaseQuery.checkInterrupted(); } + private static void scanAndAggregateHistorical1SimpleDoubleAgg( + PooledTopNParams params, + int[] positions, + SimpleDoubleBufferAggregator aggregator, + Cursor cursor, + Historical1AggPooledTopNScanner prototypeScanner + ) + { + String runtimeShape = StringRuntimeShape.of(aggregator); + HistoricalCursor historicalCursor = (HistoricalCursor) cursor; + SpecializationState specializationState = + SpecializationService.getSpecializationState( + prototypeScanner.getClass(), + runtimeShape, + ImmutableMap., Class>of(Offset.class, historicalCursor.getOffset().getClass()) + ); + Historical1AggPooledTopNScanner scanner = specializationState.getSpecializedOrDefault(prototypeScanner); + + long scannedRows = scanner.scanAndAggregate( + (HistoricalDimensionSelector) params.getDimSelector(), + aggregator.getSelector(), + aggregator, + params.getAggregatorSizes()[0], + historicalCursor, + positions, + params.getResultsBuf() + ); + specializationState.accountLoopIterations(scannedRows); + } + private static void scanAndAggregateGeneric1Agg( PooledTopNParams params, int[] positions, diff --git a/processing/src/main/java/io/druid/query/topn/TopNUtils.java b/processing/src/main/java/io/druid/query/topn/TopNUtils.java new file mode 100644 index 000000000000..08afbcb3afdc --- /dev/null +++ b/processing/src/main/java/io/druid/query/topn/TopNUtils.java @@ -0,0 +1,45 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.topn; + +import io.druid.segment.historical.HistoricalCursor; + +final class TopNUtils +{ + /** + * Returns Object, so javac couldn't remove cast in methods like + * {@link HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype#scanAndAggregate}. That cast is + * needed, because when TopNScannerPrototype is specialized, occurrences of {@link io.druid.segment.data.Offset} are + * replaced with the specific Offset subtype in the TopNScannerPrototype bytecode, via {@link + * io.druid.query.monomorphicprocessing.SpecializationService#getSpecializationState(Class, String, + * com.google.common.collect.ImmutableMap)}, providing ImmutableMap.of(Offset.class, specificOffsetSubtype) as the + * classRemapping argument. + * + * Casting to the specific Offset subtype helps Hotspot JIT (OpenJDK 8) to generate better assembly. It shouldn't be + * so, because the Offset subtype is still always the same (otherwise cast wouldn't be possible), so JIT should + * generate equivalent code. In OpenJDK 9 Hotspot could be improved and this "casting hack" is not needed anymore. + */ + static Object copyOffset(HistoricalCursor cursor) + { + return cursor.getOffset().clone(); + } + + private TopNUtils() {} +} diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index 23cfabf143a1..e11d4435c910 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -24,6 +24,7 @@ import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.spatial.ImmutableRTree; import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; @@ -99,6 +100,12 @@ public Iterator iterator() { return IndexedIterable.create(this).iterator(); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("column", column); + } }; } diff --git a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java index e2a47e36eb5c..18429e89d80e 100644 --- a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java @@ -21,6 +21,7 @@ import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier; import io.druid.segment.data.IndexedInts; @@ -214,6 +215,12 @@ public IntIterator iterator() { return new IndexedIntsIterator(this); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("values", values); + } }; } @@ -229,6 +236,12 @@ public Iterator iterator() return IndexedIterable.create(this).iterator(); } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("offsets", offsets); + inspector.visit("values", values); + } } } diff --git a/processing/src/main/java/io/druid/segment/FilteredOffset.java b/processing/src/main/java/io/druid/segment/FilteredOffset.java new file mode 100644 index 000000000000..49821fe9987b --- /dev/null +++ b/processing/src/main/java/io/druid/segment/FilteredOffset.java @@ -0,0 +1,197 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.query.BaseQuery; +import io.druid.query.filter.BooleanFilter; +import io.druid.query.filter.Filter; +import io.druid.query.filter.RowOffsetMatcherFactory; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.data.Offset; +import io.druid.segment.filter.BooleanValueMatcher; +import io.druid.segment.historical.HistoricalCursor; +import io.druid.segment.historical.OffsetHolder; +import org.roaringbitmap.IntIterator; + +final class FilteredOffset implements Offset +{ + private Offset baseOffset; + private final ValueMatcher filterMatcher; + + FilteredOffset( + HistoricalCursor cursor, + boolean descending, + Filter postFilter, + ColumnSelectorBitmapIndexSelector bitmapIndexSelector + ) + { + RowOffsetMatcherFactory rowOffsetMatcherFactory = new CursorOffsetHolderRowOffsetMatcherFactory( + cursor, + descending + ); + if (postFilter instanceof BooleanFilter) { + filterMatcher = ((BooleanFilter) postFilter).makeMatcher( + bitmapIndexSelector, + cursor, + rowOffsetMatcherFactory + ); + } else { + if (postFilter.supportsBitmapIndex(bitmapIndexSelector)) { + filterMatcher = rowOffsetMatcherFactory.makeRowOffsetMatcher( + postFilter.getBitmapIndex(bitmapIndexSelector) + ); + } else { + filterMatcher = postFilter.makeMatcher(cursor); + } + } + } + + void reset(Offset baseOffset) + { + this.baseOffset = baseOffset; + if (baseOffset.withinBounds()) { + if (!filterMatcher.matches()) { + BaseQuery.checkInterrupted(); + incrementInterruptibly(); + } + } + } + + @Override + public void increment() + { + baseOffset.increment(); + + while (baseOffset.withinBounds() && !Thread.currentThread().isInterrupted()) { + if (filterMatcher.matches()) { + return; + } else { + baseOffset.increment(); + } + } + } + + void incrementInterruptibly() + { + baseOffset.increment(); + while (baseOffset.withinBounds()) { + BaseQuery.checkInterrupted(); + if (filterMatcher.matches()) { + return; + } else { + baseOffset.increment(); + } + } + } + + @Override + public boolean withinBounds() + { + return baseOffset.withinBounds(); + } + + @Override + public Offset clone() + { + try { + FilteredOffset offset = (FilteredOffset) super.clone(); + offset.baseOffset = offset.baseOffset.clone(); + return offset; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + @Override + public int getOffset() + { + return baseOffset.getOffset(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("baseOffset", baseOffset); + inspector.visit("filterMatcher", filterMatcher); + } + + private static class CursorOffsetHolderRowOffsetMatcherFactory implements RowOffsetMatcherFactory + { + private final OffsetHolder holder; + private final boolean descending; + + CursorOffsetHolderRowOffsetMatcherFactory(OffsetHolder holder, boolean descending) + { + this.holder = holder; + this.descending = descending; + } + + // Use an iterator-based implementation, ImmutableBitmap.get(index) works differently for Concise and Roaring. + // ImmutableConciseSet.get(index) is also inefficient, it performs a linear scan on each call + @Override + public ValueMatcher makeRowOffsetMatcher(final ImmutableBitmap rowBitmap) + { + final IntIterator iter = descending ? + BitmapOffset.getReverseBitmapOffsetIterator(rowBitmap) : + rowBitmap.iterator(); + + if (!iter.hasNext()) { + return BooleanValueMatcher.of(false); + } + + if (descending) { + return new ValueMatcher() + { + int iterOffset = Integer.MAX_VALUE; + + @Override + public boolean matches() + { + int currentOffset = holder.getOffset().getOffset(); + while (iterOffset > currentOffset && iter.hasNext()) { + iterOffset = iter.next(); + } + + return iterOffset == currentOffset; + } + }; + } else { + return new ValueMatcher() + { + int iterOffset = -1; + + @Override + public boolean matches() + { + int currentOffset = holder.getOffset().getOffset(); + while (iterOffset < currentOffset && iter.hasNext()) { + iterOffset = iter.next(); + } + + return iterOffset == currentOffset; + } + }; + } + } + } +} diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index f7fbb49727dd..dd66948d6e29 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -63,8 +62,6 @@ import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.IOPeon; import io.druid.segment.data.Indexed; -import io.druid.segment.data.IndexedIterable; -import io.druid.segment.data.ListIndexed; import io.druid.segment.data.LongSupplierSerializer; import io.druid.segment.data.TmpFileIOPeon; import io.druid.segment.incremental.IncrementalIndex; @@ -101,10 +98,8 @@ public class IndexMerger { private static final Logger log = new Logger(IndexMerger.class); - protected static final ListIndexed EMPTY_STR_DIM_VAL = new ListIndexed<>(Arrays.asList(""), String.class); protected static final SerializerUtils serializerUtils = new SerializerUtils(); protected static final int INVALID_ROW = -1; - protected static final Splitter SPLITTER = Splitter.on(","); protected final ObjectMapper mapper; protected final IndexIO indexIO; @@ -1176,46 +1171,6 @@ public Rowboat apply(@Nullable Rowboat input) } } - public static class AggFactoryStringIndexed implements Indexed - { - private final AggregatorFactory[] metricAggs; - - public AggFactoryStringIndexed(AggregatorFactory[] metricAggs) - { - this.metricAggs = metricAggs; - } - - @Override - public Class getClazz() - { - return String.class; - } - - @Override - public int size() - { - return metricAggs.length; - } - - @Override - public String get(int index) - { - return metricAggs[index].getName(); - } - - @Override - public int indexOf(String value) - { - throw new UnsupportedOperationException(); - } - - @Override - public Iterator iterator() - { - return IndexedIterable.create(this).iterator(); - } - } - public static class RowboatMergeFunction implements BinaryFn { private final AggregatorFactory[] metricAggs; diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index 385bbad0c7d6..9eebce5dbf07 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -26,10 +26,11 @@ import io.druid.segment.data.IndexedInts; import io.druid.segment.data.ZeroIndexedInts; import io.druid.segment.filter.BooleanValueMatcher; +import io.druid.segment.historical.SingleValueHistoricalDimensionSelector; import javax.annotation.Nullable; -public class NullDimensionSelector implements DimensionSelector, IdLookup +public class NullDimensionSelector implements SingleValueHistoricalDimensionSelector, IdLookup { private static final NullDimensionSelector INSTANCE = new NullDimensionSelector(); @@ -49,6 +50,24 @@ public IndexedInts getRow() return ZeroIndexedInts.instance(); } + @Override + public int getRowValue() + { + return 0; + } + + @Override + public int getRowValue(int offset) + { + return 0; + } + + @Override + public IndexedInts getRow(int offset) + { + return getRow(); + } + @Override public ValueMatcher makeValueMatcher(String value) { diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java index 6532b695a42a..dab6c7a627a9 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java @@ -28,6 +28,7 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.logger.Logger; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; @@ -165,6 +166,12 @@ public Iterator iterator() { return IndexedIterable.create(this).iterator(); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("dict", dict); + } }; } diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index ffa6d527fe48..9a44b68ad71d 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -20,24 +20,18 @@ package io.druid.segment; import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Closer; -import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.java.util.common.granularity.Granularity; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.query.BaseQuery; import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; -import io.druid.query.filter.BooleanFilter; import io.druid.query.filter.Filter; -import io.druid.query.filter.RowOffsetMatcherFactory; -import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; @@ -50,19 +44,16 @@ import io.druid.segment.data.IndexedInts; import io.druid.segment.data.Offset; import io.druid.segment.filter.AndFilter; -import io.druid.segment.filter.BooleanValueMatcher; -import it.unimi.dsi.fastutil.ints.IntIterators; +import io.druid.segment.historical.HistoricalCursor; +import io.druid.segment.historical.HistoricalFloatColumnSelector; import org.joda.time.DateTime; import org.joda.time.Interval; -import org.roaringbitmap.IntIterator; -import javax.annotation.Nullable; import java.io.Closeable; -import java.io.IOException; import java.util.ArrayList; -import java.util.BitSet; import java.util.List; import java.util.Map; +import java.util.Objects; /** */ @@ -305,7 +296,7 @@ public Sequence makeCursors( postFilter, selector ).build(), - Predicates.notNull() + Objects::nonNull ); } @@ -320,7 +311,6 @@ private static ColumnCapabilities getColumnCapabilites(ColumnSelector index, Str private static class CursorSequenceBuilder { - private final StorageAdapter storageAdapter; private final QueryableIndex index; private final Interval interval; private final VirtualColumns virtualColumns; @@ -345,7 +335,6 @@ public CursorSequenceBuilder( ColumnSelectorBitmapIndexSelector bitmapIndexSelector ) { - this.storageAdapter = storageAdapter; this.index = storageAdapter.index; this.interval = interval; this.virtualColumns = virtualColumns; @@ -418,11 +407,44 @@ public Cursor apply(final Interval inputInterval) final Offset initOffset = offset.clone(); final DateTime myBucket = gran.toDateTime(inputInterval.getStartMillis()); - final CursorOffsetHolder cursorOffsetHolder = new CursorOffsetHolder(); - abstract class QueryableIndexBaseCursor implements Cursor + abstract class QueryableIndexBaseCursor implements HistoricalCursor { - Offset cursorOffset; + OffsetType cursorOffset; + + @Override + public OffsetType getOffset() + { + return cursorOffset; + } + + @Override + public DateTime getTime() + { + return myBucket; + } + + @Override + public void advanceTo(int offset) + { + int count = 0; + while (count < offset && !isDone()) { + advance(); + count++; + } + } + + @Override + public boolean isDone() + { + return !cursorOffset.withinBounds(); + } + + @Override + public boolean isDoneOrInterrupted() + { + return isDone() || Thread.currentThread().isInterrupted(); + } @Override public DimensionSelector makeDimensionSelector( @@ -472,204 +494,13 @@ private DimensionSelector makeDimensionSelectorUndecorated( } final DictionaryEncodedColumn column = cachedColumn; - - abstract class QueryableDimensionSelector implements DimensionSelector, IdLookup - { - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("column", column); - inspector.visit("cursorOffset", cursorOffset); - inspector.visit("extractionFn", extractionFn); - } - } if (column == null) { return NullDimensionSelector.instance(); - } else if (columnDesc.getCapabilities().hasMultipleValues()) { - class MultiValueDimensionSelector extends QueryableDimensionSelector - { - @Override - public IndexedInts getRow() - { - return column.getMultiValueRow(cursorOffset.getOffset()); - } - - @Override - public ValueMatcher makeValueMatcher(String value) - { - return DimensionSelectorUtils.makeValueMatcherGeneric(this, value); - } - - @Override - public ValueMatcher makeValueMatcher(Predicate predicate) - { - return DimensionSelectorUtils.makeValueMatcherGeneric(this, predicate); - } - - @Override - public int getValueCardinality() - { - return column.getCardinality(); - } - - @Override - public String lookupName(int id) - { - final String value = column.lookupName(id); - return extractionFn == null ? - value : - extractionFn.apply(value); - } - - @Override - public boolean nameLookupPossibleInAdvance() - { - return true; - } - - @Nullable - @Override - public IdLookup idLookup() - { - return extractionFn == null ? this : null; - } - - @Override - public int lookupId(String name) - { - if (extractionFn != null) { - throw new UnsupportedOperationException( - "cannot perform lookup when applying an extraction function" - ); - } - return column.lookupId(name); - } - } - return new MultiValueDimensionSelector(); } else { - class SingleValueDimensionSelector extends QueryableDimensionSelector - { - @Override - public IndexedInts getRow() - { - // using an anonymous class is faster than creating a class that stores a copy of the value - return new IndexedInts() - { - @Override - public int size() - { - return 1; - } - - @Override - public int get(int index) - { - return column.getSingleValueRow(cursorOffset.getOffset()); - } - - @Override - public it.unimi.dsi.fastutil.ints.IntIterator iterator() - { - return IntIterators.singleton(column.getSingleValueRow(cursorOffset.getOffset())); - } - - @Override - public void fill(int index, int[] toFill) - { - throw new UnsupportedOperationException("fill not supported"); - } - - @Override - public void close() throws IOException - { - - } - }; - } - - @Override - public ValueMatcher makeValueMatcher(final String value) - { - if (extractionFn == null) { - final int valueId = lookupId(value); - if (valueId >= 0) { - return new ValueMatcher() - { - @Override - public boolean matches() - { - return column.getSingleValueRow(cursorOffset.getOffset()) == valueId; - } - }; - } else { - return BooleanValueMatcher.of(false); - } - } else { - // Employ precomputed BitSet optimization - return makeValueMatcher(Predicates.equalTo(value)); - } - } - - @Override - public ValueMatcher makeValueMatcher(final Predicate predicate) - { - final BitSet predicateMatchingValueIds = DimensionSelectorUtils.makePredicateMatchingSet( - this, - predicate - ); - return new ValueMatcher() - { - @Override - public boolean matches() - { - int rowValueId = column.getSingleValueRow(cursorOffset.getOffset()); - return predicateMatchingValueIds.get(rowValueId); - } - }; - } - - @Override - public int getValueCardinality() - { - return column.getCardinality(); - } - - @Override - public String lookupName(int id) - { - final String value = column.lookupName(id); - return extractionFn == null ? value : extractionFn.apply(value); - } - - @Override - public boolean nameLookupPossibleInAdvance() - { - return true; - } - - @Nullable - @Override - public IdLookup idLookup() - { - return extractionFn == null ? this : null; - } - - @Override - public int lookupId(String name) - { - if (extractionFn != null) { - throw new UnsupportedOperationException( - "cannot perform lookup when applying an extraction function" - ); - } - return column.lookupId(name); - } - } - return new SingleValueDimensionSelector(); + return column.makeDimensionSelector(this, extractionFn); } } - @Override public FloatColumnSelector makeFloatColumnSelector(String columnName) { @@ -694,7 +525,7 @@ public FloatColumnSelector makeFloatColumnSelector(String columnName) } final GenericColumn metricVals = cachedMetricVals; - return new FloatColumnSelector() + return new HistoricalFloatColumnSelector() { @Override public float get() @@ -702,6 +533,12 @@ public float get() return metricVals.getFloatSingleValueRow(cursorOffset.getOffset()); } + @Override + public float get(int offset) + { + return metricVals.getFloatSingleValueRow(offset); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -922,18 +759,12 @@ public ColumnCapabilities getColumnCapabilities(String columnName) } if (postFilter == null) { - return new QueryableIndexBaseCursor() + return new QueryableIndexBaseCursor() { { reset(); } - @Override - public DateTime getTime() - { - return myBucket; - } - @Override public void advance() { @@ -947,139 +778,39 @@ public void advanceUninterruptibly() cursorOffset.increment(); } - @Override - public void advanceTo(int offset) - { - int count = 0; - while (count < offset && !isDone()) { - advance(); - count++; - } - } - - @Override - public boolean isDone() - { - return !cursorOffset.withinBounds(); - } - - @Override - public boolean isDoneOrInterrupted() - { - return isDone() || Thread.currentThread().isInterrupted(); - } - @Override public void reset() { cursorOffset = initOffset.clone(); - cursorOffsetHolder.set(cursorOffset); } }; } else { - return new QueryableIndexBaseCursor() + return new QueryableIndexBaseCursor() { - RowOffsetMatcherFactory rowOffsetMatcherFactory = new CursorOffsetHolderRowOffsetMatcherFactory( - cursorOffsetHolder, - descending - ); - - final ValueMatcher filterMatcher; - - { - if (postFilter instanceof BooleanFilter) { - filterMatcher = ((BooleanFilter) postFilter).makeMatcher( - bitmapIndexSelector, - this, - rowOffsetMatcherFactory - ); - } else { - if (postFilter.supportsBitmapIndex(bitmapIndexSelector)) { - filterMatcher = rowOffsetMatcherFactory.makeRowOffsetMatcher(postFilter.getBitmapIndex( - bitmapIndexSelector)); - } else { - filterMatcher = postFilter.makeMatcher(this); - } - } - } - { + cursorOffset = new FilteredOffset(this, descending, postFilter, bitmapIndexSelector); reset(); } - @Override - public DateTime getTime() - { - return myBucket; - } - @Override public void advance() { BaseQuery.checkInterrupted(); - cursorOffset.increment(); - - while (!isDone()) { - BaseQuery.checkInterrupted(); - if (filterMatcher.matches()) { - return; - } else { - cursorOffset.increment(); - } - } + cursorOffset.incrementInterruptibly(); } @Override public void advanceUninterruptibly() { - if (Thread.currentThread().isInterrupted()) { - return; - } - cursorOffset.increment(); - - while (!isDoneOrInterrupted()) { - if (filterMatcher.matches()) { - return; - } else { - cursorOffset.increment(); - } + if (!Thread.currentThread().isInterrupted()) { + cursorOffset.increment(); } } - @Override - public void advanceTo(int offset) - { - int count = 0; - while (count < offset && !isDone()) { - advance(); - count++; - } - } - - @Override - public boolean isDone() - { - return !cursorOffset.withinBounds(); - } - - @Override - public boolean isDoneOrInterrupted() - { - return isDone() || Thread.currentThread().isInterrupted(); - } - @Override public void reset() { - cursorOffset = initOffset.clone(); - cursorOffsetHolder.set(cursorOffset); - if (!isDone()) { - if (filterMatcher.matches()) { - return; - } else { - advance(); - } - } + cursorOffset.reset(initOffset.clone()); } }; } @@ -1092,82 +823,6 @@ public void reset() } } - public static class CursorOffsetHolder - { - Offset currOffset = null; - - public Offset get() - { - return currOffset; - } - - public void set(Offset currOffset) - { - this.currOffset = currOffset; - } - } - - private static class CursorOffsetHolderRowOffsetMatcherFactory implements RowOffsetMatcherFactory - { - private final CursorOffsetHolder holder; - private final boolean descending; - - public CursorOffsetHolderRowOffsetMatcherFactory(CursorOffsetHolder holder, boolean descending) - { - this.holder = holder; - this.descending = descending; - } - - // Use an iterator-based implementation, ImmutableBitmap.get(index) works differently for Concise and Roaring. - // ImmutableConciseSet.get(index) is also inefficient, it performs a linear scan on each call - @Override - public ValueMatcher makeRowOffsetMatcher(final ImmutableBitmap rowBitmap) - { - final IntIterator iter = descending ? - BitmapOffset.getReverseBitmapOffsetIterator(rowBitmap) : - rowBitmap.iterator(); - - if (!iter.hasNext()) { - return BooleanValueMatcher.of(false); - } - - if (descending) { - return new ValueMatcher() - { - int iterOffset = Integer.MAX_VALUE; - - @Override - public boolean matches() - { - int currentOffset = holder.get().getOffset(); - while (iterOffset > currentOffset && iter.hasNext()) { - iterOffset = iter.next(); - } - - return iterOffset == currentOffset; - } - }; - } else { - return new ValueMatcher() - { - int iterOffset = -1; - - @Override - public boolean matches() - { - int currentOffset = holder.get().getOffset(); - while (iterOffset < currentOffset && iter.hasNext()) { - iterOffset = iter.next(); - } - - return iterOffset == currentOffset; - } - }; - } - } - } - - private abstract static class TimestampCheckingOffset implements Offset { protected final Offset baseOffset; diff --git a/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java b/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java index 9e4f296b2102..d0c03d684e3d 100644 --- a/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java +++ b/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java @@ -31,7 +31,7 @@ import java.util.List; import java.util.Objects; -public class SingleScanTimeDimSelector implements DimensionSelector +public class SingleScanTimeDimSelector implements SingleValueDimensionSelector { private final ExtractionFn extractionFn; private final LongColumnSelector selector; @@ -64,6 +64,12 @@ public IndexedInts getRow() return new SingleIndexedInt(getDimensionValueIndex()); } + @Override + public int getRowValue() + { + return getDimensionValueIndex(); + } + @Override public ValueMatcher makeValueMatcher(final String value) { diff --git a/processing/src/main/java/io/druid/segment/SingleValueDimensionSelector.java b/processing/src/main/java/io/druid/segment/SingleValueDimensionSelector.java new file mode 100644 index 000000000000..054787e65742 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/SingleValueDimensionSelector.java @@ -0,0 +1,31 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +/** + * Specialization for {@link DimensionSelector}s, always having a single value in {@link #getRow()}. + */ +public interface SingleValueDimensionSelector extends DimensionSelector +{ + /** + * Returns a single value of {@link #getRow()}. + */ + int getRowValue(); +} diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index f7cd8b622fc3..024ce50f1adb 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -326,6 +326,11 @@ public Iterator iterator() { return IndexedIterable.create(this).iterator(); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + } }; } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index eee670ac65d7..f3782e1cf11b 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -37,6 +37,7 @@ import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; +import io.druid.segment.data.ArrayIndexed; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; @@ -49,7 +50,6 @@ import io.druid.segment.data.IndexedInts; import io.druid.segment.data.IndexedIntsWriter; import io.druid.segment.data.IndexedRTree; -import io.druid.segment.data.ListIndexed; import io.druid.segment.data.VSizeIndexedIntsWriter; import io.druid.segment.data.VSizeIndexedWriter; import io.druid.segment.serde.DictionaryEncodedColumnPartSerde; @@ -64,14 +64,13 @@ import java.nio.IntBuffer; import java.nio.MappedByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; public class StringDimensionMergerV9 implements DimensionMergerV9 { private static final Logger log = new Logger(StringDimensionMergerV9.class); - protected static final ListIndexed EMPTY_STR_DIM_VAL = new ListIndexed<>(Arrays.asList(""), String.class); + protected static final Indexed EMPTY_STR_DIM_VAL = new ArrayIndexed<>(new String[]{""}, String.class); protected static final int[] EMPTY_STR_DIM_ARRAY = new int[]{0}; protected static final Splitter SPLITTER = Splitter.on(","); diff --git a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java index 295ecd23539d..6622c0dca34a 100644 --- a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java @@ -20,8 +20,9 @@ package io.druid.segment; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.historical.HistoricalFloatColumnSelector; -public final class ZeroFloatColumnSelector implements FloatColumnSelector +public final class ZeroFloatColumnSelector implements HistoricalFloatColumnSelector { private static final ZeroFloatColumnSelector INSTANCE = new ZeroFloatColumnSelector(); @@ -41,6 +42,12 @@ public float get() return 0.0f; } + @Override + public float get(int offset) + { + return get(); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java index 22d19b606f91..b307d37e1654 100644 --- a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java @@ -19,7 +19,10 @@ package io.druid.segment.column; +import io.druid.query.extraction.ExtractionFn; import io.druid.segment.data.IndexedInts; +import io.druid.segment.historical.HistoricalDimensionSelector; +import io.druid.segment.historical.OffsetHolder; import java.io.Closeable; @@ -34,4 +37,6 @@ public interface DictionaryEncodedColumn extends public ActualType lookupName(int id); public int lookupId(ActualType name); public int getCardinality(); + + HistoricalDimensionSelector makeDimensionSelector(OffsetHolder offsetHolder, ExtractionFn extractionFn); } diff --git a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java index 35fb8d03e4c0..4e141bd5d80b 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java @@ -19,13 +19,27 @@ package io.druid.segment.column; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; import com.google.common.base.Strings; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.query.extraction.ExtractionFn; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.DimensionSelectorUtils; +import io.druid.segment.IdLookup; import io.druid.segment.data.CachingIndexed; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.IndexedMultivalue; +import io.druid.segment.data.SingleIndexedInt; +import io.druid.segment.filter.BooleanValueMatcher; +import io.druid.segment.historical.HistoricalDimensionSelector; +import io.druid.segment.historical.OffsetHolder; +import io.druid.segment.historical.SingleValueHistoricalDimensionSelector; +import javax.annotation.Nullable; import java.io.IOException; +import java.util.BitSet; /** */ @@ -90,6 +104,172 @@ public int getCardinality() return cachedLookups.size(); } + @Override + public HistoricalDimensionSelector makeDimensionSelector( + final OffsetHolder offsetHolder, + final ExtractionFn extractionFn + ) + { + abstract class QueryableDimensionSelector implements HistoricalDimensionSelector, IdLookup + { + @Override + public int getValueCardinality() + { + return getCardinality(); + } + + @Override + public String lookupName(int id) + { + final String value = SimpleDictionaryEncodedColumn.this.lookupName(id); + return extractionFn == null ? + value : + extractionFn.apply(value); + } + + @Override + public boolean nameLookupPossibleInAdvance() + { + return true; + } + + @Nullable + @Override + public IdLookup idLookup() + { + return extractionFn == null ? this : null; + } + + @Override + public int lookupId(String name) + { + if (extractionFn != null) { + throw new UnsupportedOperationException( + "cannot perform lookup when applying an extraction function" + ); + } + return SimpleDictionaryEncodedColumn.this.lookupId(name); + } + } + + if (hasMultipleValues()) { + class MultiValueDimensionSelector extends QueryableDimensionSelector + { + @Override + public IndexedInts getRow() + { + return multiValueColumn.get(offsetHolder.getOffset().getOffset()); + } + + @Override + public IndexedInts getRow(int offset) + { + return multiValueColumn.get(offset); + } + + @Override + public ValueMatcher makeValueMatcher(String value) + { + return DimensionSelectorUtils.makeValueMatcherGeneric(this, value); + } + + @Override + public ValueMatcher makeValueMatcher(Predicate predicate) + { + return DimensionSelectorUtils.makeValueMatcherGeneric(this, predicate); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("multiValueColumn", multiValueColumn); + inspector.visit("offsetHolder", offsetHolder); + inspector.visit("offset", offsetHolder.getOffset()); + inspector.visit("extractionFn", extractionFn); + } + } + return new MultiValueDimensionSelector(); + } else { + class SingleValueQueryableDimensionSelector extends QueryableDimensionSelector + implements SingleValueHistoricalDimensionSelector + { + @Override + public IndexedInts getRow() + { + return new SingleIndexedInt(getRowValue()); + } + + @Override + public int getRowValue() + { + return column.get(offsetHolder.getOffset().getOffset()); + } + + @Override + public IndexedInts getRow(int offset) + { + return new SingleIndexedInt(getRowValue(offset)); + } + + @Override + public int getRowValue(int offset) + { + return column.get(offset); + } + + @Override + public ValueMatcher makeValueMatcher(final String value) + { + if (extractionFn == null) { + final int valueId = lookupId(value); + if (valueId >= 0) { + return new ValueMatcher() + { + @Override + public boolean matches() + { + return getRowValue() == valueId; + } + }; + } else { + return BooleanValueMatcher.of(false); + } + } else { + // Employ precomputed BitSet optimization + return makeValueMatcher(Predicates.equalTo(value)); + } + } + + @Override + public ValueMatcher makeValueMatcher(final Predicate predicate) + { + final BitSet predicateMatchingValueIds = DimensionSelectorUtils.makePredicateMatchingSet( + this, + predicate + ); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return predicateMatchingValueIds.get(getRowValue()); + } + }; + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("column", column); + inspector.visit("offsetHolder", offsetHolder); + inspector.visit("offset", offsetHolder.getOffset()); + inspector.visit("extractionFn", extractionFn); + } + } + return new SingleValueQueryableDimensionSelector(); + } + } + @Override public void close() throws IOException { diff --git a/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java b/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java index 71ad4025d0aa..d21ce0e8b814 100644 --- a/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java @@ -20,6 +20,7 @@ package io.druid.segment.data; import io.druid.java.util.common.IAE; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import it.unimi.dsi.fastutil.ints.IntArrays; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntIterators; @@ -90,6 +91,10 @@ public void fill(int index, int[] toFill) @Override public void close() throws IOException { + } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { } } diff --git a/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java b/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java index 81a594e2f562..80188a89eccf 100644 --- a/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java @@ -19,6 +19,8 @@ package io.druid.segment.data; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; + import java.util.Arrays; import java.util.Iterator; @@ -67,4 +69,9 @@ public Iterator iterator() { return Arrays.asList(baseArray).iterator(); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + } } diff --git a/processing/src/main/java/io/druid/segment/data/BitmapCompressedIndexedInts.java b/processing/src/main/java/io/druid/segment/data/BitmapCompressedIndexedInts.java index 2c78d39429cf..a1f6346dedc0 100644 --- a/processing/src/main/java/io/druid/segment/data/BitmapCompressedIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/BitmapCompressedIndexedInts.java @@ -21,6 +21,7 @@ import com.google.common.collect.Ordering; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.IntIteratorUtils; import it.unimi.dsi.fastutil.ints.IntIterator; @@ -96,6 +97,11 @@ public void fill(int index, int[] toFill) @Override public void close() throws IOException { + } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("immutableBitmap", immutableBitmap); } } diff --git a/processing/src/main/java/io/druid/segment/data/CachingIndexed.java b/processing/src/main/java/io/druid/segment/data/CachingIndexed.java index 16b941718439..e7e00743a41c 100644 --- a/processing/src/main/java/io/druid/segment/data/CachingIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/CachingIndexed.java @@ -21,6 +21,7 @@ import io.druid.java.util.common.Pair; import io.druid.java.util.common.logger.Logger; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import java.io.Closeable; import java.io.IOException; @@ -108,7 +109,13 @@ public void close() throws IOException } } -private static class SizedLRUMap extends LinkedHashMap> + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("delegate", delegate); + } + + private static class SizedLRUMap extends LinkedHashMap> { private final int maxBytes; private int numBytes = 0; diff --git a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java index 4e56126354b0..d36ef7ca47f2 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java @@ -27,6 +27,7 @@ import io.druid.java.util.common.IAE; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.CompressedPools; import it.unimi.dsi.fastutil.ints.IntIterator; @@ -365,5 +366,13 @@ public void close() throws IOException { Closeables.close(holder, false); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + // ideally should inspect buffer, but at the moment of inspectRuntimeShape() call buffer might be null although + // during the processing it is not null, hence "visiting" null is not representative. + inspector.visit("singleThreadedIntBuffers", singleThreadedIntBuffers); + } } } diff --git a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java index dd678c44c581..38d8ae266cb6 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.io.Closeables; import com.google.common.primitives.Ints; @@ -28,6 +29,7 @@ import io.druid.java.util.common.IAE; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.CompressedPools; import it.unimi.dsi.fastutil.ints.IntIterator; @@ -141,9 +143,7 @@ public void writeToChannel(WritableByteChannel channel) throws IOException baseBuffers.writeToChannel(channel); } - /** - * For testing. Do not use unless you like things breaking - */ + @VisibleForTesting GenericIndexed> getBaseBuffers() { return baseBuffers; @@ -283,6 +283,12 @@ protected int _get(int index) { return intBuffer.get(intBuffer.position() + index); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("intBuffer", intBuffer); + } } private class CompressedShortSizeIndexedInts extends CompressedVSizeIndexedInts @@ -302,6 +308,12 @@ protected int _get(int index) // removes the need for padding return shortBuffer.get(shortBuffer.position() + index) & 0xFFFF; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("shortBuffer", shortBuffer); + } } private class CompressedByteSizeIndexedInts extends CompressedVSizeIndexedInts @@ -312,6 +324,12 @@ protected int _get(int index) // removes the need for padding return buffer.get(buffer.position() + index) & 0xFF; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("buffer", buffer); + } } private class CompressedVSizeIndexedInts implements IndexedInts @@ -409,5 +427,12 @@ public void close() throws IOException { Closeables.close(holder, false); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("buffer", buffer); + inspector.visit("bigEndian", bigEndian); + } } } diff --git a/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java b/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java index 7b604b950aa6..71f26f56bd71 100644 --- a/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntIterators; @@ -61,6 +62,10 @@ public void fill(int index, int[] toFill) @Override public void close() throws IOException { + } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { } } diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 6e278ea79b1c..b14f329a0e43 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -27,6 +27,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import it.unimi.dsi.fastutil.bytes.ByteArrays; import java.io.ByteArrayOutputStream; @@ -525,6 +526,9 @@ public int size() return size; } + @Override + public abstract T get(int index); + protected T _get(ByteBuffer copyValueBuffer, int startOffset, int endOffset) { final int size = endOffset - startOffset; @@ -582,6 +586,19 @@ public Iterator iterator() { return IndexedIterable.create(this).iterator(); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("theBuffer", theBuffer); + inspector.visit("headerBuffer", headerBuffer); + inspector.visit("strategy", strategy); + } } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("bufferedIndexed", bufferIndexed); + } } diff --git a/processing/src/main/java/io/druid/segment/data/Indexed.java b/processing/src/main/java/io/druid/segment/data/Indexed.java index d64ccf81f3fd..1f6ee2ebf1ca 100644 --- a/processing/src/main/java/io/druid/segment/data/Indexed.java +++ b/processing/src/main/java/io/druid/segment/data/Indexed.java @@ -19,12 +19,16 @@ package io.druid.segment.data; -public interface Indexed extends Iterable +import io.druid.query.monomorphicprocessing.CalledFromHotLoop; +import io.druid.query.monomorphicprocessing.HotLoopCallee; + +public interface Indexed extends Iterable, HotLoopCallee { Class getClazz(); int size(); + @CalledFromHotLoop T get(int index); /** diff --git a/processing/src/main/java/io/druid/segment/data/IndexedInts.java b/processing/src/main/java/io/druid/segment/data/IndexedInts.java index 2e097b24618c..63af05f01699 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedInts.java @@ -19,6 +19,8 @@ package io.druid.segment.data; +import io.druid.query.monomorphicprocessing.CalledFromHotLoop; +import io.druid.query.monomorphicprocessing.HotLoopCallee; import it.unimi.dsi.fastutil.ints.IntIterable; import java.io.Closeable; @@ -26,9 +28,11 @@ /** * Get a int an index (array or list lookup abstraction without boxing). */ -public interface IndexedInts extends IntIterable, Closeable +public interface IndexedInts extends IntIterable, Closeable, HotLoopCallee { + @CalledFromHotLoop int size(); + @CalledFromHotLoop int get(int index); void fill(int index, int[] toFill); } diff --git a/processing/src/main/java/io/druid/segment/data/IndexedMultivalue.java b/processing/src/main/java/io/druid/segment/data/IndexedMultivalue.java index 1303396e7134..1c3906ee148a 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedMultivalue.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedMultivalue.java @@ -23,4 +23,6 @@ public interface IndexedMultivalue extends Indexed, Closeable { + @Override + T get(int index); } diff --git a/processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java b/processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java deleted file mode 100644 index e8712bcd3ca6..000000000000 --- a/processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - -import com.google.common.collect.Ordering; -import com.google.common.primitives.Ints; -import io.druid.collections.IntList; -import it.unimi.dsi.fastutil.ints.IntIterator; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.IntBuffer; - -/** - */ -public class IntBufferIndexedInts implements IndexedInts, Comparable -{ - public static ObjectStrategy objectStrategy = - new IntBufferIndexedIntsObjectStrategy(); - - public static IntBufferIndexedInts fromArray(int[] array) - { - final ByteBuffer buffer = ByteBuffer.allocate(array.length * Ints.BYTES); - buffer.asIntBuffer().put(array); - - return new IntBufferIndexedInts(buffer.asReadOnlyBuffer()); - } - - public static IntBufferIndexedInts fromIntList(IntList intList) - { - final ByteBuffer buffer = ByteBuffer.allocate(intList.length() * Ints.BYTES); - final IntBuffer intBuf = buffer.asIntBuffer(); - - for (int i = 0; i < intList.length(); ++i) { - intBuf.put(intList.get(i)); - } - - return new IntBufferIndexedInts(buffer.asReadOnlyBuffer()); - } - - private final ByteBuffer buffer; - - public IntBufferIndexedInts(ByteBuffer buffer) - { - this.buffer = buffer; - } - - @Override - public int size() - { - return buffer.remaining() / 4; - } - - @Override - public int get(int index) - { - return buffer.getInt(buffer.position() + (index * 4)); - } - - public ByteBuffer getBuffer() - { - return buffer.asReadOnlyBuffer(); - } - - @Override - public int compareTo(IntBufferIndexedInts o) - { - return buffer.compareTo(o.getBuffer()); - } - - @Override - public IntIterator iterator() - { - return new IndexedIntsIterator(this); - } - - private static class IntBufferIndexedIntsObjectStrategy implements ObjectStrategy - { - @Override - public Class getClazz() - { - return IntBufferIndexedInts.class; - } - - @Override - public IntBufferIndexedInts fromByteBuffer(ByteBuffer buffer, int numBytes) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); - return new IntBufferIndexedInts(readOnlyBuffer); - } - - @Override - public byte[] toBytes(IntBufferIndexedInts val) - { - ByteBuffer buffer = val.getBuffer(); - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes); - - return bytes; - } - - @Override - public int compare(IntBufferIndexedInts o1, IntBufferIndexedInts o2) - { - return Ordering.natural().nullsFirst().compare(o1, o2); - } - } - - @Override - public void fill(int index, int[] toFill) - { - throw new UnsupportedOperationException("fill not supported"); - } - - @Override - public void close() throws IOException - { - - } -} diff --git a/processing/src/main/java/io/druid/segment/data/ListIndexed.java b/processing/src/main/java/io/druid/segment/data/ListIndexed.java index feffc0d63a1c..b06b7ef7c1ec 100644 --- a/processing/src/main/java/io/druid/segment/data/ListIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/ListIndexed.java @@ -19,6 +19,8 @@ package io.druid.segment.data; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; + import java.util.Iterator; import java.util.List; @@ -67,4 +69,10 @@ public Iterator iterator() { return baseList.iterator(); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("baseList", baseList); + } } diff --git a/processing/src/main/java/io/druid/segment/data/RangeIndexedInts.java b/processing/src/main/java/io/druid/segment/data/RangeIndexedInts.java index 172c74e307f7..6f62ee398c49 100644 --- a/processing/src/main/java/io/druid/segment/data/RangeIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/RangeIndexedInts.java @@ -20,6 +20,7 @@ package io.druid.segment.data; import com.google.common.base.Preconditions; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntIterators; @@ -86,6 +87,10 @@ public IntIterator iterator() @Override public void close() throws IOException { + } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { } } diff --git a/processing/src/main/java/io/druid/segment/data/SingleIndexedInt.java b/processing/src/main/java/io/druid/segment/data/SingleIndexedInt.java index 2799cf9b31a7..16e0de3dffc5 100644 --- a/processing/src/main/java/io/druid/segment/data/SingleIndexedInt.java +++ b/processing/src/main/java/io/druid/segment/data/SingleIndexedInt.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntIterators; @@ -64,4 +65,9 @@ public void fill(int index, int[] toFill) public void close() throws IOException { } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + } } diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java index 4696cc97befb..f6fa582a6cb7 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java @@ -22,6 +22,7 @@ import com.google.common.primitives.Ints; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -189,6 +190,12 @@ public void close() throws IOException // no-op } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("theBuffer", theBuffer); + } + public WritableSupplier> asWritableSupplier() { return new VSizeIndexedSupplier(this); } diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java index f8f9c9fdcf19..fdf21f7181c2 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java @@ -22,6 +22,7 @@ import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import io.druid.java.util.common.IAE; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import it.unimi.dsi.fastutil.ints.IntIterator; import java.io.IOException; @@ -219,7 +220,12 @@ public void fill(int index, int[] toFill) @Override public void close() throws IOException { + } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("buffer", buffer); } public WritableSupplier asWritableSupplier() { diff --git a/processing/src/main/java/io/druid/segment/data/ZeroIndexedInts.java b/processing/src/main/java/io/druid/segment/data/ZeroIndexedInts.java index 1ee254da7235..d5a03f7bb158 100644 --- a/processing/src/main/java/io/druid/segment/data/ZeroIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/ZeroIndexedInts.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntIterators; @@ -70,6 +71,10 @@ public IntIterator iterator() @Override public void close() throws IOException { + } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { } } diff --git a/processing/src/main/java/io/druid/segment/historical/HistoricalCursor.java b/processing/src/main/java/io/druid/segment/historical/HistoricalCursor.java new file mode 100644 index 000000000000..ce289ce27d0c --- /dev/null +++ b/processing/src/main/java/io/druid/segment/historical/HistoricalCursor.java @@ -0,0 +1,26 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.historical; + +import io.druid.segment.Cursor; + +public interface HistoricalCursor extends Cursor, OffsetHolder +{ +} diff --git a/processing/src/main/java/io/druid/segment/data/ListBasedIndexedInts.java b/processing/src/main/java/io/druid/segment/historical/HistoricalDimensionSelector.java similarity index 52% rename from processing/src/main/java/io/druid/segment/data/ListBasedIndexedInts.java rename to processing/src/main/java/io/druid/segment/historical/HistoricalDimensionSelector.java index a76c5f2fe515..5723c7d37e9e 100644 --- a/processing/src/main/java/io/druid/segment/data/ListBasedIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/historical/HistoricalDimensionSelector.java @@ -17,48 +17,15 @@ * under the License. */ -package io.druid.segment.data; +package io.druid.segment.historical; -import it.unimi.dsi.fastutil.ints.IntIterator; - -import java.io.IOException; -import java.util.List; +import io.druid.segment.DimensionSelector; +import io.druid.segment.data.IndexedInts; /** + * Specialization for {@link DimensionSelector} queryable via offsets from {@link HistoricalCursor}. */ -public class ListBasedIndexedInts implements IndexedInts +public interface HistoricalDimensionSelector extends DimensionSelector { - private final List expansion; - - public ListBasedIndexedInts(List expansion) {this.expansion = expansion;} - - @Override - public int size() - { - return expansion.size(); - } - - @Override - public int get(int index) - { - return expansion.get(index); - } - - @Override - public IntIterator iterator() - { - return new IndexedIntsIterator(this); - } - - @Override - public void fill(int index, int[] toFill) - { - throw new UnsupportedOperationException("fill not supported"); - } - - @Override - public void close() throws IOException - { - - } + IndexedInts getRow(int offset); } diff --git a/processing/src/main/java/io/druid/segment/historical/HistoricalFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/historical/HistoricalFloatColumnSelector.java new file mode 100644 index 000000000000..7d7eedb7c6f4 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/historical/HistoricalFloatColumnSelector.java @@ -0,0 +1,27 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.historical; + +import io.druid.segment.FloatColumnSelector; + +public interface HistoricalFloatColumnSelector extends FloatColumnSelector +{ + float get(int offset); +} diff --git a/processing/src/main/java/io/druid/segment/historical/OffsetHolder.java b/processing/src/main/java/io/druid/segment/historical/OffsetHolder.java new file mode 100644 index 000000000000..438617d7207f --- /dev/null +++ b/processing/src/main/java/io/druid/segment/historical/OffsetHolder.java @@ -0,0 +1,27 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.historical; + +import io.druid.segment.data.Offset; + +public interface OffsetHolder +{ + Offset getOffset(); +} diff --git a/processing/src/main/java/io/druid/segment/historical/SingleValueHistoricalDimensionSelector.java b/processing/src/main/java/io/druid/segment/historical/SingleValueHistoricalDimensionSelector.java new file mode 100644 index 000000000000..727b8ae8e6fb --- /dev/null +++ b/processing/src/main/java/io/druid/segment/historical/SingleValueHistoricalDimensionSelector.java @@ -0,0 +1,28 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.historical; + +import io.druid.segment.SingleValueDimensionSelector; + +public interface SingleValueHistoricalDimensionSelector + extends HistoricalDimensionSelector, SingleValueDimensionSelector +{ + int getRowValue(int offset); +} diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java index 72cd2707049f..f6b0cd141fb4 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java @@ -25,6 +25,7 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.logger.Logger; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionHandler; import io.druid.segment.DimensionIndexer; import io.druid.segment.IndexableAdapter; @@ -301,6 +302,12 @@ public void fill(int index, int[] toFill) public void close() throws IOException { } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("bitmapIndex", bitmapIndex); + } } @Override diff --git a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java index 1207c275c9f8..78976f818fae 100644 --- a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java @@ -24,13 +24,14 @@ import io.druid.query.monomorphicprocessing.CalledFromHotLoop; import io.druid.segment.DimensionSelector; import io.druid.segment.IdLookup; +import io.druid.segment.SingleValueDimensionSelector; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.ZeroIndexedInts; import javax.annotation.Nullable; import java.util.Objects; -public abstract class BaseSingleValueDimensionSelector implements DimensionSelector +public abstract class BaseSingleValueDimensionSelector implements SingleValueDimensionSelector { @CalledFromHotLoop protected abstract String getValue(); @@ -41,6 +42,12 @@ public IndexedInts getRow() return ZeroIndexedInts.instance(); } + @Override + public int getRowValue() + { + return 0; + } + @Override public int getValueCardinality() { diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index a19a7396fcd9..4e7d27b134dc 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -157,7 +157,11 @@ public void fill(int index, int[] toFill) @Override public void close() throws IOException { + } + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { } }; } diff --git a/processing/src/test/java/io/druid/segment/data/IndexedIntsTest.java b/processing/src/test/java/io/druid/segment/data/IndexedIntsTest.java index d95a2d218771..ef284067997b 100644 --- a/processing/src/test/java/io/druid/segment/data/IndexedIntsTest.java +++ b/processing/src/test/java/io/druid/segment/data/IndexedIntsTest.java @@ -43,7 +43,7 @@ public static Collection constructorFeeder() throws IOException return Arrays.asList( new Object[][]{ {VSizeIndexedInts.fromArray(array)}, - {IntBufferIndexedInts.fromArray(array)} + {ArrayBasedIndexedInts.of(array)} } ); } diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java index 7b8f8c8abf32..0f9b567175c5 100644 --- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -46,15 +46,13 @@ import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; -import it.unimi.dsi.fastutil.ints.IntIterator; -import it.unimi.dsi.fastutil.ints.IntIterators; +import io.druid.segment.data.ZeroIndexedInts; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import javax.annotation.Nullable; -import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -280,38 +278,7 @@ public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, Colu @Override public IndexedInts getRow() { - return new IndexedInts() - { - @Override - public int size() - { - return 1; - } - - @Override - public int get(int index) - { - return 0; - } - - @Override - public IntIterator iterator() - { - return IntIterators.singleton(0); - } - - @Override - public void fill(int index, int[] toFill) - { - throw new UnsupportedOperationException("fill not supported"); - } - - @Override - public void close() throws IOException - { - - } - }; + return ZeroIndexedInts.instance(); } @Override From d2d53c504907b15c09ffb91e202470c956f53ac8 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 17 Mar 2017 18:28:12 -0600 Subject: [PATCH 02/30] Add CalledFromHotLoop annocations to specialized methods in SimpleDoubleBufferAggregator --- .../druid/query/aggregation/SimpleDoubleBufferAggregator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleBufferAggregator.java index a8fb46c7c7dd..33090f5463c4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleBufferAggregator.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation; +import io.druid.query.monomorphicprocessing.CalledFromHotLoop; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.FloatColumnSelector; @@ -43,8 +44,10 @@ public FloatColumnSelector getSelector() * aggregator.init(buf, position); * aggregator.aggregate(buf, position, value); */ + @CalledFromHotLoop public abstract void putFirst(ByteBuffer buf, int position, double value); + @CalledFromHotLoop public abstract void aggregate(ByteBuffer buf, int position, double value); @Override From 719e979c08a29943f3e89e7c31bf39b50f607b92 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 17 Mar 2017 20:26:10 -0600 Subject: [PATCH 03/30] Fix a bug in Historical1SimpleDoubleAggPooledTopNScannerPrototype --- .../Historical1SimpleDoubleAggPooledTopNScannerPrototype.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java index 81cb86cc0a81..e7d8aaf52a67 100644 --- a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -62,7 +62,7 @@ public long scanAndAggregate( aggregator.aggregate(resultsBuffer, position, metric); } else if (position == TopNAlgorithm.INIT_POSITION_VALUE) { positions[dimIndex] = positionToAllocate; - aggregator.putFirst(resultsBuffer, position, metric); + aggregator.putFirst(resultsBuffer, positionToAllocate, metric); positionToAllocate += aggregatorSize; } } From 37b04c3984a4a2489ef65450028ec62e17db2d5a Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 21 Mar 2017 17:49:32 -0600 Subject: [PATCH 04/30] Fix a bug in SpecializationService --- .../query/monomorphicprocessing/SpecializationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java index 2eb625459d20..ead91c8ad1d3 100644 --- a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java +++ b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java @@ -329,7 +329,7 @@ private long addAndGetTotalIterationsOverTheLastHour(long newIterations) } } if (!currentMinutePresent) { - perMinuteIterations.computeIfAbsent(currentMinute, AtomicLong::new).addAndGet(newIterations); + perMinuteIterations.computeIfAbsent(currentMinute, m -> new AtomicLong()).addAndGet(newIterations); totalIterations += newIterations; } return totalIterations; From c63901ba472d083dc8358c520448cac940977b05 Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 22 Mar 2017 17:27:46 -0600 Subject: [PATCH 05/30] In SpecializationService, emit maxSpecializations warning only once --- .../SpecializationService.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java index ead91c8ad1d3..9b66a98fab24 100644 --- a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java +++ b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java @@ -100,6 +100,7 @@ public final class SpecializationService * specialization takes some JVM memory (machine code cache, byte code, etc.) */ private static final int maxSpecializations = Integer.getInteger("maxSpecializations", 1000); + private static final AtomicBoolean maxSpecializationsWarningEmitted = new AtomicBoolean(false); private static final ExecutorService classSpecializationExecutor = Execs.singleThreaded("class-specialization-%d"); @@ -346,14 +347,16 @@ public void run() // PerPrototypeClassState.specializationStates. But it might be that nobody ever hits even the current // maxSpecializations limit, so implementing cache eviction is an unnecessary complexity. specialized = perPrototypeClassState.prototypeClass.newInstance(); - LOG.warn( - "SpecializationService couldn't make more than [%d] specializations. " - + "Not doing specialization for runtime shape[%s] and class remapping[%s], using the prototype class[%s]", - maxSpecializations, - specializationId.runtimeShape, - specializationId.classRemapping, - perPrototypeClassState.prototypeClass - ); + if (!maxSpecializationsWarningEmitted.get() && maxSpecializationsWarningEmitted.compareAndSet(false, true)) { + LOG.warn( + "SpecializationService couldn't make more than [%d] specializations. " + + "Not doing specialization for runtime shape[%s] and class remapping[%s], using the prototype class[%s]", + maxSpecializations, + specializationId.runtimeShape, + specializationId.classRemapping, + perPrototypeClassState.prototypeClass + ); + } } else if (fakeSpecialize) { specialized = perPrototypeClassState.prototypeClass.newInstance(); } else { From 439c9066a8aabf01c2691810368ad84b2e9001a9 Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 23 Mar 2017 18:05:06 -0600 Subject: [PATCH 06/30] Make GenericIndexed.theBuffer final --- .../src/main/java/io/druid/segment/data/GenericIndexed.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index b14f329a0e43..35b542c326b8 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -116,7 +116,7 @@ public int compare(String o1, String o2) private final ByteBuffer headerBuffer; private int logBaseTwoOfElementsPerValueFile; - private ByteBuffer theBuffer; + private final ByteBuffer theBuffer; // used for single file version, v1 GenericIndexed( @@ -171,6 +171,7 @@ public T get(int index) int numWritten ) { + this.theBuffer = null; this.strategy = strategy; this.allowReverseLookup = allowReverseLookup; this.valueBuffers = valueBuffs; From 19b4fa5538fa10bf0e5ea7fcd9aadbea3d43a69a Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 27 Mar 2017 17:11:38 -0600 Subject: [PATCH 07/30] Address comments --- .../query/monomorphicprocessing/SpecializationService.java | 2 ++ .../src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java index 9b66a98fab24..34a556d01c8f 100644 --- a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java +++ b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java @@ -293,6 +293,8 @@ static class WindowedLoopIterationCounter extends SpecializationState impl @Override public T getSpecialized() { + // Returns null because the class is not yet specialized. The purpose of WindowedLoopIterationCounter is to decide + // whether specialization should be done, or not. return null; } diff --git a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java index c81b0fb410eb..db2c41ee4efd 100644 --- a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java @@ -269,7 +269,7 @@ private static void scanAndAggregateHistorical1SimpleDoubleAgg( SpecializationService.getSpecializationState( prototypeScanner.getClass(), runtimeShape, - ImmutableMap., Class>of(Offset.class, historicalCursor.getOffset().getClass()) + ImmutableMap.of(Offset.class, historicalCursor.getOffset().getClass()) ); Historical1AggPooledTopNScanner scanner = specializationState.getSpecializedOrDefault(prototypeScanner); From 6b233f2aeeb10b96f4140444cf7520179823274b Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 27 Mar 2017 17:31:25 -0600 Subject: [PATCH 08/30] Newline --- .../src/main/java/io/druid/segment/data/GenericIndexed.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index d851d3a419a7..e237ede53900 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -669,4 +669,4 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) } }; } -} \ No newline at end of file +} From 18e9ad0407ef9a19d6908ad78bf358e630940d7d Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 27 Mar 2017 17:33:34 -0600 Subject: [PATCH 09/30] Reapply 439c906 (Make GenericIndexed.theBuffer final) --- .../src/main/java/io/druid/segment/data/GenericIndexed.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index e237ede53900..61f3395049cb 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -167,7 +167,7 @@ static int getNumberOfFilesRequired(int bagSize, long numWritten) private int logBaseTwoOfElementsPerValueFile; private int relativeIndexMask; - private ByteBuffer theBuffer; + private final ByteBuffer theBuffer; /** * Constructor for version one. @@ -211,6 +211,7 @@ static int getNumberOfFilesRequired(int bagSize, long numWritten) { this.versionOne = false; + this.theBuffer = null; this.strategy = strategy; this.allowReverseLookup = allowReverseLookup; this.valueBuffers = valueBuffs; From 996874d1e963b9cfc63785d39befdafd77ce85a0 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 28 Mar 2017 02:49:30 -0600 Subject: [PATCH 10/30] Remove extra PooledTopNAlgorithm.capabilities field --- .../src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java index db2c41ee4efd..1cfccea88302 100644 --- a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java @@ -74,7 +74,6 @@ public class PooledTopNAlgorithm Historical1AggPooledTopNScanner defaultHistoricalSingleValueDimSelector1SimpleDoubleAggScanner = new HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype(); - private final Capabilities capabilities; private final TopNQuery query; private final StupidPool bufferPool; private static final int AGG_UNROLL_COUNT = 8; // Must be able to fit loop below @@ -86,8 +85,6 @@ public PooledTopNAlgorithm( ) { super(capabilities); - - this.capabilities = capabilities; this.query = query; this.bufferPool = bufferPool; } From 10c6735e7da8353f79e9a41ad32eeb8d36fd85b7 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 28 Mar 2017 12:47:15 -0600 Subject: [PATCH 11/30] Improve CachingIndexed.inspectRuntimeShape() --- .../src/main/java/io/druid/segment/data/CachingIndexed.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/segment/data/CachingIndexed.java b/processing/src/main/java/io/druid/segment/data/CachingIndexed.java index e7e00743a41c..3f8e68fa780f 100644 --- a/processing/src/main/java/io/druid/segment/data/CachingIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/CachingIndexed.java @@ -74,7 +74,7 @@ public int size() @Override public T get(int index) { - if(cachedValues != null) { + if (cachedValues != null) { final T cached = cachedValues.getValue(index); if (cached != null) { return cached; @@ -112,6 +112,7 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + inspector.visit("cachedValues", cachedValues != null); inspector.visit("delegate", delegate); } From 8ded30c04c54d2228d6735112bf28fdbc637ba1f Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 29 Mar 2017 14:31:30 -0600 Subject: [PATCH 12/30] Fix CompressedVSizeIntsIndexedSupplier.inspectRuntimeShape() --- .../druid/segment/data/CompressedIntsIndexedSupplier.java | 5 +++-- .../segment/data/CompressedVSizeIntsIndexedSupplier.java | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java index d36ef7ca47f2..53ce230ee816 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java @@ -370,8 +370,9 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // ideally should inspect buffer, but at the moment of inspectRuntimeShape() call buffer might be null although - // during the processing it is not null, hence "visiting" null is not representative. + // ideally should inspect buffer, but at the moment of inspectRuntimeShape() call buffer is likely to be null, + // because loadBuffer() is not yet called, although during the processing it is not null, hence "visiting" null is + // not representative. inspector.visit("singleThreadedIntBuffers", singleThreadedIntBuffers); } } diff --git a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java index 38d8ae266cb6..0b4ddb38203a 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java @@ -431,8 +431,10 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - inspector.visit("buffer", buffer); - inspector.visit("bigEndian", bigEndian); + // ideally should inspect buffer and bigEndian, but at the moment of inspectRuntimeShape() call buffer is likely + // to be null and bigEndian = false, because loadBuffer() is not yet called, although during the processing buffer + // is not null, hence "visiting" null is not representative, and visiting bigEndian = false could be misleading. + inspector.visit("singleThreadedBuffers", singleThreadedBuffers); } } } From c9339c9f6b88cafc957218b1ac27770af505ca0f Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 29 Mar 2017 14:34:57 -0600 Subject: [PATCH 13/30] Don't override inspectRuntimeShape() in subclasses of CompressedVSizeIndexedInts --- .../CompressedVSizeIntsIndexedSupplier.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java index 0b4ddb38203a..abe2233d9c62 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java @@ -283,12 +283,6 @@ protected int _get(int index) { return intBuffer.get(intBuffer.position() + index); } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("intBuffer", intBuffer); - } } private class CompressedShortSizeIndexedInts extends CompressedVSizeIndexedInts @@ -308,12 +302,6 @@ protected int _get(int index) // removes the need for padding return shortBuffer.get(shortBuffer.position() + index) & 0xFFFF; } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("shortBuffer", shortBuffer); - } } private class CompressedByteSizeIndexedInts extends CompressedVSizeIndexedInts @@ -324,12 +312,6 @@ protected int _get(int index) // removes the need for padding return buffer.get(buffer.position() + index) & 0xFF; } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("buffer", buffer); - } } private class CompressedVSizeIndexedInts implements IndexedInts From de77b767b6a393750240c4ce20a11f392c63e421 Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 30 Mar 2017 16:22:15 -0600 Subject: [PATCH 14/30] Annotate methods in specializations of DimensionSelector and FloatColumnSelector with @CalledFromHotLoop --- .../java/io/druid/segment/SingleValueDimensionSelector.java | 3 +++ .../druid/segment/historical/HistoricalDimensionSelector.java | 2 ++ .../segment/historical/HistoricalFloatColumnSelector.java | 2 ++ .../historical/SingleValueHistoricalDimensionSelector.java | 2 ++ 4 files changed, 9 insertions(+) diff --git a/processing/src/main/java/io/druid/segment/SingleValueDimensionSelector.java b/processing/src/main/java/io/druid/segment/SingleValueDimensionSelector.java index 054787e65742..39f5730b78fb 100644 --- a/processing/src/main/java/io/druid/segment/SingleValueDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/SingleValueDimensionSelector.java @@ -19,6 +19,8 @@ package io.druid.segment; +import io.druid.query.monomorphicprocessing.CalledFromHotLoop; + /** * Specialization for {@link DimensionSelector}s, always having a single value in {@link #getRow()}. */ @@ -27,5 +29,6 @@ public interface SingleValueDimensionSelector extends DimensionSelector /** * Returns a single value of {@link #getRow()}. */ + @CalledFromHotLoop int getRowValue(); } diff --git a/processing/src/main/java/io/druid/segment/historical/HistoricalDimensionSelector.java b/processing/src/main/java/io/druid/segment/historical/HistoricalDimensionSelector.java index 5723c7d37e9e..698ed2f4d454 100644 --- a/processing/src/main/java/io/druid/segment/historical/HistoricalDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/historical/HistoricalDimensionSelector.java @@ -19,6 +19,7 @@ package io.druid.segment.historical; +import io.druid.query.monomorphicprocessing.CalledFromHotLoop; import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; @@ -27,5 +28,6 @@ */ public interface HistoricalDimensionSelector extends DimensionSelector { + @CalledFromHotLoop IndexedInts getRow(int offset); } diff --git a/processing/src/main/java/io/druid/segment/historical/HistoricalFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/historical/HistoricalFloatColumnSelector.java index 7d7eedb7c6f4..d2b91dab76ad 100644 --- a/processing/src/main/java/io/druid/segment/historical/HistoricalFloatColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/historical/HistoricalFloatColumnSelector.java @@ -19,9 +19,11 @@ package io.druid.segment.historical; +import io.druid.query.monomorphicprocessing.CalledFromHotLoop; import io.druid.segment.FloatColumnSelector; public interface HistoricalFloatColumnSelector extends FloatColumnSelector { + @CalledFromHotLoop float get(int offset); } diff --git a/processing/src/main/java/io/druid/segment/historical/SingleValueHistoricalDimensionSelector.java b/processing/src/main/java/io/druid/segment/historical/SingleValueHistoricalDimensionSelector.java index 727b8ae8e6fb..f5f6ba29d65a 100644 --- a/processing/src/main/java/io/druid/segment/historical/SingleValueHistoricalDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/historical/SingleValueHistoricalDimensionSelector.java @@ -19,10 +19,12 @@ package io.druid.segment.historical; +import io.druid.query.monomorphicprocessing.CalledFromHotLoop; import io.druid.segment.SingleValueDimensionSelector; public interface SingleValueHistoricalDimensionSelector extends HistoricalDimensionSelector, SingleValueDimensionSelector { + @CalledFromHotLoop int getRowValue(int offset); } From 378f622968a79b3ddc4563a9cfffd57ed0adea55 Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 30 Mar 2017 17:33:28 -0600 Subject: [PATCH 15/30] Make ValueMatcher to implement HotLoopCallee --- .../ForwardingFilteredDimensionSelector.java | 25 ++++++++--- .../dimension/ListFilteredDimensionSpec.java | 5 +-- .../PredicateFilteredDimensionSelector.java | 15 +++++++ .../dimension/RegexFilteredDimensionSpec.java | 2 +- ...oatValueMatcherColumnSelectorStrategy.java | 14 +++++++ ...ongValueMatcherColumnSelectorStrategy.java | 14 +++++++ .../io/druid/query/filter/ValueMatcher.java | 6 ++- .../RowBasedColumnSelectorFactory.java | 42 +++++++++++++++++++ .../druid/segment/DimensionSelectorUtils.java | 32 ++++++++++++++ .../java/io/druid/segment/FilteredOffset.java | 14 +++++++ .../segment/SingleScanTimeDimSelector.java | 13 ++++++ .../druid/segment/StringDimensionIndexer.java | 12 ++++++ .../column/SimpleDictionaryEncodedColumn.java | 12 ++++++ .../io/druid/segment/filter/AndFilter.java | 27 ++++++------ .../filter/ColumnComparisonFilter.java | 11 +++++ .../segment/filter/FalseValueMatcher.java | 7 ++++ .../java/io/druid/segment/filter/Filters.java | 8 ++++ .../io/druid/segment/filter/NotFilter.java | 7 ++++ .../io/druid/segment/filter/OrFilter.java | 27 ++++++------ .../segment/filter/TrueValueMatcher.java | 7 ++++ .../BaseSingleValueDimensionSelector.java | 14 +++++++ 21 files changed, 278 insertions(+), 36 deletions(-) diff --git a/processing/src/main/java/io/druid/query/dimension/ForwardingFilteredDimensionSelector.java b/processing/src/main/java/io/druid/query/dimension/ForwardingFilteredDimensionSelector.java index 52c489cce6eb..fa19679fef6d 100644 --- a/processing/src/main/java/io/druid/query/dimension/ForwardingFilteredDimensionSelector.java +++ b/processing/src/main/java/io/druid/query/dimension/ForwardingFilteredDimensionSelector.java @@ -31,7 +31,7 @@ import io.druid.segment.data.ArrayBasedIndexedInts; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.BooleanValueMatcher; -import it.unimi.dsi.fastutil.ints.Int2IntMap; +import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import javax.annotation.Nullable; import java.util.BitSet; @@ -40,14 +40,18 @@ final class ForwardingFilteredDimensionSelector implements DimensionSelector, Id { private final DimensionSelector selector; private final IdLookup baseIdLookup; - private final Int2IntMap forwardMapping; + private final Int2IntOpenHashMap forwardMapping; private final int[] reverseMapping; /** * @param selector must return true from {@link DimensionSelector#nameLookupPossibleInAdvance()} - * @param forwardMapping must have {@link Int2IntMap#defaultReturnValue(int)} configured to -1. + * @param forwardMapping must have {@link Int2IntOpenHashMap#defaultReturnValue(int)} configured to -1. */ - ForwardingFilteredDimensionSelector(DimensionSelector selector, Int2IntMap forwardMapping, int[] reverseMapping) + ForwardingFilteredDimensionSelector( + DimensionSelector selector, + Int2IntOpenHashMap forwardMapping, + int[] reverseMapping + ) { this.selector = Preconditions.checkNotNull(selector); if (!selector.nameLookupPossibleInAdvance()) { @@ -106,6 +110,12 @@ public boolean matches() // null should match empty rows in multi-value columns return nullRow && value == null; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } }; } else { return BooleanValueMatcher.of(false); @@ -141,6 +151,12 @@ public boolean matches() // null should match empty rows in multi-value columns return nullRow && matchNull; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } }; } @@ -179,6 +195,5 @@ public int lookupId(String name) public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("selector", selector); - inspector.visit("forwardMapping", forwardMapping); } } diff --git a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java index c1234f6ebcff..c7ab3873b578 100644 --- a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java @@ -28,7 +28,6 @@ import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; import io.druid.segment.IdLookup; -import it.unimi.dsi.fastutil.ints.Int2IntMap; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import javax.annotation.Nullable; @@ -93,7 +92,7 @@ private DimensionSelector filterWhiteList(DimensionSelector selector) } final int maxPossibleFilteredCardinality = values.size(); int count = 0; - final Int2IntMap forwardMapping = new Int2IntOpenHashMap(maxPossibleFilteredCardinality); + final Int2IntOpenHashMap forwardMapping = new Int2IntOpenHashMap(maxPossibleFilteredCardinality); forwardMapping.defaultReturnValue(-1); final int[] reverseMapping = new int[maxPossibleFilteredCardinality]; IdLookup idLookup = selector.idLookup(); @@ -134,7 +133,7 @@ public boolean apply(@Nullable String input) } final int maxPossibleFilteredCardinality = selectorCardinality; int count = 0; - final Int2IntMap forwardMapping = new Int2IntOpenHashMap(maxPossibleFilteredCardinality); + final Int2IntOpenHashMap forwardMapping = new Int2IntOpenHashMap(maxPossibleFilteredCardinality); forwardMapping.defaultReturnValue(-1); final int[] reverseMapping = new int[maxPossibleFilteredCardinality]; for (int i = 0; i < selectorCardinality; i++) { diff --git a/processing/src/main/java/io/druid/query/dimension/PredicateFilteredDimensionSelector.java b/processing/src/main/java/io/druid/query/dimension/PredicateFilteredDimensionSelector.java index 44b9fc347471..a99eda5c521e 100644 --- a/processing/src/main/java/io/druid/query/dimension/PredicateFilteredDimensionSelector.java +++ b/processing/src/main/java/io/druid/query/dimension/PredicateFilteredDimensionSelector.java @@ -79,6 +79,13 @@ public boolean matches() // null should match empty rows in multi-value columns return nullRow && value == null; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + // PredicateFilteredDimensionSelector.this inspects selector and predicate as well. + inspector.visit("selector", PredicateFilteredDimensionSelector.this); + } }; } @@ -106,6 +113,14 @@ public boolean matches() // null should match empty rows in multi-value columns return nullRow && matchNull; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + // PredicateFilteredDimensionSelector.this inspects selector and predicate as well. + inspector.visit("selector", PredicateFilteredDimensionSelector.this); + inspector.visit("matcherPredicate", matcherPredicate); + } }; } diff --git a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java index 3cfd7361488a..4ebd407083f4 100644 --- a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -83,7 +83,7 @@ public boolean apply(@Nullable String input) } int count = 0; - final Int2IntMap forwardMapping = new Int2IntOpenHashMap(); + final Int2IntOpenHashMap forwardMapping = new Int2IntOpenHashMap(); forwardMapping.defaultReturnValue(-1); for (int i = 0; i < selectorCardinality; i++) { if (compiledRegex.matcher(Strings.nullToEmpty(selector.lookupName(i))).matches()) { diff --git a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java index 2e16149c1038..b209f1b65345 100644 --- a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java @@ -19,6 +19,7 @@ package io.druid.query.filter; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.FloatColumnSelector; import io.druid.segment.filter.BooleanValueMatcher; @@ -41,6 +42,12 @@ public boolean matches() { return Float.floatToIntBits(selector.get()) == matchValIntBits; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } }; } @@ -57,6 +64,13 @@ public boolean matches() { return predicate.applyFloat(selector.get()); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + inspector.visit("predicate", predicate); + } }; } diff --git a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java index a0a2f59bf0dd..3995d9e0ae76 100644 --- a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java @@ -19,6 +19,7 @@ package io.druid.query.filter; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.LongColumnSelector; import io.druid.segment.filter.BooleanValueMatcher; @@ -40,6 +41,12 @@ public boolean matches() { return selector.get() == matchValLong; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } }; } @@ -56,6 +63,13 @@ public boolean matches() { return predicate.applyLong(selector.get()); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + inspector.visit("predicate", predicate); + } }; } diff --git a/processing/src/main/java/io/druid/query/filter/ValueMatcher.java b/processing/src/main/java/io/druid/query/filter/ValueMatcher.java index b701f3ade385..7213d445b113 100644 --- a/processing/src/main/java/io/druid/query/filter/ValueMatcher.java +++ b/processing/src/main/java/io/druid/query/filter/ValueMatcher.java @@ -19,9 +19,13 @@ package io.druid.query.filter; +import io.druid.query.monomorphicprocessing.CalledFromHotLoop; +import io.druid.query.monomorphicprocessing.HotLoopCallee; + /** */ -public interface ValueMatcher +public interface ValueMatcher extends HotLoopCallee { + @CalledFromHotLoop public boolean matches(); } diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java index 4148a29e7a14..3fcb2434ae90 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java @@ -131,6 +131,13 @@ public boolean matches() String rowValue = extractionFn.apply(row.get().getTimestampFromEpoch()); return Objects.equals(rowValue, value); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("row", row); + inspector.visit("extractionFn", extractionFn); + } }; } @@ -145,6 +152,14 @@ public boolean matches() String rowValue = extractionFn.apply(row.get().getTimestampFromEpoch()); return predicate.apply(rowValue); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("row", row); + inspector.visit("extractionFn", extractionFn); + inspector.visit("predicate", predicate); + } }; } @@ -211,6 +226,12 @@ public boolean matches() } return false; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("row", row); + } }; } else { return new ValueMatcher() @@ -230,6 +251,13 @@ public boolean matches() } return false; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("row", row); + inspector.visit("extractionFn", extractionFn); + } }; } } @@ -256,6 +284,13 @@ public boolean matches() } return false; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("row", row); + inspector.visit("predicate", predicate); + } }; } else { return new ValueMatcher() @@ -275,6 +310,13 @@ public boolean matches() } return false; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("row", row); + inspector.visit("predicate", predicate); + } }; } } diff --git a/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java b/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java index 4ade259a100e..3bf165becb4f 100644 --- a/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java @@ -23,6 +23,7 @@ import com.google.common.base.Predicates; import io.druid.java.util.common.IAE; import io.druid.query.filter.ValueMatcher; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.BooleanValueMatcher; @@ -81,6 +82,12 @@ public boolean matches() return false; } } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } }; } else { if (matchNull) { @@ -93,6 +100,12 @@ public boolean matches() final int size = row.size(); return size == 0; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } }; } else { return BooleanValueMatcher.of(false); @@ -124,6 +137,12 @@ public boolean matches() return false; } } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } }; } @@ -169,6 +188,12 @@ public boolean matches() return false; } } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } }; } @@ -197,6 +222,13 @@ public boolean matches() return false; } } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + inspector.visit("predicate", predicate); + } }; } diff --git a/processing/src/main/java/io/druid/segment/FilteredOffset.java b/processing/src/main/java/io/druid/segment/FilteredOffset.java index 49821fe9987b..f637b222d2f3 100644 --- a/processing/src/main/java/io/druid/segment/FilteredOffset.java +++ b/processing/src/main/java/io/druid/segment/FilteredOffset.java @@ -174,6 +174,13 @@ public boolean matches() return iterOffset == currentOffset; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("holder", holder); + inspector.visit("iter", iter); + } }; } else { return new ValueMatcher() @@ -190,6 +197,13 @@ public boolean matches() return iterOffset == currentOffset; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("holder", holder); + inspector.visit("iter", iter); + } }; } } diff --git a/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java b/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java index d0c03d684e3d..f7e41d572962 100644 --- a/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java +++ b/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java @@ -80,6 +80,12 @@ public boolean matches() { return Objects.equals(lookupName(getDimensionValueIndex()), value); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", SingleScanTimeDimSelector.this); + } }; } @@ -93,6 +99,13 @@ public boolean matches() { return predicate.apply(lookupName(getDimensionValueIndex())); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", SingleScanTimeDimSelector.this); + inspector.visit("predicate", predicate); + } }; } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 024ce50f1adb..eb5d88e5a762 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -471,6 +471,12 @@ public boolean matches() } return false; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + // nothing to inspect + } }; } else { return BooleanValueMatcher.of(false); @@ -508,6 +514,12 @@ public boolean matches() } return false; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + // nothing to inspect + } }; } diff --git a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java index 4e141bd5d80b..754282cc2856 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java @@ -230,6 +230,12 @@ public boolean matches() { return getRowValue() == valueId; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("column", SimpleDictionaryEncodedColumn.this); + } }; } else { return BooleanValueMatcher.of(false); @@ -254,6 +260,12 @@ public boolean matches() { return predicateMatchingValueIds.get(getRowValue()); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("column", SimpleDictionaryEncodedColumn.this); + } }; } diff --git a/processing/src/main/java/io/druid/segment/filter/AndFilter.java b/processing/src/main/java/io/druid/segment/filter/AndFilter.java index dbe4c7f45f09..2674994b8a2c 100644 --- a/processing/src/main/java/io/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/AndFilter.java @@ -28,6 +28,7 @@ import io.druid.query.filter.Filter; import io.druid.query.filter.RowOffsetMatcherFactory; import io.druid.query.filter.ValueMatcher; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; @@ -117,19 +118,7 @@ public ValueMatcher makeMatcher( matchers.add(0, offsetMatcher); } - return new ValueMatcher() - { - @Override - public boolean matches() - { - for (ValueMatcher valueMatcher : matchers) { - if (!valueMatcher.matches()) { - return false; - } - } - return true; - } - }; + return makeMatcher(matchers.toArray(new ValueMatcher[0])); } @Override @@ -181,6 +170,9 @@ public String toString() private ValueMatcher makeMatcher(final ValueMatcher[] baseMatchers) { + if (baseMatchers.length == 0) { + return BooleanValueMatcher.of(true); + } if (baseMatchers.length == 1) { return baseMatchers[0]; } @@ -197,6 +189,15 @@ public boolean matches() } return true; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("firstBaseMatcher", baseMatchers[0]); + inspector.visit("secondBaseMatcher", baseMatchers[1]); + // Don't inspect the 3rd and all consequent baseMatchers, cut runtime shape combinations at this point. + // Anyway if the filter is so complex, Hotspot won't inline all calls because of the inline limit. + } }; } diff --git a/processing/src/main/java/io/druid/segment/filter/ColumnComparisonFilter.java b/processing/src/main/java/io/druid/segment/filter/ColumnComparisonFilter.java index f8e126a2424d..698414435a0c 100644 --- a/processing/src/main/java/io/druid/segment/filter/ColumnComparisonFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/ColumnComparisonFilter.java @@ -29,6 +29,7 @@ import io.druid.query.filter.ValueMatcher; import io.druid.query.filter.ValueMatcherColumnSelectorStrategy; import io.druid.query.filter.ValueMatcherColumnSelectorStrategyFactory; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DimensionHandlerUtils; @@ -74,6 +75,9 @@ public ValueMatcher makeMatcher(ColumnSelectorFactory factory) } public static ValueMatcher makeValueMatcher(final ValueGetter[] valueGetters) { + if (valueGetters.length == 0) { + return BooleanValueMatcher.of(true); + } return new ValueMatcher() { @Override @@ -93,6 +97,13 @@ public boolean matches() } return true; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + // All value getters are likely the same or similar (in terms of runtime shape), so inspecting only one of them. + inspector.visit("oneValueGetter", valueGetters[0]); + } }; } diff --git a/processing/src/main/java/io/druid/segment/filter/FalseValueMatcher.java b/processing/src/main/java/io/druid/segment/filter/FalseValueMatcher.java index 189dae6a6332..2467dd79dcbe 100644 --- a/processing/src/main/java/io/druid/segment/filter/FalseValueMatcher.java +++ b/processing/src/main/java/io/druid/segment/filter/FalseValueMatcher.java @@ -20,6 +20,7 @@ package io.druid.segment.filter; import io.druid.query.filter.ValueMatcher; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; final class FalseValueMatcher implements ValueMatcher { @@ -39,4 +40,10 @@ public boolean matches() { return false; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + // nothing to inspect + } } diff --git a/processing/src/main/java/io/druid/segment/filter/Filters.java b/processing/src/main/java/io/druid/segment/filter/Filters.java index 756bcee52e17..a8e8a34a74a1 100644 --- a/processing/src/main/java/io/druid/segment/filter/Filters.java +++ b/processing/src/main/java/io/druid/segment/filter/Filters.java @@ -38,6 +38,7 @@ import io.druid.query.filter.ValueMatcher; import io.druid.query.filter.ValueMatcherColumnSelectorStrategy; import io.druid.query.filter.ValueMatcherColumnSelectorStrategyFactory; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DimensionHandlerUtils; @@ -453,6 +454,13 @@ public boolean matches() { return predicate.applyLong(longSelector.get()); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("longSelector", longSelector); + inspector.visit("predicate", predicate); + } }; } diff --git a/processing/src/main/java/io/druid/segment/filter/NotFilter.java b/processing/src/main/java/io/druid/segment/filter/NotFilter.java index cbba340b5c86..35a365bc63ef 100644 --- a/processing/src/main/java/io/druid/segment/filter/NotFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/NotFilter.java @@ -23,6 +23,7 @@ import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.Filter; import io.druid.query.filter.ValueMatcher; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; @@ -60,6 +61,12 @@ public boolean matches() { return !baseMatcher.matches(); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("baseMatcher", baseMatcher); + } }; } diff --git a/processing/src/main/java/io/druid/segment/filter/OrFilter.java b/processing/src/main/java/io/druid/segment/filter/OrFilter.java index 02fe68f937f4..981a41db4b43 100644 --- a/processing/src/main/java/io/druid/segment/filter/OrFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/OrFilter.java @@ -27,6 +27,7 @@ import io.druid.query.filter.Filter; import io.druid.query.filter.RowOffsetMatcherFactory; import io.druid.query.filter.ValueMatcher; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; @@ -103,23 +104,14 @@ public ValueMatcher makeMatcher( matchers.add(0, offsetMatcher); } - return new ValueMatcher() - { - @Override - public boolean matches() - { - for (ValueMatcher valueMatcher : matchers) { - if (valueMatcher.matches()) { - return true; - } - } - return false; - } - }; + return makeMatcher(matchers.toArray(new ValueMatcher[0])); } private ValueMatcher makeMatcher(final ValueMatcher[] baseMatchers){ + if (baseMatchers.length == 0) { + return BooleanValueMatcher.of(false); + } if (baseMatchers.length == 1) { return baseMatchers[0]; } @@ -136,6 +128,15 @@ public boolean matches() } return false; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("firstBaseMatcher", baseMatchers[0]); + inspector.visit("secondBaseMatcher", baseMatchers[1]); + // Don't inspect the 3rd and all consequent baseMatchers, cut runtime shape combinations at this point. + // Anyway if the filter is so complex, Hotspot won't inline all calls because of the inline limit. + } }; } diff --git a/processing/src/main/java/io/druid/segment/filter/TrueValueMatcher.java b/processing/src/main/java/io/druid/segment/filter/TrueValueMatcher.java index 288e43870824..bfcdb7a32131 100644 --- a/processing/src/main/java/io/druid/segment/filter/TrueValueMatcher.java +++ b/processing/src/main/java/io/druid/segment/filter/TrueValueMatcher.java @@ -20,6 +20,7 @@ package io.druid.segment.filter; import io.druid.query.filter.ValueMatcher; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; final class TrueValueMatcher implements ValueMatcher { @@ -39,4 +40,10 @@ public boolean matches() { return true; } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + // nothing to inspect + } } diff --git a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java index 78976f818fae..d7425cd08c0c 100644 --- a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicate; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.CalledFromHotLoop; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionSelector; import io.druid.segment.IdLookup; import io.druid.segment.SingleValueDimensionSelector; @@ -70,6 +71,12 @@ public boolean matches() { return Objects.equals(getValue(), value); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", BaseSingleValueDimensionSelector.this); + } }; } @@ -83,6 +90,13 @@ public boolean matches() { return predicate.apply(getValue()); } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", BaseSingleValueDimensionSelector.this); + inspector.visit("predicate", predicate); + } }; } From 7b893373eb2105aa7d7612d7452d23202eaf5ba8 Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 30 Mar 2017 17:54:22 -0600 Subject: [PATCH 16/30] Doc fix --- .../io/druid/query/monomorphicprocessing/HotLoopCallee.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/monomorphicprocessing/HotLoopCallee.java b/processing/src/main/java/io/druid/query/monomorphicprocessing/HotLoopCallee.java index af82900fc803..19aa3f4219e6 100644 --- a/processing/src/main/java/io/druid/query/monomorphicprocessing/HotLoopCallee.java +++ b/processing/src/main/java/io/druid/query/monomorphicprocessing/HotLoopCallee.java @@ -37,7 +37,7 @@ public interface HotLoopCallee * this instance (the instance on which inspectRuntimeShape() is called) is configured. * d. ByteBuffer or similar objects, where byte order matters * e. boolean flags, affecting branch taking - * f. Arrays of objects, meeting conditions any of conditions a-e. + * f. Arrays of objects, meeting any of conditions a-e. */ void inspectRuntimeShape(RuntimeShapeInspector inspector); } From 5cc302b144a659970f46e91d3b19a4672d59c93e Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 30 Mar 2017 17:59:52 -0600 Subject: [PATCH 17/30] Fix inspectRuntimeShape() impl in ExpressionSelectors --- .../main/java/io/druid/segment/virtual/ExpressionSelectors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 9db9f9ddab45..757f7deddef7 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -130,6 +130,7 @@ protected String getValue() public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("baseSelector", baseSelector); + inspector.visit("extractionFn", extractionFn); } } return new ExtractionExpressionDimensionSelector(); From 554a94cea367ac192bc748e9f09f7bab490160aa Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 30 Mar 2017 19:14:33 -0600 Subject: [PATCH 18/30] INFO logging of specialization events --- .../SpecializationService.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java index 34a556d01c8f..5b590b472315 100644 --- a/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java +++ b/processing/src/main/java/io/druid/query/monomorphicprocessing/SpecializationService.java @@ -361,8 +361,21 @@ public void run() } } else if (fakeSpecialize) { specialized = perPrototypeClassState.prototypeClass.newInstance(); + LOG.info( + "Not specializing prototype class[%s] for runtime shape[%s] and class remapping[%s] because " + + "fakeSpecialize=true, using the prototype class instead", + perPrototypeClassState.prototypeClass, + specializationId.runtimeShape, + specializationId.classRemapping + ); } else { specialized = perPrototypeClassState.specialize(specializationId.classRemapping); + LOG.info( + "Specializing prototype class[%s] for runtime shape[%s] and class remapping[%s]", + perPrototypeClassState.prototypeClass, + specializationId.runtimeShape, + specializationId.classRemapping + ); } perPrototypeClassState.specializationStates.put(specializationId, new Specialized<>(specialized)); } From 2ec1911a07d6814c562d1547af94531fed1c5a29 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 31 Mar 2017 13:54:25 -0600 Subject: [PATCH 19/30] Remove modificator --- .../src/main/java/io/druid/query/filter/ValueMatcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/filter/ValueMatcher.java b/processing/src/main/java/io/druid/query/filter/ValueMatcher.java index 7213d445b113..cad255474d8b 100644 --- a/processing/src/main/java/io/druid/query/filter/ValueMatcher.java +++ b/processing/src/main/java/io/druid/query/filter/ValueMatcher.java @@ -27,5 +27,5 @@ public interface ValueMatcher extends HotLoopCallee { @CalledFromHotLoop - public boolean matches(); + boolean matches(); } From 5633e9771b2c5620905745c3b73656d992fcc9e9 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 31 Mar 2017 19:05:05 -0600 Subject: [PATCH 20/30] Fix OrFilter --- .../java/io/druid/segment/filter/OrFilter.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/filter/OrFilter.java b/processing/src/main/java/io/druid/segment/filter/OrFilter.java index 981a41db4b43..44e8ba137263 100644 --- a/processing/src/main/java/io/druid/segment/filter/OrFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/OrFilter.java @@ -20,6 +20,7 @@ package io.druid.segment.filter; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.filter.BitmapIndexSelector; @@ -42,13 +43,9 @@ public class OrFilter implements BooleanFilter private final List filters; - public OrFilter( - List filters - ) + public OrFilter(List filters) { - if (filters.size() == 0) { - throw new IllegalArgumentException("Can't construct empty OrFilter (the universe does not exist)"); - } + Preconditions.checkArgument(filters.size() > 0, "Can't construct empty OrFilter (the universe does not exist)"); this.filters = filters; } @@ -109,9 +106,8 @@ public ValueMatcher makeMatcher( private ValueMatcher makeMatcher(final ValueMatcher[] baseMatchers){ - if (baseMatchers.length == 0) { - return BooleanValueMatcher.of(false); - } + Preconditions.checkState(baseMatchers.length > 0); + if (baseMatchers.length == 1) { return baseMatchers[0]; } From d962c86f496f96fc77c109f13a36665f0828eb1a Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 31 Mar 2017 19:31:47 -0600 Subject: [PATCH 21/30] Fix AndFilter --- .../java/io/druid/segment/filter/AndFilter.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/filter/AndFilter.java b/processing/src/main/java/io/druid/segment/filter/AndFilter.java index 2674994b8a2c..dd66eee6f55a 100644 --- a/processing/src/main/java/io/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/AndFilter.java @@ -43,10 +43,9 @@ public class AndFilter implements BooleanFilter private final List filters; - public AndFilter( - List filters - ) + public AndFilter(List filters) { + Preconditions.checkArgument(filters.size() > 0, "Can't construct empty AndFilter"); this.filters = filters; } @@ -81,10 +80,6 @@ public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector) @Override public ValueMatcher makeMatcher(ColumnSelectorFactory factory) { - if (filters.size() == 0) { - return BooleanValueMatcher.of(false); - } - final ValueMatcher[] matchers = new ValueMatcher[filters.size()]; for (int i = 0; i < filters.size(); i++) { @@ -170,9 +165,7 @@ public String toString() private ValueMatcher makeMatcher(final ValueMatcher[] baseMatchers) { - if (baseMatchers.length == 0) { - return BooleanValueMatcher.of(true); - } + Preconditions.checkState(baseMatchers.length > 0); if (baseMatchers.length == 1) { return baseMatchers[0]; } From 9771c3a9cdbdd79e142a2a1540b54219e28e75b8 Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 10 May 2017 13:58:21 -0500 Subject: [PATCH 22/30] Refactor PooledTopNAlgorithm.scanAndAggregate() --- .../druid/query/topn/PooledTopNAlgorithm.java | 122 ++++++++++++------ 1 file changed, 84 insertions(+), 38 deletions(-) diff --git a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java index 1cfccea88302..68a2660d53a6 100644 --- a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java @@ -45,7 +45,9 @@ import io.druid.segment.historical.SingleValueHistoricalDimensionSelector; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** */ @@ -74,6 +76,86 @@ public class PooledTopNAlgorithm Historical1AggPooledTopNScanner defaultHistoricalSingleValueDimSelector1SimpleDoubleAggScanner = new HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype(); + private interface ScanAndAggregate + { + /** + * If this implementation of ScanAndAggregate is executable with the given parameters, run it and return true. + * Otherwise return false (scanning and aggregation is not done). + */ + boolean scanAndAggregate( + final PooledTopNParams params, + final int[] positions, + final BufferAggregator[] theAggregators + ); + } + + private static final List specializedScanAndAggregateImplementations = new ArrayList<>(); + static { + // The order of the following `if` blocks matters, "more specialized" implementations go first + if (specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN) { + specializedScanAndAggregateImplementations.add((params, positions, theAggregators) -> { + if (theAggregators.length == 1) { + BufferAggregator aggregator = theAggregators[0]; + final Cursor cursor = params.getCursor(); + if (cursor instanceof HistoricalCursor && aggregator instanceof SimpleDoubleBufferAggregator) { + if (params.getDimSelector() instanceof SingleValueHistoricalDimensionSelector && + ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalFloatColumnSelector) { + scanAndAggregateHistorical1SimpleDoubleAgg( + params, + positions, + (SimpleDoubleBufferAggregator) aggregator, + cursor, + defaultHistoricalSingleValueDimSelector1SimpleDoubleAggScanner + ); + return true; + } + } + } + return false; + }); + } + if (specializeHistorical1SimpleDoubleAggPooledTopN) { + specializedScanAndAggregateImplementations.add((params, positions, theAggregators) -> { + if (theAggregators.length == 1) { + BufferAggregator aggregator = theAggregators[0]; + final Cursor cursor = params.getCursor(); + if (cursor instanceof HistoricalCursor && aggregator instanceof SimpleDoubleBufferAggregator) { + if (params.getDimSelector() instanceof HistoricalDimensionSelector && + ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalFloatColumnSelector) { + scanAndAggregateHistorical1SimpleDoubleAgg( + params, + positions, + (SimpleDoubleBufferAggregator) aggregator, + cursor, + defaultHistorical1SimpleDoubleAggScanner + ); + return true; + } + } + } + return false; + }); + } + if (specializeGeneric1AggPooledTopN) { + specializedScanAndAggregateImplementations.add((params, positions, theAggregators) -> { + if (theAggregators.length == 1) { + scanAndAggregateGeneric1Agg(params, positions, theAggregators[0], params.getCursor()); + return true; + } + return false; + }); + } + if (specializeGeneric2AggPooledTopN) { + specializedScanAndAggregateImplementations.add((params, positions, theAggregators) -> { + if (theAggregators.length == 2) { + scanAndAggregateGeneric2Agg(params, positions, theAggregators, params.getCursor()); + return true; + } + return false; + }); + } + } + private final TopNQuery query; private final StupidPool bufferPool; private static final int AGG_UNROLL_COUNT = 8; // Must be able to fit loop below @@ -206,48 +288,12 @@ protected void scanAndAggregate( final int numProcessed ) { - final Cursor cursor = params.getCursor(); - if (theAggregators.length == 1) { - BufferAggregator aggregator = theAggregators[0]; - if (cursor instanceof HistoricalCursor && aggregator instanceof SimpleDoubleBufferAggregator) { - if (specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN && - params.getDimSelector() instanceof SingleValueHistoricalDimensionSelector && - ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalFloatColumnSelector) { - scanAndAggregateHistorical1SimpleDoubleAgg( - params, - positions, - (SimpleDoubleBufferAggregator) aggregator, - cursor, - defaultHistoricalSingleValueDimSelector1SimpleDoubleAggScanner - ); - BaseQuery.checkInterrupted(); - return; - } - if (specializeHistorical1SimpleDoubleAggPooledTopN && - params.getDimSelector() instanceof HistoricalDimensionSelector && - ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalFloatColumnSelector) { - scanAndAggregateHistorical1SimpleDoubleAgg( - params, - positions, - (SimpleDoubleBufferAggregator) aggregator, - cursor, - defaultHistorical1SimpleDoubleAggScanner - ); - BaseQuery.checkInterrupted(); - return; - } - } - if (specializeGeneric1AggPooledTopN) { - scanAndAggregateGeneric1Agg(params, positions, aggregator, cursor); + for (ScanAndAggregate specializedScanAndAggregate : specializedScanAndAggregateImplementations) { + if (specializedScanAndAggregate.scanAndAggregate(params, positions, theAggregators)) { BaseQuery.checkInterrupted(); return; } } - if (specializeGeneric2AggPooledTopN && theAggregators.length == 2) { - scanAndAggregateGeneric2Agg(params, positions, theAggregators, cursor); - BaseQuery.checkInterrupted(); - return; - } scanAndAggregateDefault(params, positions, theAggregators); BaseQuery.checkInterrupted(); } From d405b97cb0693ae621cedce346e8ed5e9046965c Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 10 May 2017 14:53:58 -0500 Subject: [PATCH 23/30] Small refactoring --- .../java/io/druid/query/topn/PooledTopNAlgorithm.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java index 68a2660d53a6..596e031701ca 100644 --- a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java @@ -104,7 +104,7 @@ boolean scanAndAggregate( params, positions, (SimpleDoubleBufferAggregator) aggregator, - cursor, + (HistoricalCursor) cursor, defaultHistoricalSingleValueDimSelector1SimpleDoubleAggScanner ); return true; @@ -126,7 +126,7 @@ boolean scanAndAggregate( params, positions, (SimpleDoubleBufferAggregator) aggregator, - cursor, + (HistoricalCursor) cursor, defaultHistorical1SimpleDoubleAggScanner ); return true; @@ -302,17 +302,16 @@ private static void scanAndAggregateHistorical1SimpleDoubleAgg( PooledTopNParams params, int[] positions, SimpleDoubleBufferAggregator aggregator, - Cursor cursor, + HistoricalCursor cursor, Historical1AggPooledTopNScanner prototypeScanner ) { String runtimeShape = StringRuntimeShape.of(aggregator); - HistoricalCursor historicalCursor = (HistoricalCursor) cursor; SpecializationState specializationState = SpecializationService.getSpecializationState( prototypeScanner.getClass(), runtimeShape, - ImmutableMap.of(Offset.class, historicalCursor.getOffset().getClass()) + ImmutableMap.of(Offset.class, cursor.getOffset().getClass()) ); Historical1AggPooledTopNScanner scanner = specializationState.getSpecializedOrDefault(prototypeScanner); @@ -321,7 +320,7 @@ private static void scanAndAggregateHistorical1SimpleDoubleAgg( aggregator.getSelector(), aggregator, params.getAggregatorSizes()[0], - historicalCursor, + cursor, positions, params.getResultsBuf() ); From 58f0821e76c54ae2ab2967873d6bfccf970d7eb5 Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 10 May 2017 19:26:42 -0500 Subject: [PATCH 24/30] Add 'nothing to inspect' messages in empty HotLoopCallee.inspectRuntimeShape() implementations --- .../distinctcount/EmptyDistinctCountBufferAggregator.java | 1 + .../datasketches/theta/EmptySketchBufferAggregator.java | 1 + .../main/java/io/druid/query/aggregation/BufferAggregator.java | 1 + .../java/io/druid/query/aggregation/CountBufferAggregator.java | 1 + .../java/io/druid/query/aggregation/NoopBufferAggregator.java | 1 + .../src/main/java/io/druid/segment/FloatDimensionIndexer.java | 2 ++ .../src/main/java/io/druid/segment/LongDimensionIndexer.java | 2 ++ .../src/main/java/io/druid/segment/NullDimensionSelector.java | 1 + .../src/main/java/io/druid/segment/StringDimensionIndexer.java | 1 + .../src/main/java/io/druid/segment/ZeroFloatColumnSelector.java | 1 + .../src/main/java/io/druid/segment/ZeroLongColumnSelector.java | 1 + .../main/java/io/druid/segment/data/ArrayBasedIndexedInts.java | 1 + .../src/main/java/io/druid/segment/data/ArrayBasedOffset.java | 1 + .../src/main/java/io/druid/segment/data/ArrayIndexed.java | 1 + .../src/main/java/io/druid/segment/data/EmptyIndexedInts.java | 1 + .../src/main/java/io/druid/segment/data/RangeIndexedInts.java | 1 + .../src/main/java/io/druid/segment/data/SingleIndexedInt.java | 1 + .../src/main/java/io/druid/segment/data/ZeroIndexedInts.java | 1 + .../segment/incremental/IncrementalIndexStorageAdapter.java | 1 + .../java/io/druid/query/aggregation/FilteredAggregatorTest.java | 1 + .../aggregation/cardinality/CardinalityAggregatorTest.java | 2 ++ .../java/io/druid/query/dimension/TestDimensionSelector.java | 1 + .../query/monomorphicprocessing/StringRuntimeShapeTest.java | 1 + .../src/test/java/io/druid/segment/TestFloatColumnSelector.java | 1 + .../src/test/java/io/druid/segment/TestLongColumnSelector.java | 1 + .../test/java/io/druid/segment/virtual/VirtualColumnsTest.java | 1 + 26 files changed, 29 insertions(+) diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/EmptyDistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/EmptyDistinctCountBufferAggregator.java index 62e95a2c0936..25096ab27b64 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/EmptyDistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/EmptyDistinctCountBufferAggregator.java @@ -77,5 +77,6 @@ public void close() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchBufferAggregator.java index 2014314675c7..d97622d5ff83 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchBufferAggregator.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchBufferAggregator.java @@ -73,5 +73,6 @@ public void close() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java index 1951a6df4ae8..139278af9b60 100644 --- a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java @@ -125,6 +125,7 @@ public interface BufferAggregator extends HotLoopCallee */ default void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } /* diff --git a/processing/src/main/java/io/druid/query/aggregation/CountBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/CountBufferAggregator.java index daedab3f585f..ae8ee742ff71 100644 --- a/processing/src/main/java/io/druid/query/aggregation/CountBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/CountBufferAggregator.java @@ -68,5 +68,6 @@ public void close() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/query/aggregation/NoopBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NoopBufferAggregator.java index fd4d6bd51d7c..93ee651b449a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NoopBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NoopBufferAggregator.java @@ -73,5 +73,6 @@ public void close() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java index 36e0d2ce3ca5..a7f5f75450c5 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java @@ -120,6 +120,7 @@ public long get() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } @@ -150,6 +151,7 @@ public float get() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java index 963c03383a9d..0b24837db1c8 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java @@ -119,6 +119,7 @@ public long get() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } @@ -150,6 +151,7 @@ public float get() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index 9eebce5dbf07..ef7fd5838ab2 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -115,5 +115,6 @@ public int lookupId(String name) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index eb5d88e5a762..b9c4ac6a4186 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -330,6 +330,7 @@ public Iterator iterator() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } }; } diff --git a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java index 6622c0dca34a..ffcd67e83fa6 100644 --- a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java @@ -51,5 +51,6 @@ public float get(int offset) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java index fbf765fd93e0..71cb8128ce1f 100644 --- a/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java @@ -44,5 +44,6 @@ public long get() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java b/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java index d21ce0e8b814..f78ba2fca54a 100644 --- a/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java @@ -96,5 +96,6 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java b/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java index 41385ec77c7d..1b59c8a268f5 100644 --- a/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java @@ -73,5 +73,6 @@ public Offset clone() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java b/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java index 80188a89eccf..5cbc4bcf0a8e 100644 --- a/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java @@ -73,5 +73,6 @@ public Iterator iterator() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java b/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java index 71f26f56bd71..870808ce5eb3 100644 --- a/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java @@ -67,5 +67,6 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/data/RangeIndexedInts.java b/processing/src/main/java/io/druid/segment/data/RangeIndexedInts.java index 6f62ee398c49..0f426f32ec73 100644 --- a/processing/src/main/java/io/druid/segment/data/RangeIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/RangeIndexedInts.java @@ -92,5 +92,6 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/data/SingleIndexedInt.java b/processing/src/main/java/io/druid/segment/data/SingleIndexedInt.java index 16e0de3dffc5..0d7eaac4b80f 100644 --- a/processing/src/main/java/io/druid/segment/data/SingleIndexedInt.java +++ b/processing/src/main/java/io/druid/segment/data/SingleIndexedInt.java @@ -69,5 +69,6 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/data/ZeroIndexedInts.java b/processing/src/main/java/io/druid/segment/data/ZeroIndexedInts.java index d5a03f7bb158..a26e215d8a28 100644 --- a/processing/src/main/java/io/druid/segment/data/ZeroIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/ZeroIndexedInts.java @@ -76,5 +76,6 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index ea524bbb7114..c391efd9c3af 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -470,6 +470,7 @@ public long get() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } return new TimeLongColumnSelector(); diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index b2db0636c671..7eb71971487f 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -172,6 +172,7 @@ public int lookupId(String name) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } ); diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index 4e7d27b134dc..af1b64219ed5 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -162,6 +162,7 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } }; } @@ -214,6 +215,7 @@ public int lookupId(String s) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java b/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java index e31b51a7ac13..2f7c00ca070c 100644 --- a/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java +++ b/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java @@ -97,5 +97,6 @@ public int lookupId(String name) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/test/java/io/druid/query/monomorphicprocessing/StringRuntimeShapeTest.java b/processing/src/test/java/io/druid/query/monomorphicprocessing/StringRuntimeShapeTest.java index 189230710a0b..dd4762c7d0bd 100644 --- a/processing/src/test/java/io/druid/query/monomorphicprocessing/StringRuntimeShapeTest.java +++ b/processing/src/test/java/io/druid/query/monomorphicprocessing/StringRuntimeShapeTest.java @@ -34,6 +34,7 @@ static class Empty implements HotLoopCallee @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/test/java/io/druid/segment/TestFloatColumnSelector.java b/processing/src/test/java/io/druid/segment/TestFloatColumnSelector.java index 3d53ac0effc4..1e1babd960a5 100644 --- a/processing/src/test/java/io/druid/segment/TestFloatColumnSelector.java +++ b/processing/src/test/java/io/druid/segment/TestFloatColumnSelector.java @@ -29,5 +29,6 @@ public abstract class TestFloatColumnSelector implements FloatColumnSelector @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/test/java/io/druid/segment/TestLongColumnSelector.java b/processing/src/test/java/io/druid/segment/TestLongColumnSelector.java index f4b5a32f2585..c897b937724d 100644 --- a/processing/src/test/java/io/druid/segment/TestLongColumnSelector.java +++ b/processing/src/test/java/io/druid/segment/TestLongColumnSelector.java @@ -29,5 +29,6 @@ public abstract class TestLongColumnSelector implements LongColumnSelector @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } } diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java index 0f9b567175c5..ed045753866b 100644 --- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -329,6 +329,7 @@ public int lookupId(final String name) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { + // nothing to inspect } }; From f6efc31d83e8c6173a8187223c2337d2a2a11d77 Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 10 May 2017 19:35:02 -0500 Subject: [PATCH 25/30] Don't care about runtime shape in tests --- .../io/druid/query/aggregation/FilteredAggregatorTest.java | 2 +- .../aggregation/cardinality/CardinalityAggregatorTest.java | 4 ++-- .../java/io/druid/query/dimension/TestDimensionSelector.java | 2 +- .../query/monomorphicprocessing/StringRuntimeShapeTest.java | 2 +- .../test/java/io/druid/segment/TestFloatColumnSelector.java | 5 +---- .../test/java/io/druid/segment/TestLongColumnSelector.java | 5 +---- .../java/io/druid/segment/virtual/VirtualColumnsTest.java | 2 +- 7 files changed, 8 insertions(+), 14 deletions(-) diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 7eb71971487f..7671700040c8 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -172,7 +172,7 @@ public int lookupId(String name) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // nothing to inspect + // Don't care about runtime shape in tests } } ); diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index af1b64219ed5..0f53090c482c 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -162,7 +162,7 @@ public void close() throws IOException @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // nothing to inspect + // Don't care about runtime shape in tests } }; } @@ -215,7 +215,7 @@ public int lookupId(String s) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // nothing to inspect + // Don't care about runtime shape in tests } } diff --git a/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java b/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java index 2f7c00ca070c..a105b6e2378d 100644 --- a/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java +++ b/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java @@ -97,6 +97,6 @@ public int lookupId(String name) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // nothing to inspect + // Don't care about runtime shape in tests } } diff --git a/processing/src/test/java/io/druid/query/monomorphicprocessing/StringRuntimeShapeTest.java b/processing/src/test/java/io/druid/query/monomorphicprocessing/StringRuntimeShapeTest.java index dd4762c7d0bd..a7a7566feb64 100644 --- a/processing/src/test/java/io/druid/query/monomorphicprocessing/StringRuntimeShapeTest.java +++ b/processing/src/test/java/io/druid/query/monomorphicprocessing/StringRuntimeShapeTest.java @@ -34,7 +34,7 @@ static class Empty implements HotLoopCallee @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // nothing to inspect + // Don't care about runtime shape in tests } } diff --git a/processing/src/test/java/io/druid/segment/TestFloatColumnSelector.java b/processing/src/test/java/io/druid/segment/TestFloatColumnSelector.java index 1e1babd960a5..8d6bcf45fcb3 100644 --- a/processing/src/test/java/io/druid/segment/TestFloatColumnSelector.java +++ b/processing/src/test/java/io/druid/segment/TestFloatColumnSelector.java @@ -23,12 +23,9 @@ public abstract class TestFloatColumnSelector implements FloatColumnSelector { - /** - * Don't care about runtime shape in tests - */ @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // nothing to inspect + // Don't care about runtime shape in tests } } diff --git a/processing/src/test/java/io/druid/segment/TestLongColumnSelector.java b/processing/src/test/java/io/druid/segment/TestLongColumnSelector.java index c897b937724d..6c6db1da47a0 100644 --- a/processing/src/test/java/io/druid/segment/TestLongColumnSelector.java +++ b/processing/src/test/java/io/druid/segment/TestLongColumnSelector.java @@ -23,12 +23,9 @@ public abstract class TestLongColumnSelector implements LongColumnSelector { - /** - * Don't care about runtime shape in tests - */ @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // nothing to inspect + // Don't care about runtime shape in tests } } diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java index ed045753866b..7235ccc5b2a0 100644 --- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -329,7 +329,7 @@ public int lookupId(final String name) @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // nothing to inspect + // Don't care about runtime shape in tests } }; From d63b26e38fa46e025664d0b9a7894a8f6d9cadfb Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 11 May 2017 13:38:18 -0500 Subject: [PATCH 26/30] Fix accessor bugs in Historical1SimpleDoubleAggPooledTopNScannerPrototype and HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype, cover them with tests --- ...leDoubleAggPooledTopNScannerPrototype.java | 7 +- ...leDoubleAggPooledTopNScannerPrototype.java | 7 +- .../druid/query/topn/PooledTopNAlgorithm.java | 46 +++- .../java/io/druid/segment/BitmapOffset.java | 2 +- .../java/io/druid/segment/FilteredOffset.java | 13 +- .../segment/QueryableIndexStorageAdapter.java | 4 +- .../druid/segment/data/ArrayBasedOffset.java | 2 +- .../segment/data/IntersectingOffset.java | 2 +- .../java/io/druid/segment/data/Offset.java | 17 +- .../io/druid/segment/data/UnioningOffset.java | 2 +- .../java/sun/reflect/MagicAccessorBridge.java | 27 +++ .../druid/query/topn/TopNQueryRunnerTest.java | 202 ++++++++++-------- 12 files changed, 212 insertions(+), 119 deletions(-) create mode 100644 processing/src/main/java/sun/reflect/MagicAccessorBridge.java diff --git a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java index e7d8aaf52a67..7cc3baeb108c 100644 --- a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -25,10 +25,15 @@ import io.druid.segment.historical.HistoricalCursor; import io.druid.segment.historical.HistoricalDimensionSelector; import io.druid.segment.historical.HistoricalFloatColumnSelector; +import sun.reflect.MagicAccessorBridge; import java.nio.ByteBuffer; -public class Historical1SimpleDoubleAggPooledTopNScannerPrototype +/** + * Needs to extend {@link MagicAccessorBridge} because prototype specialization (including replacement of {@link Offset} + * occurrences with a subclass) may appear to access a private class. + */ +public class Historical1SimpleDoubleAggPooledTopNScannerPrototype extends MagicAccessorBridge implements Historical1AggPooledTopNScanner< HistoricalDimensionSelector, HistoricalFloatColumnSelector, diff --git a/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java index 91f77c4a8091..9974004887d2 100644 --- a/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -24,10 +24,15 @@ import io.druid.segment.historical.HistoricalCursor; import io.druid.segment.historical.HistoricalFloatColumnSelector; import io.druid.segment.historical.SingleValueHistoricalDimensionSelector; +import sun.reflect.MagicAccessorBridge; import java.nio.ByteBuffer; -public class HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype +/** + * Needs to extend {@link MagicAccessorBridge} because prototype specialization (including replacement of {@link Offset} + * occurrences with a subclass) may appear to access a private class. + */ +public class HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype extends MagicAccessorBridge implements Historical1AggPooledTopNScanner< SingleValueHistoricalDimensionSelector, HistoricalFloatColumnSelector, diff --git a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java index 596e031701ca..0158a4795ee8 100644 --- a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java @@ -54,18 +54,44 @@ public class PooledTopNAlgorithm extends BaseTopNAlgorithm { - /** Non-final fields for testing, see TopNQueryRunnerTest */ - @VisibleForTesting - static boolean specializeGeneric1AggPooledTopN = + private static boolean specializeGeneric1AggPooledTopN = !Boolean.getBoolean("dontSpecializeGeneric1AggPooledTopN"); - @VisibleForTesting - static boolean specializeGeneric2AggPooledTopN = + private static boolean specializeGeneric2AggPooledTopN = !Boolean.getBoolean("dontSpecializeGeneric2AggPooledTopN"); - private static final boolean specializeHistorical1SimpleDoubleAggPooledTopN = + private static boolean specializeHistorical1SimpleDoubleAggPooledTopN = !Boolean.getBoolean("dontSpecializeHistorical1SimpleDoubleAggPooledTopN"); - private static final boolean specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN = + private static boolean specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN = !Boolean.getBoolean("dontSpecializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN"); + /** See TopNQueryRunnerTest */ + @VisibleForTesting + static void setSpecializeGeneric1AggPooledTopN(boolean value) + { + PooledTopNAlgorithm.specializeGeneric1AggPooledTopN = value; + computeSpecializedScanAndAggregateImplementations(); + } + + @VisibleForTesting + static void setSpecializeGeneric2AggPooledTopN(boolean value) + { + PooledTopNAlgorithm.specializeGeneric2AggPooledTopN = value; + computeSpecializedScanAndAggregateImplementations(); + } + + @VisibleForTesting + static void setSpecializeHistorical1SimpleDoubleAggPooledTopN(boolean value) + { + PooledTopNAlgorithm.specializeHistorical1SimpleDoubleAggPooledTopN = value; + computeSpecializedScanAndAggregateImplementations(); + } + + @VisibleForTesting + static void setSpecializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN(boolean value) + { + PooledTopNAlgorithm.specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN = value; + computeSpecializedScanAndAggregateImplementations(); + } + private static final Generic1AggPooledTopNScanner defaultGeneric1AggScanner = new Generic1AggPooledTopNScannerPrototype(); private static final Generic2AggPooledTopNScanner defaultGeneric2AggScanner = @@ -91,6 +117,12 @@ boolean scanAndAggregate( private static final List specializedScanAndAggregateImplementations = new ArrayList<>(); static { + computeSpecializedScanAndAggregateImplementations(); + } + + private static void computeSpecializedScanAndAggregateImplementations() + { + specializedScanAndAggregateImplementations.clear(); // The order of the following `if` blocks matters, "more specialized" implementations go first if (specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN) { specializedScanAndAggregateImplementations.add((params, positions, theAggregators) -> { diff --git a/processing/src/main/java/io/druid/segment/BitmapOffset.java b/processing/src/main/java/io/druid/segment/BitmapOffset.java index 8b1d02979523..87f815d179d5 100644 --- a/processing/src/main/java/io/druid/segment/BitmapOffset.java +++ b/processing/src/main/java/io/druid/segment/BitmapOffset.java @@ -36,7 +36,7 @@ /** */ -public class BitmapOffset implements Offset +public class BitmapOffset extends Offset { private static final int INVALID_VALUE = -1; private static final BitmapFactory ROARING_BITMAP_FACTORY = new RoaringBitmapSerdeFactory(false).getBitmapFactory(); diff --git a/processing/src/main/java/io/druid/segment/FilteredOffset.java b/processing/src/main/java/io/druid/segment/FilteredOffset.java index f637b222d2f3..14daf44a7971 100644 --- a/processing/src/main/java/io/druid/segment/FilteredOffset.java +++ b/processing/src/main/java/io/druid/segment/FilteredOffset.java @@ -32,7 +32,7 @@ import io.druid.segment.historical.OffsetHolder; import org.roaringbitmap.IntIterator; -final class FilteredOffset implements Offset +final class FilteredOffset extends Offset { private Offset baseOffset; private final ValueMatcher filterMatcher; @@ -112,14 +112,9 @@ public boolean withinBounds() @Override public Offset clone() { - try { - FilteredOffset offset = (FilteredOffset) super.clone(); - offset.baseOffset = offset.baseOffset.clone(); - return offset; - } - catch (CloneNotSupportedException e) { - throw new RuntimeException(e); - } + FilteredOffset offset = (FilteredOffset) super.clone(); + offset.baseOffset = offset.baseOffset.clone(); + return offset; } @Override diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index c21d6152cebe..5b1b54a4f25d 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -823,7 +823,7 @@ public void reset() } } - private abstract static class TimestampCheckingOffset implements Offset + private abstract static class TimestampCheckingOffset extends Offset { protected final Offset baseOffset; protected final GenericColumn timestamps; @@ -950,7 +950,7 @@ public Offset clone() } } - private static class NoFilterOffset implements Offset + private static class NoFilterOffset extends Offset { private final int rowCount; private final boolean descending; diff --git a/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java b/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java index 1b59c8a268f5..90ce1da5dbee 100644 --- a/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java @@ -23,7 +23,7 @@ /** */ -public class ArrayBasedOffset implements Offset +public class ArrayBasedOffset extends Offset { private final int[] ints; private int currIndex; diff --git a/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java b/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java index f3da84d0a1e1..94e4cdc3792b 100644 --- a/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java +++ b/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java @@ -23,7 +23,7 @@ /** */ -public class IntersectingOffset implements Offset { +public class IntersectingOffset extends Offset { private final Offset lhs; private final Offset rhs; diff --git a/processing/src/main/java/io/druid/segment/data/Offset.java b/processing/src/main/java/io/druid/segment/data/Offset.java index 7e44dd7b8e7f..47fe4a7d5813 100644 --- a/processing/src/main/java/io/druid/segment/data/Offset.java +++ b/processing/src/main/java/io/druid/segment/data/Offset.java @@ -25,13 +25,22 @@ * The "mutable" version of a ReadableOffset. Introduces "increment()" and "withinBounds()" methods, which are * very similar to "next()" and "hasNext()" on the Iterator interface except increment() does not return a value. */ -public interface Offset extends ReadableOffset +public abstract class Offset implements ReadableOffset, Cloneable { @CalledFromHotLoop - void increment(); + public abstract void increment(); @CalledFromHotLoop - boolean withinBounds(); + public abstract boolean withinBounds(); - Offset clone(); + @Override + public Offset clone() + { + try { + return (Offset) super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } } diff --git a/processing/src/main/java/io/druid/segment/data/UnioningOffset.java b/processing/src/main/java/io/druid/segment/data/UnioningOffset.java index 05be50016b5e..225a18961f1c 100644 --- a/processing/src/main/java/io/druid/segment/data/UnioningOffset.java +++ b/processing/src/main/java/io/druid/segment/data/UnioningOffset.java @@ -23,7 +23,7 @@ /** */ -public class UnioningOffset implements Offset +public class UnioningOffset extends Offset { private final Offset[] offsets = new Offset[2]; private final int[] offsetVals = new int[2]; diff --git a/processing/src/main/java/sun/reflect/MagicAccessorBridge.java b/processing/src/main/java/sun/reflect/MagicAccessorBridge.java new file mode 100644 index 000000000000..460cd97218fa --- /dev/null +++ b/processing/src/main/java/sun/reflect/MagicAccessorBridge.java @@ -0,0 +1,27 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package sun.reflect; + +/** + * Public bridge is needed, because MagicAccessorImpl is a package-private class. + */ +public class MagicAccessorBridge extends MagicAccessorImpl +{ +} diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 67dcbe3780ec..5bcc27c7ac12 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -28,9 +28,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.common.primitives.Doubles; +import com.google.common.primitives.Longs; import io.druid.collections.StupidPool; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.granularity.Granularity; import io.druid.java.util.common.guava.Sequence; @@ -76,7 +79,6 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.ordering.StringComparators; import io.druid.query.spec.MultipleIntervalSegmentSpec; -import io.druid.query.timeseries.TimeseriesQuery; import io.druid.segment.TestHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; @@ -100,6 +102,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** */ @@ -111,13 +114,15 @@ public static Iterable constructorFeeder() throws IOException { List>> retVal = queryRunners(); List parameters = new ArrayList<>(); - for (int i = 0; i < 8; i++) { + for (int i = 0; i < 32; i++) { for (QueryRunner> firstParameter : retVal) { - Object[] params = new Object[4]; + Object[] params = new Object[6]; params[0] = firstParameter; params[1] = (i & 1) != 0; params[2] = (i & 2) != 0; params[3] = (i & 4) != 0; + params[4] = (i & 8) != 0; + params[5] = (i & 16) != 0; parameters.add(params); } } @@ -174,12 +179,20 @@ public TopNQueryRunnerTest( QueryRunner> runner, boolean specializeGeneric1AggPooledTopN, boolean specializeGeneric2AggPooledTopN, + boolean specializeHistorical1SimpleDoubleAggPooledTopN, + boolean specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN, boolean duplicateSingleAggregatorQueries ) { this.runner = runner; - PooledTopNAlgorithm.specializeGeneric1AggPooledTopN = specializeGeneric1AggPooledTopN; - PooledTopNAlgorithm.specializeGeneric2AggPooledTopN = specializeGeneric2AggPooledTopN; + PooledTopNAlgorithm.setSpecializeGeneric1AggPooledTopN(specializeGeneric1AggPooledTopN); + PooledTopNAlgorithm.setSpecializeGeneric2AggPooledTopN(specializeGeneric2AggPooledTopN); + PooledTopNAlgorithm.setSpecializeHistorical1SimpleDoubleAggPooledTopN( + specializeHistorical1SimpleDoubleAggPooledTopN + ); + PooledTopNAlgorithm.setSpecializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN( + specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN + ); this.duplicateSingleAggregatorQueries = duplicateSingleAggregatorQueries; } @@ -1886,19 +1899,7 @@ public void testTopNDimExtractionToOne() throws IOException .build(); Granularity gran = Granularities.DAY; - TimeseriesQuery tsQuery = Druids.newTimeseriesQueryBuilder() - .dataSource(QueryRunnerTestHelper.dataSource) - .granularity(gran) - .intervals(QueryRunnerTestHelper.fullOnInterval) - .aggregators( - Arrays.asList( - QueryRunnerTestHelper.rowsCount, - QueryRunnerTestHelper.indexDoubleSum, - QueryRunnerTestHelper.qualityUniques - ) - ) - .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) - .build(); + List> expectedResults = Arrays.asList( new Result<>( new DateTime("2011-01-12T00:00:00.000Z"), @@ -4966,79 +4967,98 @@ public void testFullOnTopNDecorationOnNumeric() @Test public void testFullOnTopNWithAggsOnNumericDims() { - TopNQuery query = new TopNQueryBuilder() - .dataSource(QueryRunnerTestHelper.dataSource) - .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) - .metric(QueryRunnerTestHelper.indexMetric) - .threshold(4) - .intervals(QueryRunnerTestHelper.fullOnInterval) - .aggregators( - Lists.newArrayList( - Iterables.concat( - QueryRunnerTestHelper.commonAggregators, - Lists.newArrayList( - new DoubleMaxAggregatorFactory("maxIndex", "index"), - new DoubleMinAggregatorFactory("minIndex", "index"), - new LongSumAggregatorFactory("qlLong", "qualityLong"), - new DoubleSumAggregatorFactory("qlFloat", "qualityLong"), - new DoubleSumAggregatorFactory("qfFloat", "qualityFloat"), - new LongSumAggregatorFactory("qfLong", "qualityFloat") - ) - ) - ) - ) - .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) - .build(); - - List> expectedResults = Arrays.asList( - new Result( - new DateTime("2011-01-12T00:00:00.000Z"), - new TopNResultValue( - Arrays.>asList( - ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "total_market") - .put("rows", 186L) - .put("index", 215679.82879638672D) - .put("addRowsIndexConstant", 215866.82879638672D) - .put("uniques", QueryRunnerTestHelper.UNIQUES_2) - .put("maxIndex", 1743.9217529296875D) - .put("minIndex", 792.3260498046875D) - .put("qlLong", 279000L) - .put("qlFloat", 279000.0) - .put("qfFloat", 2790000.0) - .put("qfLong", 2790000L) - .build(), - ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "upfront") - .put("rows", 186L) - .put("index", 192046.1060180664D) - .put("addRowsIndexConstant", 192233.1060180664D) - .put("uniques", QueryRunnerTestHelper.UNIQUES_2) - .put("maxIndex", 1870.06103515625D) - .put("minIndex", 545.9906005859375D) - .put("qlLong", 279000L) - .put("qlFloat", 279000.0) - .put("qfFloat", 2790000.0) - .put("qfLong", 2790000L) - .build(), - ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "spot") - .put("rows", 837L) - .put("index", 95606.57232284546D) - .put("addRowsIndexConstant", 96444.57232284546D) - .put("uniques", QueryRunnerTestHelper.UNIQUES_9) - .put("maxIndex", 277.2735290527344D) - .put("minIndex", 59.02102279663086D) - .put("qlLong", 1171800L) - .put("qlFloat", 1171800.0) - .put("qfFloat", 11718000.0) - .put("qfLong", 11718000L) - .build() - ) - ) - ) + List>> aggregations = new ArrayList<>(); + aggregations.add(new Pair<>( + QueryRunnerTestHelper.rowsCount, + Longs.asList(186L, 186L, 837L) + )); + Pair> indexAggregation = new Pair<>( + QueryRunnerTestHelper.indexDoubleSum, + Doubles.asList(215679.82879638672D, 192046.1060180664D, 95606.57232284546D) ); - assertExpectedResults(expectedResults, query); + aggregations.add(indexAggregation); + aggregations.add(new Pair<>( + QueryRunnerTestHelper.qualityUniques, + Doubles.asList(QueryRunnerTestHelper.UNIQUES_2, QueryRunnerTestHelper.UNIQUES_2, QueryRunnerTestHelper.UNIQUES_9) + )); + aggregations.add(new Pair<>( + new DoubleMaxAggregatorFactory("maxIndex", "index"), + Doubles.asList(1743.9217529296875D, 1870.06103515625D, 277.2735290527344D) + )); + aggregations.add(new Pair<>( + new DoubleMinAggregatorFactory("minIndex", "index"), + Doubles.asList(792.3260498046875D, 545.9906005859375D, 59.02102279663086D) + )); + aggregations.add(new Pair<>( + new LongSumAggregatorFactory("qlLong", "qualityLong"), + Longs.asList(279000L, 279000L, 1171800L) + )); + aggregations.add(new Pair<>( + new DoubleSumAggregatorFactory("qlFloat", "qualityLong"), + Doubles.asList(279000.0, 279000.0, 1171800.0) + )); + aggregations.add(new Pair<>( + new DoubleSumAggregatorFactory("qfFloat", "qualityFloat"), + Doubles.asList(2790000.0, 2790000.0, 11718000.0) + )); + aggregations.add(new Pair<>( + new LongSumAggregatorFactory("qfLong", "qualityFloat"), + Longs.asList(2790000L, 2790000L, 11718000L) + )); + + List>>> aggregationCombinations = new ArrayList<>(); + for (Pair> aggregation : aggregations) { + aggregationCombinations.add(Collections.singletonList(aggregation)); + } + aggregationCombinations.add(aggregations); + + for (List>> aggregationCombination : aggregationCombinations) { + boolean hasIndexAggregator = aggregationCombination.stream().anyMatch(agg -> "index".equals(agg.lhs.getName())); + boolean hasRowsAggregator = aggregationCombination.stream().anyMatch(agg -> "rows".equals(agg.lhs.getName())); + TopNQueryBuilder queryBuilder = new TopNQueryBuilder() + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.allGran) + .dimension(QueryRunnerTestHelper.marketDimension) + .threshold(4) + .intervals(QueryRunnerTestHelper.fullOnInterval) + .aggregators(aggregationCombination.stream().map(agg -> agg.lhs).collect(Collectors.toList())); + String metric; + if (hasIndexAggregator) { + metric = "index"; + } else { + metric = aggregationCombination.get(0).lhs.getName(); + } + queryBuilder.metric(metric); + if (hasIndexAggregator && hasRowsAggregator) { + queryBuilder.postAggregators(Collections.singletonList(QueryRunnerTestHelper.addRowsIndexConstant)); + } + TopNQuery query = queryBuilder.build(); + + ImmutableMap.Builder row1 = ImmutableMap.builder() + .put(QueryRunnerTestHelper.marketDimension, "total_market"); + ImmutableMap.Builder row2 = ImmutableMap.builder() + .put(QueryRunnerTestHelper.marketDimension, "upfront"); + ImmutableMap.Builder row3 = ImmutableMap.builder() + .put(QueryRunnerTestHelper.marketDimension, "spot"); + if (hasIndexAggregator && hasRowsAggregator) { + row1.put("addRowsIndexConstant", 215866.82879638672D); + row2.put("addRowsIndexConstant", 192233.1060180664D); + row3.put("addRowsIndexConstant", 96444.57232284546D); + } + aggregationCombination.forEach(agg -> { + row1.put(agg.lhs.getName(), agg.rhs.get(0)); + row2.put(agg.lhs.getName(), agg.rhs.get(1)); + row3.put(agg.lhs.getName(), agg.rhs.get(2)); + }); + List> rows = Lists.newArrayList(row1.build(), row2.build(), row3.build()); + rows.sort((r1, r2) -> ((Comparable) r2.get(metric)).compareTo(r1.get(metric))); + List> expectedResults = Collections.singletonList( + new Result<>( + new DateTime("2011-01-12T00:00:00.000Z"), + new TopNResultValue(rows) + ) + ); + assertExpectedResults(expectedResults, query); + } } } From 35341a56bbb62a7f2c9bacad308b757b6d185c2c Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 11 May 2017 14:42:52 -0500 Subject: [PATCH 27/30] Doc wording --- .../Historical1SimpleDoubleAggPooledTopNScannerPrototype.java | 4 ++-- ...DimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java index 7cc3baeb108c..e10331453d60 100644 --- a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -30,8 +30,8 @@ import java.nio.ByteBuffer; /** - * Needs to extend {@link MagicAccessorBridge} because prototype specialization (including replacement of {@link Offset} - * occurrences with a subclass) may appear to access a private class. + * Needs to extend {@link MagicAccessorBridge} because prototype specialization (with {@link Offset} occurrences + * replaced with a subclass) may appear to access a private class. */ public class Historical1SimpleDoubleAggPooledTopNScannerPrototype extends MagicAccessorBridge implements Historical1AggPooledTopNScanner< diff --git a/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java index 9974004887d2..c6f6567ccbb6 100644 --- a/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -29,8 +29,8 @@ import java.nio.ByteBuffer; /** - * Needs to extend {@link MagicAccessorBridge} because prototype specialization (including replacement of {@link Offset} - * occurrences with a subclass) may appear to access a private class. + * Needs to extend {@link MagicAccessorBridge} because prototype specialization (with {@link Offset} occurrences + * replaced with a subclass) may appear to access a private class. */ public class HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype extends MagicAccessorBridge implements Historical1AggPooledTopNScanner< From 0a48d897317283c69e5024c61e6d110baf4d0c66 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 12 May 2017 16:49:54 -0500 Subject: [PATCH 28/30] Address comments --- .../java/io/druid/segment/column/DictionaryEncodedColumn.java | 4 ++-- .../src/main/java/io/druid/segment/filter/AndFilter.java | 3 ++- .../src/main/java/io/druid/segment/filter/OrFilter.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java index b307d37e1654..0a6cd57c8c2a 100644 --- a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java @@ -20,8 +20,8 @@ package io.druid.segment.column; import io.druid.query.extraction.ExtractionFn; +import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; -import io.druid.segment.historical.HistoricalDimensionSelector; import io.druid.segment.historical.OffsetHolder; import java.io.Closeable; @@ -38,5 +38,5 @@ public interface DictionaryEncodedColumn extends public int lookupId(ActualType name); public int getCardinality(); - HistoricalDimensionSelector makeDimensionSelector(OffsetHolder offsetHolder, ExtractionFn extractionFn); + DimensionSelector makeDimensionSelector(OffsetHolder offsetHolder, ExtractionFn extractionFn); } diff --git a/processing/src/main/java/io/druid/segment/filter/AndFilter.java b/processing/src/main/java/io/druid/segment/filter/AndFilter.java index dd66eee6f55a..5b993b263e6b 100644 --- a/processing/src/main/java/io/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/AndFilter.java @@ -40,6 +40,7 @@ public class AndFilter implements BooleanFilter { private static final Joiner AND_JOINER = Joiner.on(" && "); + static final ValueMatcher[] EMPTY_VALUE_MATCHER_ARRAY = new ValueMatcher[0]; private final List filters; @@ -113,7 +114,7 @@ public ValueMatcher makeMatcher( matchers.add(0, offsetMatcher); } - return makeMatcher(matchers.toArray(new ValueMatcher[0])); + return makeMatcher(matchers.toArray(EMPTY_VALUE_MATCHER_ARRAY)); } @Override diff --git a/processing/src/main/java/io/druid/segment/filter/OrFilter.java b/processing/src/main/java/io/druid/segment/filter/OrFilter.java index 44e8ba137263..67bf64597c40 100644 --- a/processing/src/main/java/io/druid/segment/filter/OrFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/OrFilter.java @@ -101,7 +101,7 @@ public ValueMatcher makeMatcher( matchers.add(0, offsetMatcher); } - return makeMatcher(matchers.toArray(new ValueMatcher[0])); + return makeMatcher(matchers.toArray(AndFilter.EMPTY_VALUE_MATCHER_ARRAY)); } From 331a32a08d063b99f4927f8de32cf6620debfcef Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 12 May 2017 17:57:57 -0500 Subject: [PATCH 29/30] Remove MagicAccessorBridge and ensure Offset subclasses are public --- .../annotations/SubclassesMustBePublic.java | 18 +++++-- ...lassesMustBePublicAnnotationProcessor.java | 50 +++++++++++++++++++ .../javax.annotation.processing.Processor | 1 + processing/pom.xml | 16 ++++++ ...leDoubleAggPooledTopNScannerPrototype.java | 7 +-- ...leDoubleAggPooledTopNScannerPrototype.java | 7 +-- .../java/io/druid/segment/FilteredOffset.java | 2 +- .../segment/QueryableIndexStorageAdapter.java | 8 +-- .../java/io/druid/segment/data/Offset.java | 7 +++ 9 files changed, 94 insertions(+), 22 deletions(-) rename processing/src/main/java/sun/reflect/MagicAccessorBridge.java => common/src/main/java/io/druid/annotations/SubclassesMustBePublic.java (67%) create mode 100644 common/src/main/java/io/druid/annotations/SubclassesMustBePublicAnnotationProcessor.java create mode 100644 common/src/main/resources/services/javax.annotation.processing.Processor diff --git a/processing/src/main/java/sun/reflect/MagicAccessorBridge.java b/common/src/main/java/io/druid/annotations/SubclassesMustBePublic.java similarity index 67% rename from processing/src/main/java/sun/reflect/MagicAccessorBridge.java rename to common/src/main/java/io/druid/annotations/SubclassesMustBePublic.java index 460cd97218fa..2d8771fe8f09 100644 --- a/processing/src/main/java/sun/reflect/MagicAccessorBridge.java +++ b/common/src/main/java/io/druid/annotations/SubclassesMustBePublic.java @@ -17,11 +17,19 @@ * under the License. */ -package sun.reflect; +package io.druid.annotations; -/** - * Public bridge is needed, because MagicAccessorImpl is a package-private class. - */ -public class MagicAccessorBridge extends MagicAccessorImpl +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Documented +@Inherited +@Retention(RetentionPolicy.SOURCE) +@Target(ElementType.TYPE) +public @interface SubclassesMustBePublic { } diff --git a/common/src/main/java/io/druid/annotations/SubclassesMustBePublicAnnotationProcessor.java b/common/src/main/java/io/druid/annotations/SubclassesMustBePublicAnnotationProcessor.java new file mode 100644 index 000000000000..3f8fb1dcd9c0 --- /dev/null +++ b/common/src/main/java/io/druid/annotations/SubclassesMustBePublicAnnotationProcessor.java @@ -0,0 +1,50 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.annotations; + +import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.RoundEnvironment; +import javax.annotation.processing.SupportedAnnotationTypes; +import javax.lang.model.element.Element; +import javax.lang.model.element.Modifier; +import javax.lang.model.element.TypeElement; +import javax.tools.Diagnostic; +import java.util.Set; + +@SupportedAnnotationTypes("io.druid.annotations.SubclassesMustBePublic") +public class SubclassesMustBePublicAnnotationProcessor extends AbstractProcessor +{ + @Override + public boolean process(Set annotations, RoundEnvironment roundEnv) + { + for (TypeElement annotation : annotations) { + Set elementsAnnotatedWith = roundEnv.getElementsAnnotatedWith(annotation); + for (Element element : elementsAnnotatedWith) { + if (!element.getModifiers().contains(Modifier.PUBLIC)) { + processingEnv.getMessager().printMessage( + Diagnostic.Kind.ERROR, + element.getSimpleName() + " must be public" + ); + } + } + } + return false; + } +} diff --git a/common/src/main/resources/services/javax.annotation.processing.Processor b/common/src/main/resources/services/javax.annotation.processing.Processor new file mode 100644 index 000000000000..b19a8c2290c3 --- /dev/null +++ b/common/src/main/resources/services/javax.annotation.processing.Processor @@ -0,0 +1 @@ +io.druid.annotations.SubclassesMustBePublicAnnotationProcessor \ No newline at end of file diff --git a/processing/pom.xml b/processing/pom.xml index ad9a805e6d15..469a9ce7cac0 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -132,6 +132,22 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + + io.druid + druid-common + ${project.parent.version} + + + + io.druid.annotations.SubclassesMustBePublicAnnotationProcessor + + + org.apache.maven.plugins maven-jar-plugin diff --git a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java index e10331453d60..e7d8aaf52a67 100644 --- a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -25,15 +25,10 @@ import io.druid.segment.historical.HistoricalCursor; import io.druid.segment.historical.HistoricalDimensionSelector; import io.druid.segment.historical.HistoricalFloatColumnSelector; -import sun.reflect.MagicAccessorBridge; import java.nio.ByteBuffer; -/** - * Needs to extend {@link MagicAccessorBridge} because prototype specialization (with {@link Offset} occurrences - * replaced with a subclass) may appear to access a private class. - */ -public class Historical1SimpleDoubleAggPooledTopNScannerPrototype extends MagicAccessorBridge +public class Historical1SimpleDoubleAggPooledTopNScannerPrototype implements Historical1AggPooledTopNScanner< HistoricalDimensionSelector, HistoricalFloatColumnSelector, diff --git a/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java index c6f6567ccbb6..91f77c4a8091 100644 --- a/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -24,15 +24,10 @@ import io.druid.segment.historical.HistoricalCursor; import io.druid.segment.historical.HistoricalFloatColumnSelector; import io.druid.segment.historical.SingleValueHistoricalDimensionSelector; -import sun.reflect.MagicAccessorBridge; import java.nio.ByteBuffer; -/** - * Needs to extend {@link MagicAccessorBridge} because prototype specialization (with {@link Offset} occurrences - * replaced with a subclass) may appear to access a private class. - */ -public class HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype extends MagicAccessorBridge +public class HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype implements Historical1AggPooledTopNScanner< SingleValueHistoricalDimensionSelector, HistoricalFloatColumnSelector, diff --git a/processing/src/main/java/io/druid/segment/FilteredOffset.java b/processing/src/main/java/io/druid/segment/FilteredOffset.java index 14daf44a7971..8152e1a791c1 100644 --- a/processing/src/main/java/io/druid/segment/FilteredOffset.java +++ b/processing/src/main/java/io/druid/segment/FilteredOffset.java @@ -32,7 +32,7 @@ import io.druid.segment.historical.OffsetHolder; import org.roaringbitmap.IntIterator; -final class FilteredOffset extends Offset +public final class FilteredOffset extends Offset { private Offset baseOffset; private final ValueMatcher filterMatcher; diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index 5b1b54a4f25d..f94aa788c654 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -823,7 +823,7 @@ public void reset() } } - private abstract static class TimestampCheckingOffset extends Offset + public abstract static class TimestampCheckingOffset extends Offset { protected final Offset baseOffset; protected final GenericColumn timestamps; @@ -885,7 +885,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) } } - private static class AscendingTimestampCheckingOffset extends TimestampCheckingOffset + public static class AscendingTimestampCheckingOffset extends TimestampCheckingOffset { public AscendingTimestampCheckingOffset( Offset baseOffset, @@ -917,7 +917,7 @@ public Offset clone() } } - private static class DescendingTimestampCheckingOffset extends TimestampCheckingOffset + public static class DescendingTimestampCheckingOffset extends TimestampCheckingOffset { public DescendingTimestampCheckingOffset( Offset baseOffset, @@ -950,7 +950,7 @@ public Offset clone() } } - private static class NoFilterOffset extends Offset + public static class NoFilterOffset extends Offset { private final int rowCount; private final boolean descending; diff --git a/processing/src/main/java/io/druid/segment/data/Offset.java b/processing/src/main/java/io/druid/segment/data/Offset.java index 47fe4a7d5813..7eff391e65d8 100644 --- a/processing/src/main/java/io/druid/segment/data/Offset.java +++ b/processing/src/main/java/io/druid/segment/data/Offset.java @@ -19,12 +19,19 @@ package io.druid.segment.data; +import io.druid.annotations.SubclassesMustBePublic; import io.druid.query.monomorphicprocessing.CalledFromHotLoop; /** * The "mutable" version of a ReadableOffset. Introduces "increment()" and "withinBounds()" methods, which are * very similar to "next()" and "hasNext()" on the Iterator interface except increment() does not return a value. + * + * Annotated with {@link SubclassesMustBePublic} because Offset occurrences are replaced with a subclass in {@link + * io.druid.query.topn.Historical1SimpleDoubleAggPooledTopNScannerPrototype} and {@link + * io.druid.query.topn.HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype} during + * specialization, and specialized version of those prototypes must be able to any subclass of Offset. */ +@SubclassesMustBePublic public abstract class Offset implements ReadableOffset, Cloneable { @CalledFromHotLoop From 69988b5a78e5b447bb6fcf51be2ac1d32309b2fb Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 12 May 2017 18:15:59 -0500 Subject: [PATCH 30/30] Attach error message to element --- .../annotations/SubclassesMustBePublicAnnotationProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/io/druid/annotations/SubclassesMustBePublicAnnotationProcessor.java b/common/src/main/java/io/druid/annotations/SubclassesMustBePublicAnnotationProcessor.java index 3f8fb1dcd9c0..2a8f30981072 100644 --- a/common/src/main/java/io/druid/annotations/SubclassesMustBePublicAnnotationProcessor.java +++ b/common/src/main/java/io/druid/annotations/SubclassesMustBePublicAnnotationProcessor.java @@ -40,7 +40,8 @@ public boolean process(Set annotations, RoundEnvironment if (!element.getModifiers().contains(Modifier.PUBLIC)) { processingEnv.getMessager().printMessage( Diagnostic.Kind.ERROR, - element.getSimpleName() + " must be public" + element.getSimpleName() + " must be public", + element ); } }