From 9359961a37636a348eac1286cca031169e213dc5 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 6 Jul 2018 15:00:09 +0200 Subject: [PATCH 01/29] Various changes --- .../main/java/io/druid/data/input/Row.java | 3 + .../main/java/io/druid/data/input/Rows.java | 11 +- .../java/io/druid/benchmark/package-info.java | 23 ++++ .../intset/ImmutableConciseSet.java | 7 -- .../DoublesSketchAggregatorFactory.java | 11 +- .../DoublesSketchComplexMetricSerde.java | 6 +- .../datasketches/quantiles/package-info.java | 23 ++++ .../theta/SketchBuildComplexMetricSerde.java | 6 +- ...y.java => SketchHolderObjectStrategy.java} | 27 ++--- .../theta/SketchMergeComplexMetricSerde.java | 32 +++--- .../datasketches/theta/package-info.java | 23 ++++ ...fDoublesSketchMergeComplexMetricSerde.java | 12 +- .../ArrayOfDoublesSketchObjectStrategy.java | 12 +- .../datasketches/tuple/package-info.java | 23 ++++ .../ApproximateHistogramAggregator.java | 11 +- .../ApproximateHistogramFoldingSerde.java | 31 ++---- .../aggregation/histogram/package-info.java | 23 ++++ .../namespace/UriExtractionNamespace.java | 4 +- .../aggregation/variance/VarianceSerde.java | 10 +- .../aggregation/variance/package-info.java | 23 ++++ .../io/druid/hll/HyperLogLogCollector.java | 12 +- ...va => VersionOneHyperLogLogCollector.java} | 6 +- ...a => VersionZeroHyperLogLogCollector.java} | 4 +- .../druid/hll/HyperLogLogCollectorTest.java | 10 +- .../hll/HyperLogLogSerdeBenchmarkTest.java | 6 +- .../java/io/druid/indexer/InputRowSerde.java | 1 + .../common/task/CompactionTaskTest.java | 2 +- .../bitmap/ConciseBitmapFactory.java | 2 +- .../bitmap/WrappedImmutableConciseBitmap.java | 6 +- .../collections/spatial/ImmutableRTree.java | 20 +++- .../query/aggregation/AggregateCombiner.java | 1 - .../query/aggregation/AggregatorUtil.java | 28 ++--- .../JavaScriptAggregatorFactory.java | 20 +++- .../SimpleDoubleAggregatorFactory.java | 9 +- .../SimpleFloatAggregatorFactory.java | 9 +- .../SimpleLongAggregatorFactory.java | 9 +- .../hyperloglog/HyperUniquesSerde.java | 26 ++--- .../aggregation/hyperloglog/package-info.java | 23 ++++ .../druid/query/aggregation/package-info.java | 23 ++++ .../query/aggregation/post/package-info.java | 23 ++++ .../druid/query/dimension/DimensionSpec.java | 3 + .../ForwardingFilteredDimensionSelector.java | 10 +- .../PredicateFilteredDimensionSelector.java | 10 +- .../io/druid/query/groupby/GroupByQuery.java | 1 + .../RowBasedColumnSelectorFactory.java | 103 +----------------- .../groupby/epinephelinae/package-info.java | 23 ++++ .../io/druid/query/groupby/package-info.java | 23 ++++ .../io/druid/query/scan/ScanQueryEngine.java | 46 ++++---- .../druid/query/search/SearchQueryRunner.java | 2 +- .../query/search/UseIndexesStrategy.java | 6 +- .../druid/query/topn/PooledTopNAlgorithm.java | 6 +- .../segment/AbstractDimensionSelector.java | 38 +++++++ .../java/io/druid/segment/AbstractIndex.java | 7 +- .../BaseObjectColumnValueSelector.java | 2 +- .../BaseSingleValueDimensionSelector.java | 12 +- .../ColumnSelectorBitmapIndexSelector.java | 4 +- ...nSerializer.java => ColumnSerializer.java} | 7 +- .../segment/ConstantColumnValueSelector.java | 10 +- .../druid/segment/DimensionHandlerUtils.java | 21 ++-- .../io/druid/segment/DimensionIndexer.java | 14 +-- .../io/druid/segment/DimensionSelector.java | 20 +++- .../druid/segment/DimensionSelectorUtils.java | 5 +- .../druid/segment/DoubleColumnSelector.java | 8 +- .../druid/segment/DoubleColumnSerializer.java | 4 +- .../segment/DoubleColumnSerializerV2.java | 6 +- .../druid/segment/DoubleDimensionIndexer.java | 2 +- .../segment/DoubleDimensionMergerV9.java | 2 +- .../DoubleWrappingDimensionSelector.java | 13 ++- .../io/druid/segment/FloatColumnSelector.java | 8 +- .../druid/segment/FloatColumnSerializer.java | 4 +- .../segment/FloatColumnSerializerV2.java | 6 +- .../druid/segment/FloatDimensionIndexer.java | 2 +- .../druid/segment/FloatDimensionMergerV9.java | 2 +- .../FloatWrappingDimensionSelector.java | 18 +-- .../main/java/io/druid/segment/IndexIO.java | 72 ++++++------ .../java/io/druid/segment/IndexMergerV9.java | 66 +++++------ .../io/druid/segment/LongColumnSelector.java | 8 +- .../druid/segment/LongColumnSerializer.java | 4 +- .../druid/segment/LongColumnSerializerV2.java | 6 +- .../druid/segment/LongDimensionIndexer.java | 2 +- .../druid/segment/LongDimensionMergerV9.java | 2 +- .../LongWrappingDimensionSelector.java | 18 +-- .../java/io/druid/segment/MetricHolder.java | 36 +++--- .../druid/segment/NullDimensionSelector.java | 4 +- .../segment/NumericDimensionMergerV9.java | 4 +- .../druid/segment/ObjectColumnSelector.java | 6 +- .../QueryableIndexColumnSelectorFactory.java | 4 +- .../QueryableIndexIndexableAdapter.java | 18 +-- .../segment/QueryableIndexStorageAdapter.java | 16 +-- ...a => SingleScanTimeDimensionSelector.java} | 35 +++--- .../druid/segment/StringDimensionIndexer.java | 32 ++---- .../io/druid/segment/TimeAndDimsPointer.java | 8 +- .../io/druid/segment/column/BaseColumn.java | 2 +- .../java/io/druid/segment/column/Column.java | 4 +- .../druid/segment/column/ColumnBuilder.java | 24 ++-- .../druid/segment/column/ComplexColumn.java | 41 +++++-- .../column/DictionaryEncodedColumn.java | 4 +- .../druid/segment/column/DoublesColumn.java | 4 +- .../column/DoublesColumnWithNulls.java | 2 +- .../io/druid/segment/column/FloatsColumn.java | 4 +- .../segment/column/FloatsColumnWithNulls.java | 2 +- .../io/druid/segment/column/LongsColumn.java | 7 +- .../segment/column/LongsColumnWithNulls.java | 2 +- ...{GenericColumn.java => NumericColumn.java} | 2 +- .../io/druid/segment/column/SimpleColumn.java | 26 ++--- ...ava => StringDictionaryEncodedColumn.java} | 32 +++--- .../io/druid/segment/column/ValueType.java | 22 ++-- ...edComplexColumn.java => package-info.java} | 40 +------ .../io/druid/segment/data/BitmapValues.java | 2 +- .../io/druid/segment/data/CachingIndexed.java | 11 +- .../io/druid/segment/data/ColumnarInts.java | 4 +- .../data/ConciseBitmapSerdeFactory.java | 2 +- .../io/druid/segment/data/GenericIndexed.java | 6 +- .../java/io/druid/segment/data/Indexed.java | 5 +- .../io/druid/segment/data/IndexedInts.java | 2 +- .../data/RoaringBitmapSerdeFactory.java | 5 +- ...eObjectStrategy.java => package-info.java} | 7 +- .../segment/incremental/IncrementalIndex.java | 4 +- ...IncrementalIndexColumnSelectorFactory.java | 4 +- .../incremental/OffheapIncrementalIndex.java | 2 +- .../incremental/OnheapIncrementalIndex.java | 15 +-- .../settable/SettableColumnValueSelector.java | 2 +- .../SettableDimensionValueSelector.java | 4 +- .../SettableDoubleColumnValueSelector.java | 2 +- .../SettableFloatColumnValueSelector.java | 2 +- .../SettableLongColumnValueSelector.java | 2 +- .../selector/settable/package-info.java | 23 ++++ .../druid/segment/serde/ColumnPartSerde.java | 14 ++- .../segment/serde/ComplexColumnPartSerde.java | 6 +- .../serde/ComplexColumnPartSupplier.java | 7 +- .../serde/ComplexColumnSerializer.java | 6 +- .../segment/serde/ComplexMetricExtractor.java | 10 +- .../segment/serde/ComplexMetricSerde.java | 6 +- .../druid/segment/serde/ComplexMetrics.java | 2 + .../DictionaryEncodedColumnPartSerde.java | 4 +- .../DictionaryEncodedColumnSupplier.java | 18 +-- ...java => DoubleNumericColumnPartSerde.java} | 22 ++-- ...va => DoubleNumericColumnPartSerdeV2.java} | 55 +++++----- ....java => DoubleNumericColumnSupplier.java} | 8 +- ....java => FloatNumericColumnPartSerde.java} | 25 +++-- ...ava => FloatNumericColumnPartSerdeV2.java} | 48 ++++---- ...r.java => FloatNumericColumnSupplier.java} | 8 +- ...olumnSupportedComplexColumnSerializer.java | 21 ++-- ...e.java => LongNumericColumnPartSerde.java} | 25 +++-- ...java => LongNumericColumnPartSerdeV2.java} | 49 +++++---- ...er.java => LongNumericColumnSupplier.java} | 8 +- .../segment/virtual/ExpressionSelectors.java | 1 + ...tCachingExpressionColumnValueSelector.java | 2 + .../SingleStringInputDimensionSelector.java | 4 +- .../druid/segment/virtual/package-info.java | 23 ++++ .../collections/bitmap/BitmapBenchmark.java | 2 +- .../aggregation/FilteredAggregatorTest.java | 10 +- .../CardinalityAggregatorTest.java | 10 +- .../HyperUniquesAggregatorFactoryTest.java | 4 +- .../dimension/TestDimensionSelector.java | 11 +- .../io/druid/segment/IndexMergerTestBase.java | 6 +- .../serde/HyperUniquesSerdeForTest.java | 6 +- .../ExpressionColumnValueSelectorTest.java | 1 + .../main/java/io/druid/cli/DumpSegment.java | 98 +++-------------- .../druid/sql/calcite/CalciteQueryTest.java | 42 +++---- 160 files changed, 1171 insertions(+), 1063 deletions(-) create mode 100644 benchmarks/src/main/java/io/druid/benchmark/package-info.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/package-info.java rename extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/{SketchObjectStrategy.java => SketchHolderObjectStrategy.java} (69%) create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/package-info.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/package-info.java create mode 100644 extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/package-info.java create mode 100644 extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/package-info.java rename hll/src/main/java/io/druid/hll/{HLLCV1.java => VersionOneHyperLogLogCollector.java} (96%) rename hll/src/main/java/io/druid/hll/{HLLCV0.java => VersionZeroHyperLogLogCollector.java} (96%) create mode 100644 processing/src/main/java/io/druid/query/aggregation/hyperloglog/package-info.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/package-info.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/post/package-info.java create mode 100644 processing/src/main/java/io/druid/query/groupby/epinephelinae/package-info.java create mode 100644 processing/src/main/java/io/druid/query/groupby/package-info.java create mode 100644 processing/src/main/java/io/druid/segment/AbstractDimensionSelector.java rename processing/src/main/java/io/druid/segment/{virtual => }/BaseSingleValueDimensionSelector.java (92%) rename processing/src/main/java/io/druid/segment/{GenericColumnSerializer.java => ColumnSerializer.java} (84%) rename processing/src/main/java/io/druid/segment/{SingleScanTimeDimSelector.java => SingleScanTimeDimensionSelector.java} (83%) rename processing/src/main/java/io/druid/segment/column/{GenericColumn.java => NumericColumn.java} (94%) rename processing/src/main/java/io/druid/segment/column/{SimpleDictionaryEncodedColumn.java => StringDictionaryEncodedColumn.java} (89%) rename processing/src/main/java/io/druid/segment/column/{IndexedComplexColumn.java => package-info.java} (58%) rename processing/src/main/java/io/druid/segment/data/{CacheableObjectStrategy.java => package-info.java} (78%) create mode 100644 processing/src/main/java/io/druid/segment/selector/settable/package-info.java rename processing/src/main/java/io/druid/segment/serde/{DoubleGenericColumnPartSerde.java => DoubleNumericColumnPartSerde.java} (79%) rename processing/src/main/java/io/druid/segment/serde/{DoubleGenericColumnPartSerdeV2.java => DoubleNumericColumnPartSerdeV2.java} (79%) rename processing/src/main/java/io/druid/segment/serde/{DoubleGenericColumnSupplier.java => DoubleNumericColumnSupplier.java} (86%) rename processing/src/main/java/io/druid/segment/serde/{FloatGenericColumnPartSerde.java => FloatNumericColumnPartSerde.java} (78%) rename processing/src/main/java/io/druid/segment/serde/{FloatGenericColumnPartSerdeV2.java => FloatNumericColumnPartSerdeV2.java} (80%) rename processing/src/main/java/io/druid/segment/serde/{FloatGenericColumnSupplier.java => FloatNumericColumnSupplier.java} (88%) rename processing/src/main/java/io/druid/segment/serde/{LongGenericColumnPartSerde.java => LongNumericColumnPartSerde.java} (78%) rename processing/src/main/java/io/druid/segment/serde/{LongGenericColumnPartSerdeV2.java => LongNumericColumnPartSerdeV2.java} (79%) rename processing/src/main/java/io/druid/segment/serde/{LongGenericColumnSupplier.java => LongNumericColumnSupplier.java} (87%) create mode 100644 processing/src/main/java/io/druid/segment/virtual/package-info.java diff --git a/api/src/main/java/io/druid/data/input/Row.java b/api/src/main/java/io/druid/data/input/Row.java index 4a314fb76e19..2d943330d06f 100644 --- a/api/src/main/java/io/druid/data/input/Row.java +++ b/api/src/main/java/io/druid/data/input/Row.java @@ -24,6 +24,7 @@ import io.druid.guice.annotations.PublicApi; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.List; /** @@ -71,6 +72,7 @@ public interface Row extends Comparable * * @return the value of the provided column name */ + @Nullable Object getRaw(String dimension); /** @@ -79,5 +81,6 @@ public interface Row extends Comparable * 1. If the column is absent in the row, numeric zero is returned, rather than null. * 2. If the column has string value, an attempt is made to parse this value as a number. */ + @Nullable Number getMetric(String metric); } diff --git a/api/src/main/java/io/druid/data/input/Rows.java b/api/src/main/java/io/druid/data/input/Rows.java index 0ef09e9fa246..56200aa3e3a1 100644 --- a/api/src/main/java/io/druid/data/input/Rows.java +++ b/api/src/main/java/io/druid/data/input/Rows.java @@ -26,11 +26,11 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.parsers.ParseException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** */ @@ -68,14 +68,7 @@ public static List objectToStrings(final Object inputValue) return Collections.emptyList(); } else if (inputValue instanceof List) { // guava's toString function fails on null objects, so please do not use it - final List values = (List) inputValue; - - final List retVal = new ArrayList<>(values.size()); - for (Object val : values) { - retVal.add(String.valueOf(val)); - } - - return retVal; + return ((List) inputValue).stream().map(String::valueOf).collect(Collectors.toList()); } else { return Collections.singletonList(String.valueOf(inputValue)); } diff --git a/benchmarks/src/main/java/io/druid/benchmark/package-info.java b/benchmarks/src/main/java/io/druid/benchmark/package-info.java new file mode 100644 index 000000000000..2efaefa21376 --- /dev/null +++ b/benchmarks/src/main/java/io/druid/benchmark/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.benchmark; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/extendedset/src/main/java/io/druid/extendedset/intset/ImmutableConciseSet.java b/extendedset/src/main/java/io/druid/extendedset/intset/ImmutableConciseSet.java index d6ea87b76a54..95e35445cbfd 100755 --- a/extendedset/src/main/java/io/druid/extendedset/intset/ImmutableConciseSet.java +++ b/extendedset/src/main/java/io/druid/extendedset/intset/ImmutableConciseSet.java @@ -791,13 +791,6 @@ public ImmutableConciseSet() this.size = 0; } - public ImmutableConciseSet(ByteBuffer byteBuffer) - { - this.words = byteBuffer.asIntBuffer(); - this.lastWordIndex = words.capacity() - 1; - this.size = calcSize(); - } - public ImmutableConciseSet(IntBuffer buffer) { this.words = buffer; diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index 199ec271a9a3..073b1d41bec5 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -44,6 +44,8 @@ public class DoublesSketchAggregatorFactory extends AggregatorFactory { + public static final Comparator COMPARATOR = + Comparator.nullsFirst(Comparator.comparingLong(DoublesSketch::getN)); private static final int DEFAULT_K = 128; @@ -121,15 +123,6 @@ public Object deserialize(final Object object) return DoublesSketchOperations.deserialize(object); } - public static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(DoublesSketch a, DoublesSketch b) - { - return Long.compare(a.getN(), b.getN()); - } - }; - @Override public Comparator getComparator() { diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java index ab54f7d0b0b6..22fba30abe72 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java @@ -24,7 +24,7 @@ import com.yahoo.sketches.quantiles.UpdateDoublesSketch; import io.druid.data.input.InputRow; import io.druid.java.util.common.IAE; -import io.druid.segment.GenericColumnSerializer; +import io.druid.segment.ColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -104,12 +104,12 @@ public Object extractValue(final InputRow inputRow, final String metricName) public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder) { final GenericIndexed column = GenericIndexed.read(buffer, strategy, builder.getFileMapper()); - builder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); + builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); } // support large columns @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/package-info.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/package-info.java new file mode 100644 index 000000000000..afb0d18750a0 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.aggregation.datasketches.quantiles; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBuildComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBuildComplexMetricSerde.java index 85841116fc5b..d555e77d6c34 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBuildComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBuildComplexMetricSerde.java @@ -27,13 +27,13 @@ public class SketchBuildComplexMetricSerde extends SketchMergeComplexMetricSerde { @Override - public ComplexMetricExtractor getExtractor() + public ComplexMetricExtractor getExtractor() { - return new ComplexMetricExtractor() + return new ComplexMetricExtractor() { @Override - public Class extractedClass() + public Class extractedClass() { return Object.class; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchHolderObjectStrategy.java similarity index 69% rename from extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchHolderObjectStrategy.java index 714c5d73bca4..df8054f43677 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchHolderObjectStrategy.java @@ -21,30 +21,29 @@ import com.yahoo.memory.Memory; import com.yahoo.sketches.theta.Sketch; -import io.druid.java.util.common.IAE; import io.druid.segment.data.ObjectStrategy; +import it.unimi.dsi.fastutil.bytes.ByteArrays; +import javax.annotation.Nullable; import java.nio.ByteBuffer; -public class SketchObjectStrategy implements ObjectStrategy +public class SketchHolderObjectStrategy implements ObjectStrategy { - private static final byte[] EMPTY_BYTES = new byte[]{}; - @Override - public int compare(Object s1, Object s2) + public int compare(SketchHolder s1, SketchHolder s2) { return SketchHolder.COMPARATOR.compare(s1, s2); } @Override - public Class getClazz() + public Class getClazz() { - return Object.class; + return SketchHolder.class; } @Override - public Object fromByteBuffer(ByteBuffer buffer, int numBytes) + public SketchHolder fromByteBuffer(ByteBuffer buffer, int numBytes) { if (numBytes == 0) { return SketchHolder.EMPTY; @@ -54,18 +53,16 @@ public Object fromByteBuffer(ByteBuffer buffer, int numBytes) } @Override - public byte[] toBytes(Object obj) + public byte[] toBytes(@Nullable SketchHolder obj) { - if (obj instanceof SketchHolder) { - Sketch sketch = ((SketchHolder) obj).getSketch(); + if (obj != null) { + Sketch sketch = obj.getSketch(); if (sketch.isEmpty()) { - return EMPTY_BYTES; + return ByteArrays.EMPTY_ARRAY; } return sketch.toByteArray(); - } else if (obj == null) { - return EMPTY_BYTES; } else { - throw new IAE("Unknown class[%s], toString[%s]", obj.getClass(), obj); + return ByteArrays.EMPTY_ARRAY; } } } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java index 6be52eb405ad..cade1310e7a7 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java @@ -19,10 +19,8 @@ package io.druid.query.aggregation.datasketches.theta; -import com.yahoo.sketches.theta.Sketch; import io.druid.data.input.InputRow; -import io.druid.segment.writeout.SegmentWriteOutMedium; -import io.druid.segment.GenericColumnSerializer; +import io.druid.segment.ColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -30,12 +28,14 @@ import io.druid.segment.serde.ComplexMetricExtractor; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import javax.annotation.Nullable; import java.nio.ByteBuffer; public class SketchMergeComplexMetricSerde extends ComplexMetricSerde { - private SketchObjectStrategy strategy = new SketchObjectStrategy(); + private SketchHolderObjectStrategy strategy = new SketchHolderObjectStrategy(); @Override public String getTypeName() @@ -44,24 +44,22 @@ public String getTypeName() } @Override - public ComplexMetricExtractor getExtractor() + public ComplexMetricExtractor getExtractor() { - return new ComplexMetricExtractor() + return new ComplexMetricExtractor() { @Override - public Class extractedClass() + public Class extractedClass() { - return Object.class; + return SketchHolder.class; } @Override - public Object extractValue(InputRow inputRow, String metricName) + @Nullable + public SketchHolder extractValue(InputRow inputRow, String metricName) { final Object object = inputRow.getRaw(metricName); - if (object == null) { - return object; - } - return SketchHolder.deserialize(object); + return object == null ? null : SketchHolder.deserialize(object); } }; } @@ -69,18 +67,18 @@ public Object extractValue(InputRow inputRow, String metricName) @Override public void deserializeColumn(ByteBuffer buffer, ColumnBuilder builder) { - GenericIndexed ge = GenericIndexed.read(buffer, strategy, builder.getFileMapper()); - builder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), ge)); + GenericIndexed ge = GenericIndexed.read(buffer, strategy, builder.getFileMapper()); + builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), ge)); } @Override - public ObjectStrategy getObjectStrategy() + public ObjectStrategy getObjectStrategy() { return strategy; } @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/package-info.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/package-info.java new file mode 100644 index 000000000000..6ab2f4ffe78a --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.aggregation.datasketches.theta; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java index 06fcedf4405d..ce5c98b63314 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java @@ -19,10 +19,9 @@ package io.druid.query.aggregation.datasketches.tuple; -import java.nio.ByteBuffer; - +import com.yahoo.sketches.tuple.ArrayOfDoublesSketch; import io.druid.data.input.InputRow; -import io.druid.segment.GenericColumnSerializer; +import io.druid.segment.ColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -32,7 +31,7 @@ import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; import io.druid.segment.writeout.SegmentWriteOutMedium; -import com.yahoo.sketches.tuple.ArrayOfDoublesSketch; +import java.nio.ByteBuffer; public class ArrayOfDoublesSketchMergeComplexMetricSerde extends ComplexMetricSerde { @@ -70,7 +69,7 @@ public Object extractValue(final InputRow inputRow, final String metricName) public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder) { final GenericIndexed ge = GenericIndexed.read(buffer, ArrayOfDoublesSketchObjectStrategy.STRATEGY); - builder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), ge)); + builder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), ge)); } @Override @@ -79,9 +78,8 @@ public ObjectStrategy getObjectStrategy() return ArrayOfDoublesSketchObjectStrategy.STRATEGY; } - // support large columns @Override - public GenericColumnSerializer getSerializer(final SegmentWriteOutMedium segmentWriteOutMedium, final String column) + public ColumnSerializer getSerializer(final SegmentWriteOutMedium segmentWriteOutMedium, final String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchObjectStrategy.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchObjectStrategy.java index 4aadba5215a2..bedc328c42b5 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchObjectStrategy.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchObjectStrategy.java @@ -19,15 +19,13 @@ package io.druid.query.aggregation.datasketches.tuple; -import io.druid.segment.data.ObjectStrategy; - -import java.nio.ByteBuffer; - -import javax.annotation.Nullable; - import com.yahoo.memory.Memory; import com.yahoo.sketches.tuple.ArrayOfDoublesSketch; import com.yahoo.sketches.tuple.ArrayOfDoublesSketches; +import io.druid.segment.data.ObjectStrategy; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; public class ArrayOfDoublesSketchObjectStrategy implements ObjectStrategy { @@ -41,7 +39,7 @@ public int compare(final ArrayOfDoublesSketch s1, final ArrayOfDoublesSketch s2) } @Override - public Class getClazz() + public Class getClazz() { return ArrayOfDoublesSketch.class; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/package-info.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/package-info.java new file mode 100644 index 000000000000..c7cba4854aca --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.aggregation.datasketches.tuple; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java index 6ed641d79c98..87abb461b0b1 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java @@ -19,7 +19,6 @@ package io.druid.query.aggregation.histogram; -import com.google.common.primitives.Longs; import io.druid.query.aggregation.Aggregator; import io.druid.segment.BaseFloatColumnValueSelector; @@ -27,14 +26,8 @@ public class ApproximateHistogramAggregator implements Aggregator { - public static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return Longs.compare(((ApproximateHistogram) o).count(), ((ApproximateHistogram) o1).count()); - } - }; + public static final Comparator COMPARATOR = + Comparator.nullsFirst(Comparator.comparingLong(ApproximateHistogram::count)); static ApproximateHistogram combineHistograms(Object lhs, Object rhs) { diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java index 7d2abf691fa5..8d1574e287e4 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java @@ -19,10 +19,9 @@ package io.druid.query.aggregation.histogram; -import com.google.common.collect.Ordering; import io.druid.data.input.InputRow; import io.druid.data.input.Rows; -import io.druid.segment.GenericColumnSerializer; +import io.druid.segment.ColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -31,22 +30,13 @@ import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; import io.druid.segment.writeout.SegmentWriteOutMedium; +import it.unimi.dsi.fastutil.bytes.ByteArrays; import java.nio.ByteBuffer; import java.util.Collection; public class ApproximateHistogramFoldingSerde extends ComplexMetricSerde { - private static Ordering comparator = new Ordering() - { - @Override - public int compare( - ApproximateHistogram arg1, ApproximateHistogram arg2 - ) - { - return ApproximateHistogramAggregator.COMPARATOR.compare(arg1, arg2); - } - }.nullsFirst(); @Override public String getTypeName() @@ -94,22 +84,21 @@ public ApproximateHistogram extractValue(InputRow inputRow, String metricName) } @Override - public void deserializeColumn( - ByteBuffer byteBuffer, ColumnBuilder columnBuilder - ) + public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder) { - final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); - columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); + final GenericIndexed column = + GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); + columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); } @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } @Override - public ObjectStrategy getObjectStrategy() + public ObjectStrategy getObjectStrategy() { return new ObjectStrategy() { @@ -130,7 +119,7 @@ public ApproximateHistogram fromByteBuffer(ByteBuffer buffer, int numBytes) public byte[] toBytes(ApproximateHistogram h) { if (h == null) { - return new byte[]{}; + return ByteArrays.EMPTY_ARRAY; } return h.toBytes(); } @@ -138,7 +127,7 @@ public byte[] toBytes(ApproximateHistogram h) @Override public int compare(ApproximateHistogram o1, ApproximateHistogram o2) { - return comparator.compare(o1, o2); + return ApproximateHistogramAggregator.COMPARATOR.compare(o1, o2); } }; } diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/package-info.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/package-info.java new file mode 100644 index 000000000000..96299e0dcbe1 --- /dev/null +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.aggregation.histogram; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java index ff0a93c4da2c..0042d85b0138 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java @@ -415,13 +415,13 @@ public TSVFlatDataParser( columns.contains(this.keyColumn), "Column [%s] not found int columns: %s", this.keyColumn, - Arrays.toString(columns.toArray()) + columns ); Preconditions.checkArgument( columns.contains(this.valueColumn), "Column [%s] not found int columns: %s", this.valueColumn, - Arrays.toString(columns.toArray()) + columns ); this.parser = new DelegateParser(delegate, this.keyColumn, this.valueColumn); diff --git a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java index cc41bf73bfd8..3e3dc8185434 100644 --- a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java +++ b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java @@ -21,8 +21,8 @@ import com.google.common.collect.Ordering; import io.druid.data.input.InputRow; +import io.druid.segment.ColumnSerializer; import io.druid.segment.writeout.SegmentWriteOutMedium; -import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -81,12 +81,10 @@ public VarianceAggregatorCollector extractValue(InputRow inputRow, String metric } @Override - public void deserializeColumn( - ByteBuffer byteBuffer, ColumnBuilder columnBuilder - ) + public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder) { final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); - columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); + columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); } @Override @@ -122,7 +120,7 @@ public int compare(VarianceAggregatorCollector o1, VarianceAggregatorCollector o } @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/package-info.java b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/package-info.java new file mode 100644 index 000000000000..543848b165cc --- /dev/null +++ b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.aggregation.variance; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/hll/src/main/java/io/druid/hll/HyperLogLogCollector.java b/hll/src/main/java/io/druid/hll/HyperLogLogCollector.java index bb26587abaa8..6f1241c50b80 100644 --- a/hll/src/main/java/io/druid/hll/HyperLogLogCollector.java +++ b/hll/src/main/java/io/druid/hll/HyperLogLogCollector.java @@ -93,7 +93,7 @@ public abstract class HyperLogLogCollector implements Comparable getParameters() ); } - private static final class priorByteBufferSerializer extends HLLCV1 + private static final class priorByteBufferSerializer extends VersionOneHyperLogLogCollector { @Override public ByteBuffer toByteBuffer() @@ -108,7 +108,7 @@ public ByteBuffer toByteBuffer() } } - private static final class newByteBufferSerializer extends HLLCV1 + private static final class newByteBufferSerializer extends VersionOneHyperLogLogCollector { @Override public ByteBuffer toByteBuffer() @@ -163,7 +163,7 @@ public ByteBuffer toByteBuffer() } - private static final class newByteBufferSerializerWithPuts extends HLLCV1 + private static final class newByteBufferSerializerWithPuts extends VersionOneHyperLogLogCollector { @Override public ByteBuffer toByteBuffer() diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index f80c4c68a58d..6a746d24c82c 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -341,6 +341,7 @@ public static final SerializeResult toBytes( //its a complex metric Object val = agg.get(); ComplexMetricSerde serde = getComplexMetricSerde(t); + //noinspection unchecked writeBytes(serde.toBytes(val), out); } } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java index e83999f37a93..dc4f814f5830 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java @@ -695,7 +695,7 @@ private static Column createColumn(DimensionSchema dimensionSchema) { return new ColumnBuilder() .setType(IncrementalIndex.TYPE_MAP.get(dimensionSchema.getValueType())) - .setDictionaryEncodedColumn(() -> null) + .setDictionaryEncodedColumnSupplier(() -> null) .setBitmapIndex(() -> null) .build(); } diff --git a/processing/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java b/processing/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java index 7e2af749130c..83af2bec4b01 100644 --- a/processing/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java +++ b/processing/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java @@ -105,7 +105,7 @@ public ImmutableBitmap makeImmutableBitmap(MutableBitmap mutableBitmap) @Override public ImmutableBitmap mapImmutableBitmap(ByteBuffer b) { - return new WrappedImmutableConciseBitmap(b); + return new WrappedImmutableConciseBitmap(b.asIntBuffer()); } @Override diff --git a/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java index 349fb493743e..6346c0a479cf 100644 --- a/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java +++ b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java @@ -23,7 +23,7 @@ import io.druid.extendedset.intset.ImmutableConciseSet; import org.roaringbitmap.IntIterator; -import java.nio.ByteBuffer; +import java.nio.IntBuffer; public class WrappedImmutableConciseBitmap implements ImmutableBitmap { @@ -32,9 +32,9 @@ public class WrappedImmutableConciseBitmap implements ImmutableBitmap */ private final ImmutableConciseSet bitmap; - public WrappedImmutableConciseBitmap(ByteBuffer byteBuffer) + public WrappedImmutableConciseBitmap(IntBuffer buffer) { - this.bitmap = new ImmutableConciseSet(byteBuffer); + this.bitmap = new ImmutableConciseSet(buffer); } /** diff --git a/processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java b/processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java index b1f174e99f26..b276e24ffd73 100644 --- a/processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java +++ b/processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java @@ -36,7 +36,7 @@ /** * An immutable representation of an {@link RTree} for spatial indexing. */ -public final class ImmutableRTree +public final class ImmutableRTree implements Comparable { private static final byte VERSION = 0x0; @@ -61,7 +61,6 @@ private ImmutableRTree() public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory) { - data = data.asReadOnlyBuffer(); final int initPosition = data.position(); Preconditions.checkArgument(data.get(initPosition) == VERSION, "Mismatching versions"); this.numDims = data.getInt(1 + initPosition) & 0x7FFF; @@ -75,7 +74,7 @@ public static ImmutableRTree newImmutableFromMutable(RTree rTree) return empty(); } - ByteBuffer buffer = ByteBuffer.wrap(new byte[calcNumBytes(rTree)]); + ByteBuffer buffer = ByteBuffer.allocate(calcNumBytes(rTree)); buffer.put(VERSION); buffer.putInt(rTree.getNumDims()); @@ -128,7 +127,7 @@ public Iterable search(SearchStrategy strategy, Bound bound) return strategy.search(root, bound); } else { // If the dimension counts don't match (for example, if this is called on a blank `new ImmutableRTree()`) - return ImmutableList.of(); + return ImmutableList.of(); } } @@ -149,8 +148,21 @@ public void writeTo(WriteOutBytes out) throws IOException } } + @Override public int compareTo(ImmutableRTree other) { return this.data.compareTo(other.data); } + + @Override + public boolean equals(Object obj) + { + return obj instanceof ImmutableRTree && data.equals(((ImmutableRTree) obj).data); + } + + @Override + public int hashCode() + { + return data.hashCode(); + } } diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java index 797f44843837..2dba0228c1ef 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java @@ -67,7 +67,6 @@ public interface AggregateCombiner extends ColumnValueSelector * * @see AggregatorFactory#combine */ - @SuppressWarnings("unused") // Going to be used when https://github.com/druid-io/druid/projects/2 is complete void fold(ColumnValueSelector selector); @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index 833eed385602..768452965bc9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -37,6 +37,7 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.virtual.ExpressionSelectors; +import javax.annotation.Nullable; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -97,8 +98,8 @@ public class AggregatorUtil /** * returns the list of dependent postAggregators that should be calculated in order to calculate given postAgg * - * @param postAggregatorList List of postAggregator, there is a restriction that the list should be in an order - * such that all the dependencies of any given aggregator should occur before that aggregator. + * @param postAggregatorList List of postAggregator, there is a restriction that the list should be in an order such + * that all the dependencies of any given aggregator should occur before that aggregator. * See AggregatorUtilTest.testOutOfOrderPruneDependentPostAgg for example. * @param postAggName name of the postAgg on which dependency is to be calculated * @@ -109,7 +110,8 @@ public static List pruneDependentPostAgg(List po LinkedList rv = Lists.newLinkedList(); Set deps = new HashSet<>(); deps.add(postAggName); - // Iterate backwards to find the last calculated aggregate and add dependent aggregator as we find dependencies in reverse order + // Iterate backwards to find the last calculated aggregate and add dependent aggregator as we find dependencies in + // reverse order for (PostAggregator agg : Lists.reverse(postAggregatorList)) { if (deps.contains(agg.getName())) { rv.addFirst(agg); // add to the beginning of List @@ -145,14 +147,14 @@ public static Pair, List> condensedAggre condensedAggs.add(aggregatorSpec); } } - return new Pair(condensedAggs, condensedPostAggs); + return new Pair<>(condensedAggs, condensedPostAggs); } - public static BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefault( + static BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, - final String fieldName, - final String fieldExpression, + @Nullable final String fieldName, + @Nullable final String fieldExpression, final float nullValue ) { @@ -189,11 +191,11 @@ public boolean isNull() throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } - public static BaseLongColumnValueSelector makeColumnValueSelectorWithLongDefault( + static BaseLongColumnValueSelector makeColumnValueSelectorWithLongDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, - final String fieldName, - final String fieldExpression, + @Nullable final String fieldName, + @Nullable final String fieldExpression, final long nullValue ) { @@ -230,11 +232,11 @@ public boolean isNull() throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } - public static BaseDoubleColumnValueSelector makeColumnValueSelectorWithDoubleDefault( + static BaseDoubleColumnValueSelector makeColumnValueSelectorWithDoubleDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, - final String fieldName, - final String fieldExpression, + @Nullable final String fieldName, + @Nullable final String fieldExpression, final double nullValue ) { diff --git a/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java index c246cda8ddcb..c1a096925fa8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java @@ -32,6 +32,7 @@ import io.druid.segment.BaseObjectColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; +import it.unimi.dsi.fastutil.objects.ObjectArrays; import org.mozilla.javascript.Context; import org.mozilla.javascript.ContextAction; import org.mozilla.javascript.ContextFactory; @@ -338,11 +339,11 @@ public double aggregate(final double current, final BaseObjectColumnValueSelecto if (arg != null && arg.getClass().isArray()) { // Context.javaToJS on an array sort of works, although it returns false for Array.isArray(...) and // may have other issues too. Let's just copy the array and wrap that. - final Object[] arrayAsObjectArray = new Object[Array.getLength(arg)]; - for (int j = 0; j < Array.getLength(arg); j++) { - arrayAsObjectArray[j] = Array.get(arg, j); - } - args[i + 1] = cx.newArray(scope, arrayAsObjectArray); + args[i + 1] = cx.newArray(scope, arrayToObjectArray(arg)); + } else if (arg instanceof List) { + // Using toArray(Object[]), instead of just toArray(), because Arrays.asList()'s impl and similar List + // impls could clone the underlying array in toArray(), that could be not Object[], but e. g. String[]. + args[i + 1] = cx.newArray(scope, ((List) arg).toArray(ObjectArrays.EMPTY_ARRAY)); } else { args[i + 1] = Context.javaToJS(arg, scope); } @@ -353,6 +354,15 @@ public double aggregate(final double current, final BaseObjectColumnValueSelecto return Context.toNumber(res); } + private Object[] arrayToObjectArray(Object array) + { + final Object[] objectArray = new Object[Array.getLength(array)]; + for (int j = 0; j < Array.getLength(array); j++) { + objectArray[j] = Array.get(array, j); + } + return objectArray; + } + @Override public double combine(final double a, final double b) { diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index af40149fa2c7..add62c8a645c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -28,6 +28,7 @@ import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -36,7 +37,9 @@ public abstract class SimpleDoubleAggregatorFactory extends AggregatorFactory { protected final String name; + @Nullable protected final String fieldName; + @Nullable protected final String expression; protected final ExprMacroTable macroTable; protected final boolean storeDoubleAsFloat; @@ -44,8 +47,8 @@ public abstract class SimpleDoubleAggregatorFactory extends AggregatorFactory public SimpleDoubleAggregatorFactory( ExprMacroTable macroTable, String name, - final String fieldName, - String expression + @Nullable final String fieldName, + @Nullable String expression ) { this.macroTable = macroTable; @@ -163,12 +166,14 @@ public String getName() return name; } + @Nullable @JsonProperty public String getFieldName() { return fieldName; } + @Nullable @JsonProperty public String getExpression() { diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java index 656996162280..4b719326fb44 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -27,6 +27,7 @@ import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -35,15 +36,17 @@ public abstract class SimpleFloatAggregatorFactory extends AggregatorFactory { protected final String name; + @Nullable protected final String fieldName; + @Nullable protected final String expression; protected final ExprMacroTable macroTable; public SimpleFloatAggregatorFactory( ExprMacroTable macroTable, String name, - final String fieldName, - String expression + @Nullable final String fieldName, + @Nullable String expression ) { this.macroTable = macroTable; @@ -157,12 +160,14 @@ public String getName() return name; } + @Nullable @JsonProperty public String getFieldName() { return fieldName; } + @Nullable @JsonProperty public String getExpression() { diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleLongAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleLongAggregatorFactory.java index 04c286fe6ee3..eb179a6e41bd 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleLongAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleLongAggregatorFactory.java @@ -27,6 +27,7 @@ import io.druid.segment.BaseLongColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -35,15 +36,17 @@ public abstract class SimpleLongAggregatorFactory extends AggregatorFactory { protected final String name; + @Nullable protected final String fieldName; + @Nullable protected final String expression; protected final ExprMacroTable macroTable; public SimpleLongAggregatorFactory( ExprMacroTable macroTable, String name, - final String fieldName, - String expression + @Nullable final String fieldName, + @Nullable String expression ) { this.macroTable = macroTable; @@ -153,12 +156,14 @@ public String getName() return name; } + @Nullable @JsonProperty public String getFieldName() { return fieldName; } + @Nullable @JsonProperty public String getExpression() { diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java index 93b5aacea0ab..2becde2091ec 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java @@ -19,12 +19,10 @@ package io.druid.query.aggregation.hyperloglog; -import com.google.common.collect.Ordering; import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogCollector; import io.druid.hll.HyperLogLogHash; -import io.druid.segment.writeout.SegmentWriteOutMedium; -import io.druid.segment.GenericColumnSerializer; +import io.druid.segment.ColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -32,22 +30,16 @@ import io.druid.segment.serde.ComplexMetricExtractor; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.nio.ByteBuffer; +import java.util.Comparator; import java.util.List; public class HyperUniquesSerde extends ComplexMetricSerde { - private static Ordering comparator = new Ordering() - { - @Override - public int compare( - HyperLogLogCollector arg1, HyperLogLogCollector arg2 - ) - { - return arg1.toByteBuffer().compareTo(arg2.toByteBuffer()); - } - }.nullsFirst(); + private static Comparator comparator = + Comparator.nullsFirst(Comparator.comparing(HyperLogLogCollector::toByteBuffer)); private final HyperLogLogHash hyperLogLogHash; @@ -63,9 +55,9 @@ public String getTypeName() } @Override - public ComplexMetricExtractor getExtractor() + public ComplexMetricExtractor getExtractor() { - return new ComplexMetricExtractor() + return new ComplexMetricExtractor() { @Override public Class extractedClass() @@ -103,7 +95,7 @@ public void deserializeColumn( ) { final GenericIndexed column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); - columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); + columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); } @Override @@ -148,7 +140,7 @@ public int compare(HyperLogLogCollector o1, HyperLogLogCollector o2) } @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/package-info.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/package-info.java new file mode 100644 index 000000000000..44d62d8b2bb7 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.aggregation.hyperloglog; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/processing/src/main/java/io/druid/query/aggregation/package-info.java b/processing/src/main/java/io/druid/query/aggregation/package-info.java new file mode 100644 index 000000000000..9c684b73bbce --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.aggregation; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/package-info.java b/processing/src/main/java/io/druid/query/aggregation/post/package-info.java new file mode 100644 index 000000000000..3a599b6a3f2b --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/post/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.aggregation.post; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java index 33f4de0fdbf5..3d492a9ff233 100644 --- a/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java @@ -26,6 +26,8 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.column.ValueType; +import javax.annotation.Nullable; + /** * Provides information about a dimension for a grouping query, like topN or groupBy. Note that this is not annotated * with {@code PublicApi}, since it is not meant to be stable for usage by non-built-in queries. @@ -47,6 +49,7 @@ public interface DimensionSpec extends Cacheable //ExtractionFn can be implemented with decorate(..) fn @Deprecated + @Nullable ExtractionFn getExtractionFn(); DimensionSelector decorate(DimensionSelector selector); diff --git a/processing/src/main/java/io/druid/query/dimension/ForwardingFilteredDimensionSelector.java b/processing/src/main/java/io/druid/query/dimension/ForwardingFilteredDimensionSelector.java index e64c10e0dc86..1b5e1a189f14 100644 --- a/processing/src/main/java/io/druid/query/dimension/ForwardingFilteredDimensionSelector.java +++ b/processing/src/main/java/io/druid/query/dimension/ForwardingFilteredDimensionSelector.java @@ -25,6 +25,7 @@ import io.druid.java.util.common.IAE; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.AbstractDimensionSelector; import io.druid.segment.DimensionSelector; import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.IdLookup; @@ -36,7 +37,7 @@ import javax.annotation.Nullable; import java.util.BitSet; -final class ForwardingFilteredDimensionSelector implements DimensionSelector, IdLookup +final class ForwardingFilteredDimensionSelector extends AbstractDimensionSelector implements IdLookup { private final DimensionSelector selector; private final IdLookup baseIdLookup; @@ -194,13 +195,6 @@ public int lookupId(String name) return forwardMapping.get(baseIdLookup.lookupId(name)); } - @Nullable - @Override - public Object getObject() - { - return defaultGetObject(); - } - @Override public Class classOfObject() { diff --git a/processing/src/main/java/io/druid/query/dimension/PredicateFilteredDimensionSelector.java b/processing/src/main/java/io/druid/query/dimension/PredicateFilteredDimensionSelector.java index 32143a857230..d49dfd9daa66 100644 --- a/processing/src/main/java/io/druid/query/dimension/PredicateFilteredDimensionSelector.java +++ b/processing/src/main/java/io/druid/query/dimension/PredicateFilteredDimensionSelector.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicate; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.AbstractDimensionSelector; import io.druid.segment.DimensionSelector; import io.druid.segment.IdLookup; import io.druid.segment.data.ArrayBasedIndexedInts; @@ -30,7 +31,7 @@ import javax.annotation.Nullable; import java.util.Objects; -final class PredicateFilteredDimensionSelector implements DimensionSelector +final class PredicateFilteredDimensionSelector extends AbstractDimensionSelector { private final DimensionSelector selector; private final Predicate predicate; @@ -152,13 +153,6 @@ public IdLookup idLookup() return selector.idLookup(); } - @Nullable - @Override - public Object getObject() - { - return defaultGetObject(); - } - @Override public Class classOfObject() { diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index ade15e96cf69..c2855b4d7bc7 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -514,6 +514,7 @@ public Ordering getRowOrdering(final boolean granular) } } + @Nullable private Comparator getTimeComparator(boolean granular) { if (Granularities.ALL.equals(getGranularity())) { diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java index d2fad0ad8566..a66a766fdf39 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java @@ -28,6 +28,7 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.BaseSingleValueDimensionSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionHandlerUtils; @@ -40,7 +41,6 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.RangeIndexedInts; -import io.druid.segment.data.ZeroIndexedInts; import javax.annotation.Nullable; import java.util.List; @@ -74,17 +74,7 @@ public static RowBasedColumnSelectorFactory create( @Nullable final Map rowSignature ) { - return new RowBasedColumnSelectorFactory( - new Supplier() - { - @Override - public Row get() - { - return row.get(); - } - }, - rowSignature - ); + return new RowBasedColumnSelectorFactory(row::get, rowSignature); } @Override @@ -105,95 +95,14 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi throw new UnsupportedOperationException("time dimension must provide an extraction function"); } - return new DimensionSelector() + return new BaseSingleValueDimensionSelector() { @Override - public IndexedInts getRow() - { - return ZeroIndexedInts.instance(); - } - - @Override - public ValueMatcher makeValueMatcher(final String value) - { - return new ValueMatcher() - { - @Override - public boolean matches() - { - String rowValue = extractionFn.apply(row.get().getTimestampFromEpoch()); - return Objects.equals(rowValue, value); - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("row", row); - inspector.visit("extractionFn", extractionFn); - } - }; - } - - @Override - public ValueMatcher makeValueMatcher(final Predicate predicate) - { - return new ValueMatcher() - { - @Override - public boolean matches() - { - String rowValue = extractionFn.apply(row.get().getTimestampFromEpoch()); - return predicate.apply(rowValue); - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("row", row); - inspector.visit("extractionFn", extractionFn); - inspector.visit("predicate", predicate); - } - }; - } - - @Override - public int getValueCardinality() - { - return DimensionSelector.CARDINALITY_UNKNOWN; - } - - @Override - public String lookupName(int id) + protected String getValue() { return extractionFn.apply(row.get().getTimestampFromEpoch()); } - @Override - public boolean nameLookupPossibleInAdvance() - { - return false; - } - - @Nullable - @Override - public IdLookup idLookup() - { - return null; - } - - @Nullable - @Override - public Object getObject() - { - return lookupName(0); - } - - @Override - public Class classOfObject() - { - return String.class; - } - @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -215,7 +124,7 @@ public IndexedInts getRow() } @Override - public ValueMatcher makeValueMatcher(final String value) + public ValueMatcher makeValueMatcher(final @Nullable String value) { if (extractionFn == null) { return new ValueMatcher() @@ -367,7 +276,7 @@ public Object getObject() if (dimensionValues.size() == 1) { return dimensionValues.get(0); } - return dimensionValues.toArray(new String[0]); + return dimensionValues; } @Override diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/package-info.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/package-info.java new file mode 100644 index 000000000000..ebdc7455142b --- /dev/null +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.groupby.epinephelinae; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/processing/src/main/java/io/druid/query/groupby/package-info.java b/processing/src/main/java/io/druid/query/groupby/package-info.java new file mode 100644 index 000000000000..6451cc6747cb --- /dev/null +++ b/processing/src/main/java/io/druid/query/groupby/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.query.groupby; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/processing/src/main/java/io/druid/query/scan/ScanQueryEngine.java b/processing/src/main/java/io/druid/query/scan/ScanQueryEngine.java index 43bb8846c5df..693d2262f66a 100644 --- a/processing/src/main/java/io/druid/query/scan/ScanQueryEngine.java +++ b/processing/src/main/java/io/druid/query/scan/ScanQueryEngine.java @@ -18,7 +18,6 @@ */ package io.druid.query.scan; -import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -34,7 +33,6 @@ import io.druid.query.QueryInterruptedException; import io.druid.query.filter.Filter; import io.druid.segment.BaseObjectColumnValueSelector; -import io.druid.segment.Cursor; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumn; @@ -49,6 +47,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.TimeoutException; @@ -124,21 +123,16 @@ public Sequence process( } final long limit = query.getLimit() - (long) responseContext.get(ScanQueryRunnerFactory.CTX_COUNT); return Sequences.concat( - Sequences.map( - adapter.makeCursors( - filter, - intervals.get(0), - query.getVirtualColumns(), - Granularities.ALL, - query.isDescending(), - null - ), - new Function>() - { - @Override - public Sequence apply(final Cursor cursor) - { - return new BaseSequence<>( + adapter + .makeCursors( + filter, + intervals.get(0), + query.getVirtualColumns(), + Granularities.ALL, + query.isDescending(), + null + ) + .map(cursor -> new BaseSequence<>( new BaseSequence.IteratorMaker>() { @Override @@ -173,6 +167,9 @@ public boolean hasNext() @Override public ScanResultValue next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } if (hasTimeout && System.currentTimeMillis() >= timeoutAt) { throw new QueryInterruptedException(new TimeoutException()); } @@ -208,9 +205,8 @@ public void remove() private List rowsToCompactedList() { final List events = new ArrayList<>(batchSize); - for (int i = 0; !cursor.isDone() - && i < batchSize - && offset < limit; cursor.advance(), i++, offset++) { + final long iterLimit = Math.min(limit, offset + batchSize); + for (; !cursor.isDone() && offset < iterLimit; cursor.advance(), offset++) { final List theEvent = new ArrayList<>(allColumns.size()); for (int j = 0; j < allColumns.size(); j++) { theEvent.add(getColumnValue(j)); @@ -223,9 +219,8 @@ private List rowsToCompactedList() private List> rowsToList() { List> events = Lists.newArrayListWithCapacity(batchSize); - for (int i = 0; !cursor.isDone() - && i < batchSize - && offset < limit; cursor.advance(), i++, offset++) { + final long iterLimit = Math.min(limit, offset + batchSize); + for (; !cursor.isDone() && offset < iterLimit; cursor.advance(), offset++) { final Map theEvent = new LinkedHashMap<>(); for (int j = 0; j < allColumns.size(); j++) { theEvent.put(allColumns.get(j), getColumnValue(j)); @@ -256,10 +251,7 @@ public void cleanup(Iterator iterFromMake) { } } - ); - } - } - ) + )) ); } } diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 27a838f0990e..72e7dc5210b5 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -127,7 +127,7 @@ public void updateSearchResultSet( final Object2IntRBTreeMap set ) { - if (selector != null && !DimensionSelectorUtils.isNilSelector(selector)) { + if (!DimensionSelectorUtils.isNilSelector(selector)) { final IndexedInts row = selector.getRow(); for (int i = 0, rowSize = row.size(); i < rowSize; ++i) { final String dimVal = selector.lookupName(row.get(i)); diff --git a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java index ec4ce2078546..5c3212e51e25 100644 --- a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java @@ -41,7 +41,7 @@ import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; -import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.NumericColumn; import it.unimi.dsi.fastutil.objects.Object2IntRBTreeMap; import org.joda.time.Interval; @@ -166,7 +166,7 @@ static ImmutableBitmap makeTimeFilteredBitmap( if (!interval.contains(segment.getDataInterval())) { final MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); final Column timeColumn = index.getColumn(Column.TIME_COLUMN_NAME); - try (final GenericColumn timeValues = timeColumn.getGenericColumn()) { + try (final NumericColumn timeValues = timeColumn.getNumericColumn()) { int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true)); int endIndex = Math.min( @@ -189,7 +189,7 @@ static ImmutableBitmap makeTimeFilteredBitmap( return timeFilteredBitmap; } - private static int getStartIndexOfTime(GenericColumn timeValues, long time, boolean inclusive) + private static int getStartIndexOfTime(NumericColumn timeValues, long time, boolean inclusive) { int low = 0; int high = timeValues.length() - 1; diff --git a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java index c191f4a6d7af..9c639c5ad41b 100644 --- a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java @@ -755,11 +755,7 @@ protected void updateResults( } - resultBuilder.addEntry( - (Comparable) retVal, - i, - vals - ); + resultBuilder.addEntry((Comparable) retVal, i, vals); } } } diff --git a/processing/src/main/java/io/druid/segment/AbstractDimensionSelector.java b/processing/src/main/java/io/druid/segment/AbstractDimensionSelector.java new file mode 100644 index 000000000000..dd06398faf0c --- /dev/null +++ b/processing/src/main/java/io/druid/segment/AbstractDimensionSelector.java @@ -0,0 +1,38 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import javax.annotation.Nullable; + +/** + * An abstract class that provides an implementation of {@link #getObject()}. + * Don't extend this abstract class if you could provide a more efficient implementation for those methods, implement + * {@link DimensionSelector} directly then. + */ +public abstract class AbstractDimensionSelector implements DimensionSelector +{ + + @Nullable + @Override + public Object getObject() + { + return defaultGetObject(); + } +} diff --git a/processing/src/main/java/io/druid/segment/AbstractIndex.java b/processing/src/main/java/io/druid/segment/AbstractIndex.java index 9b4af31315ba..474ece9df8cc 100644 --- a/processing/src/main/java/io/druid/segment/AbstractIndex.java +++ b/processing/src/main/java/io/druid/segment/AbstractIndex.java @@ -24,7 +24,6 @@ import io.druid.segment.column.Column; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -60,11 +59,7 @@ public String toString() sb.append(columnNames.get(i)).append('='); ColumnValueSelector selector = selectors.get(i); Object columnValue = selector.getObject(); - if (columnValue instanceof Object[]) { - sb.append(Arrays.toString((Object[]) columnValue)); - } else { - sb.append(columnValue); - } + sb.append(columnValue); sb.append(", "); } sb.setLength(sb.length() - 2); diff --git a/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java b/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java index 619e9a23d2cc..bfa16655488d 100644 --- a/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java @@ -36,5 +36,5 @@ public interface BaseObjectColumnValueSelector @Nullable T getObject(); - Class classOfObject(); + Class classOfObject(); } diff --git a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java b/processing/src/main/java/io/druid/segment/BaseSingleValueDimensionSelector.java similarity index 92% rename from processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java rename to processing/src/main/java/io/druid/segment/BaseSingleValueDimensionSelector.java index 0872b55d6e13..5301d8788d9b 100644 --- a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/BaseSingleValueDimensionSelector.java @@ -17,14 +17,12 @@ * under the License. */ -package io.druid.segment.virtual; +package io.druid.segment; import com.google.common.base.Predicate; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.CalledFromHotLoop; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.DimensionSelector; -import io.druid.segment.IdLookup; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.ZeroIndexedInts; @@ -34,6 +32,7 @@ public abstract class BaseSingleValueDimensionSelector implements DimensionSelector { @CalledFromHotLoop + @Nullable protected abstract String getValue(); @Override @@ -51,11 +50,12 @@ public int getValueCardinality() @Override public String lookupName(int id) { + assert id == 0; return getValue(); } @Override - public ValueMatcher makeValueMatcher(final String value) + public ValueMatcher makeValueMatcher(final @Nullable String value) { return new ValueMatcher() { @@ -108,13 +108,13 @@ public IdLookup idLookup() @Nullable @Override - public Object getObject() + public String getObject() { return getValue(); } @Override - public Class classOfObject() + public Class classOfObject() { return String.class; } diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index bee78e103048..f85835e043d2 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -28,7 +28,7 @@ import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.DictionaryEncodedColumn; -import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.NumericColumn; import io.druid.segment.column.ValueType; import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedIterable; @@ -122,7 +122,7 @@ public boolean hasMultipleValues(final String dimension) @Override public int getNumRows() { - try (final GenericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn()) { + try (final NumericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getNumericColumn()) { return column.length(); } } diff --git a/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java b/processing/src/main/java/io/druid/segment/ColumnSerializer.java similarity index 84% rename from processing/src/main/java/io/druid/segment/GenericColumnSerializer.java rename to processing/src/main/java/io/druid/segment/ColumnSerializer.java index 1e3ba907f5e7..82308e887776 100644 --- a/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/ColumnSerializer.java @@ -24,10 +24,13 @@ import java.io.IOException; +/** + * TODO is this really an extension point? + */ @ExtensionPoint -public interface GenericColumnSerializer extends Serializer +public interface ColumnSerializer extends Serializer { void open() throws IOException; - void serialize(ColumnValueSelector selector) throws IOException; + void serialize(ColumnValueSelector selector) throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/ConstantColumnValueSelector.java b/processing/src/main/java/io/druid/segment/ConstantColumnValueSelector.java index 88519008f1cf..3b1b965b1524 100644 --- a/processing/src/main/java/io/druid/segment/ConstantColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/ConstantColumnValueSelector.java @@ -24,13 +24,13 @@ public class ConstantColumnValueSelector implements ColumnValueSelector { - private long longValue; - private float floatValue; - private double doubleValue; + private final long longValue; + private final float floatValue; + private final double doubleValue; - private T objectValue; + private final T objectValue; - private Class objectClass; + private final Class objectClass; public ConstantColumnValueSelector( final long longValue, diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java index 278ea9c1bff7..1cca09a1e692 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java @@ -147,6 +147,7 @@ ColumnSelectorPlus[] createColumnSelectorPluses( ) { int dimCount = dimensionSpecs.size(); + @SuppressWarnings("unchecked") ColumnSelectorPlus[] dims = new ColumnSelectorPlus[dimCount]; for (int i = 0; i < dimCount; i++) { final DimensionSpec dimSpec = dimensionSpecs.get(i); @@ -172,7 +173,7 @@ ColumnSelectorPlus[] createColumnSelectorPluses( return dims; } - public static ColumnValueSelector getColumnValueSelectorFromDimensionSpec( + private static ColumnValueSelector getColumnValueSelectorFromDimensionSpec( DimensionSpec dimSpec, ColumnSelectorFactory columnSelectorFactory ) @@ -183,21 +184,19 @@ public static ColumnValueSelector getColumnValueSelectorFromDimensionSpec( switch (capabilities.getType()) { case STRING: return columnSelectorFactory.makeDimensionSelector(dimSpec); - case LONG: - case FLOAT: - case DOUBLE: - return columnSelectorFactory.makeColumnValueSelector(dimSpec.getDimension()); default: - return null; + return columnSelectorFactory.makeColumnValueSelector(dimSpec.getDimension()); } } - // When determining the capabilities of a column during query processing, this function - // adjusts the capabilities for columns that cannot be handled as-is to manageable defaults - // (e.g., treating missing columns as empty String columns) + /** + * When determining the capabilities of a column during query processing, this function + * adjusts the capabilities for columns that cannot be handled as-is to manageable defaults + * (e.g., treating missing columns as empty String columns) + */ private static ColumnCapabilities getEffectiveCapabilities( DimensionSpec dimSpec, - ColumnCapabilities capabilities + @Nullable ColumnCapabilities capabilities ) { if (capabilities == null) { @@ -229,7 +228,7 @@ private static ColumnCapabilities getEffectiveCapabilities( private static ColumnSelectorStrategyClass makeStrategy( ColumnSelectorStrategyFactory strategyFactory, DimensionSpec dimSpec, - ColumnCapabilities capabilities, + @Nullable ColumnCapabilities capabilities, ColumnValueSelector selector ) { diff --git a/processing/src/main/java/io/druid/segment/DimensionIndexer.java b/processing/src/main/java/io/druid/segment/DimensionIndexer.java index 4b762e5ba8f9..0a51bc4b6703 100644 --- a/processing/src/main/java/io/druid/segment/DimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DimensionIndexer.java @@ -268,22 +268,16 @@ boolean checkUnsortedEncodedKeyComponentsEqual( */ int getUnsortedEncodedKeyComponentHashCode(@Nullable EncodedKeyComponentType key); - boolean LIST = true; - boolean ARRAY = false; - /** * Given a row value array from a Row key, as described in the documentation for - * compareUnsortedEncodedKeyComponents(), convert the unsorted encoded values to a list or array of actual values. + * {@link #compareUnsortedEncodedKeyComponents}, convert the unsorted encoded values to a list of actual values. * - * If the key has one element, this method should return a single Object instead of an array or list, ignoring - * the asList parameter. + * If the key has one element, this method should return a single Object instead of a list. * * @param key dimension value array from a Row key - * @param asList if true, return an array; if false, return a list - * @return single value, array, or list containing the actual values corresponding to the encoded values - * in the input array + * @return single value or list containing the actual values corresponding to the encoded values in the input array */ - Object convertUnsortedEncodedKeyComponentToActualArrayOrList(EncodedKeyComponentType key, boolean asList); + Object convertUnsortedEncodedKeyComponentToActualList(EncodedKeyComponentType key); /** * Converts dictionary-encoded row values from unspecified (random) encoding order, to sorted encoding. This step diff --git a/processing/src/main/java/io/druid/segment/DimensionSelector.java b/processing/src/main/java/io/druid/segment/DimensionSelector.java index 7d6cded0ab76..f96dee876688 100644 --- a/processing/src/main/java/io/druid/segment/DimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelector.java @@ -27,11 +27,12 @@ import io.druid.segment.data.IndexedInts; import javax.annotation.Nullable; +import java.util.Arrays; /** */ @PublicApi -public interface DimensionSelector extends ColumnValueSelector, HotLoopCallee +public interface DimensionSelector extends ColumnValueSelector, HotLoopCallee { int CARDINALITY_UNKNOWN = -1; @@ -125,6 +126,11 @@ public interface DimensionSelector extends ColumnValueSelector, HotLoopCallee @Nullable IdLookup idLookup(); + /** + * @deprecated This method is marked as deprecated in DimensionSelector to minimize the probability of accidental + * calling. "Polymorphism" of DimensionSelector should be used only when operating on {@link ColumnValueSelector} + * objects. + */ @Deprecated @Override default float getFloat() @@ -133,6 +139,11 @@ default float getFloat() return 0.0f; } + /** + * @deprecated This method is marked as deprecated in DimensionSelector to minimize the probability of accidental + * calling. "Polymorphism" of DimensionSelector should be used only when operating on {@link ColumnValueSelector} + * objects. + */ @Deprecated @Override default double getDouble() @@ -141,6 +152,11 @@ default double getDouble() return 0.0; } + /** + * @deprecated This method is marked as deprecated in DimensionSelector to minimize the probability of accidental + * calling. "Polymorphism" of DimensionSelector should be used only when operating on {@link ColumnValueSelector} + * objects. + */ @Deprecated @Override default long getLong() @@ -177,7 +193,7 @@ default Object defaultGetObject() for (int i = 0; i < rowSize; i++) { strings[i] = lookupName(row.get(i)); } - return strings; + return Arrays.asList(strings); } } } diff --git a/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java b/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java index b1df53cd9045..fe10118cc3d1 100644 --- a/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java @@ -46,7 +46,7 @@ private DimensionSelectorUtils() * {@code makeValueMatcher()} to this method, but encouraged to implement {@code makeValueMatcher()} themselves, * bypassing the {@link IndexedInts} abstraction. */ - public static ValueMatcher makeValueMatcherGeneric(DimensionSelector selector, String value) + public static ValueMatcher makeValueMatcherGeneric(DimensionSelector selector, @Nullable String value) { IdLookup idLookup = selector.idLookup(); if (idLookup != null) { @@ -118,7 +118,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) private static ValueMatcher makeNonDictionaryEncodedValueMatcherGeneric( final DimensionSelector selector, - final String value + final @Nullable String value ) { return new ValueMatcher() @@ -293,5 +293,4 @@ public static boolean isNilSelector(final DimensionSelector selector) && selector.getValueCardinality() == 1 && selector.lookupName(0) == null; } - } diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java b/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java index 3ecbb90908a4..88f5867eede6 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java @@ -31,7 +31,7 @@ public interface DoubleColumnSelector extends ColumnValueSelector { /** - * @deprecated This method is marked as deprecated in DoubleColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in DoubleColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of DoubleColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -43,7 +43,7 @@ default float getFloat() } /** - * @deprecated This method is marked as deprecated in DoubleColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in DoubleColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of DoubleColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -55,7 +55,7 @@ default long getLong() } /** - * @deprecated This method is marked as deprecated in DoubleColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in DoubleColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of DoubleColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -71,7 +71,7 @@ default Double getObject() } /** - * @deprecated This method is marked as deprecated in DoubleColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in DoubleColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of DoubleColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java index fda831d27a07..8910778556f5 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java @@ -30,7 +30,7 @@ import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; -public class DoubleColumnSerializer implements GenericColumnSerializer +public class DoubleColumnSerializer implements ColumnSerializer { public static DoubleColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, @@ -73,7 +73,7 @@ public void open() throws IOException } @Override - public void serialize(ColumnValueSelector selector) throws IOException + public void serialize(ColumnValueSelector selector) throws IOException { writer.add(selector.getDouble()); } diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSerializerV2.java b/processing/src/main/java/io/druid/segment/DoubleColumnSerializerV2.java index 81162826695d..7bfb50c8d12f 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSerializerV2.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSerializerV2.java @@ -40,9 +40,9 @@ * Column Serializer for double column. * The column is serialized in two parts, first a bitmap indicating the nullability of row values * and second the actual row values. - * This class is Unsafe for concurrent use from multiple threads. + * This class is unsafe for concurrent use from multiple threads. */ -public class DoubleColumnSerializerV2 implements GenericColumnSerializer +public class DoubleColumnSerializerV2 implements ColumnSerializer { public static DoubleColumnSerializerV2 create( SegmentWriteOutMedium segmentWriteOutMedium, @@ -105,7 +105,7 @@ public void open() throws IOException } @Override - public void serialize(ColumnValueSelector selector) throws IOException + public void serialize(ColumnValueSelector selector) throws IOException { if (selector.isNull()) { nullRowsBitmap.add(rowCount); diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java index 423f3a6424c0..bca270ee373f 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java @@ -166,7 +166,7 @@ public int getUnsortedEncodedKeyComponentHashCode(@Nullable Double key) } @Override - public Object convertUnsortedEncodedKeyComponentToActualArrayOrList(Double key, boolean asList) + public Object convertUnsortedEncodedKeyComponentToActualList(Double key) { return key; } diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java index 0fd10d42bbac..da30341dc2fe 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java @@ -33,7 +33,7 @@ public class DoubleDimensionMergerV9 extends NumericDimensionMergerV9 } @Override - GenericColumnSerializer setupEncodedValueWriter() + ColumnSerializer setupEncodedValueWriter() { return IndexMergerV9.createDoubleColumnSerializer(segmentWriteOutMedium, dimensionName, indexSpec); } diff --git a/processing/src/main/java/io/druid/segment/DoubleWrappingDimensionSelector.java b/processing/src/main/java/io/druid/segment/DoubleWrappingDimensionSelector.java index 280a0d9d9135..604601ea7b24 100644 --- a/processing/src/main/java/io/druid/segment/DoubleWrappingDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/DoubleWrappingDimensionSelector.java @@ -21,20 +21,25 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.virtual.BaseSingleValueDimensionSelector; + +import javax.annotation.Nullable; public class DoubleWrappingDimensionSelector extends BaseSingleValueDimensionSelector { - - private final ExtractionFn extractionFn; private final BaseDoubleColumnValueSelector selector; + @Nullable + private final ExtractionFn extractionFn; - public DoubleWrappingDimensionSelector(BaseDoubleColumnValueSelector doubleColumnSelector, ExtractionFn extractionFn) + public DoubleWrappingDimensionSelector( + BaseDoubleColumnValueSelector doubleColumnSelector, + @Nullable ExtractionFn extractionFn + ) { this.extractionFn = extractionFn; selector = doubleColumnSelector; } + @Nullable @Override protected String getValue() { diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSelector.java b/processing/src/main/java/io/druid/segment/FloatColumnSelector.java index 2e0a394ffc27..3b25a0097600 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSelector.java @@ -31,7 +31,7 @@ public interface FloatColumnSelector extends ColumnValueSelector { /** - * @deprecated This method is marked as deprecated in FloatColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in FloatColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of FloatColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -43,7 +43,7 @@ default double getDouble() } /** - * @deprecated This method is marked as deprecated in FloatColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in FloatColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of FloatColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -55,7 +55,7 @@ default long getLong() } /** - * @deprecated This method is marked as deprecated in FloatColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in FloatColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of FloatColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -71,7 +71,7 @@ default Float getObject() } /** - * @deprecated This method is marked as deprecated in FloatColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in FloatColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of FloatColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java index 7e896f516f06..5e9a470d0d2e 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java @@ -30,7 +30,7 @@ import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; -public class FloatColumnSerializer implements GenericColumnSerializer +public class FloatColumnSerializer implements ColumnSerializer { public static FloatColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, @@ -73,7 +73,7 @@ public void open() throws IOException } @Override - public void serialize(ColumnValueSelector selector) throws IOException + public void serialize(ColumnValueSelector selector) throws IOException { writer.add(selector.getFloat()); } diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSerializerV2.java b/processing/src/main/java/io/druid/segment/FloatColumnSerializerV2.java index 366d26e20ea4..9d3e2b5c1f93 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSerializerV2.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSerializerV2.java @@ -40,9 +40,9 @@ * Column Serializer for float column. * The column is serialized in two parts, first a bitmap indicating the nullability of row values * and second the actual row values. - * This class is Unsafe for concurrent use from multiple threads. + * This class is unsafe for concurrent use from multiple threads. */ -public class FloatColumnSerializerV2 implements GenericColumnSerializer +public class FloatColumnSerializerV2 implements ColumnSerializer { public static FloatColumnSerializerV2 create( SegmentWriteOutMedium segmentWriteOutMedium, @@ -105,7 +105,7 @@ public void open() throws IOException } @Override - public void serialize(ColumnValueSelector selector) throws IOException + public void serialize(ColumnValueSelector selector) throws IOException { if (selector.isNull()) { nullRowsBitmap.add(rowCount); diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java index adca2802085f..db3d8c7778c8 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java @@ -169,7 +169,7 @@ public int getUnsortedEncodedKeyComponentHashCode(@Nullable Float key) } @Override - public Object convertUnsortedEncodedKeyComponentToActualArrayOrList(Float key, boolean asList) + public Object convertUnsortedEncodedKeyComponentToActualList(Float key) { return key; } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java index f49e7415e4b3..f5752fad15c0 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java @@ -33,7 +33,7 @@ public class FloatDimensionMergerV9 extends NumericDimensionMergerV9 } @Override - GenericColumnSerializer setupEncodedValueWriter() + ColumnSerializer setupEncodedValueWriter() { return IndexMergerV9.createFloatColumnSerializer(segmentWriteOutMedium, dimensionName, indexSpec); } diff --git a/processing/src/main/java/io/druid/segment/FloatWrappingDimensionSelector.java b/processing/src/main/java/io/druid/segment/FloatWrappingDimensionSelector.java index d82838f2abf6..4076ff38988f 100644 --- a/processing/src/main/java/io/druid/segment/FloatWrappingDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/FloatWrappingDimensionSelector.java @@ -21,21 +21,22 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.virtual.BaseSingleValueDimensionSelector; import javax.annotation.Nullable; public class FloatWrappingDimensionSelector extends BaseSingleValueDimensionSelector { private final BaseFloatColumnValueSelector selector; + @Nullable private final ExtractionFn extractionFn; - public FloatWrappingDimensionSelector(BaseFloatColumnValueSelector selector, ExtractionFn extractionFn) + public FloatWrappingDimensionSelector(BaseFloatColumnValueSelector selector, @Nullable ExtractionFn extractionFn) { this.selector = selector; this.extractionFn = extractionFn; } + @Nullable @Override protected String getValue() { @@ -46,19 +47,6 @@ protected String getValue() } } - @Nullable - @Override - public Object getObject() - { - return getValue(); - } - - @Override - public Class classOfObject() - { - return String.class; - } - @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 890a0e09249a..4665a4fb6073 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -61,8 +61,8 @@ import io.druid.segment.serde.BitmapIndexColumnPartSupplier; import io.druid.segment.serde.ComplexColumnPartSupplier; import io.druid.segment.serde.DictionaryEncodedColumnSupplier; -import io.druid.segment.serde.FloatGenericColumnSupplier; -import io.druid.segment.serde.LongGenericColumnSupplier; +import io.druid.segment.serde.FloatNumericColumnSupplier; +import io.druid.segment.serde.LongNumericColumnSupplier; import io.druid.segment.serde.SpatialIndexColumnPartSupplier; import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import org.joda.time.Interval; @@ -74,7 +74,6 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -267,18 +266,15 @@ private static void validateRowValues( rp2.getTimestamp() ); } - final Object[] dims1 = rp1.getDimensionValuesForDebug(); - final Object[] dims2 = rp2.getDimensionValuesForDebug(); - if (dims1.length != dims2.length) { - throw new SegmentValidationException( - "Dim lengths not equal %s vs %s", - Arrays.deepToString(dims1), - Arrays.deepToString(dims2) - ); + final List dims1 = rp1.getDimensionValuesForDebug(); + final List dims2 = rp2.getDimensionValuesForDebug(); + if (dims1.size() != dims2.size()) { + throw new SegmentValidationException("Dim lengths not equal %s vs %s", dims1, dims2); } final List dim1Names = adapter1.getDimensionNames(); final List dim2Names = adapter2.getDimensionNames(); - for (int i = 0; i < dims1.length; ++i) { + int dimCount = dims1.size(); + for (int i = 0; i < dimCount; ++i) { final String dim1Name = dim1Names.get(i); final String dim2Name = dim2Names.get(i); @@ -295,22 +291,22 @@ private static void validateRowValues( ); } - Object vals1 = dims1[i]; - Object vals2 = dims2[i]; + Object vals1 = dims1.get(i); + Object vals2 = dims2.get(i); if (isNullRow(vals1) ^ isNullRow(vals2)) { throw notEqualValidationException(dim1Name, vals1, vals2); } - if (vals1 instanceof Object[] && vals2 instanceof Object[]) { - if (!Arrays.equals((Object[]) vals1, (Object[]) vals2)) { - throw notEqualValidationException(dim1Name, vals1, vals2); - } - } else if (vals1 instanceof Object[]) { - if (((Object[]) vals1).length != 1 || !Objects.equals(((Object[]) vals1)[0], vals2)) { - throw notEqualValidationException(dim1Name, vals1, vals2); - } - } else if (vals2 instanceof Object[]) { - if (((Object[]) vals2).length != 1 || !Objects.equals(((Object[]) vals2)[0], vals1)) { - throw notEqualValidationException(dim1Name, vals1, vals2); + boolean vals1IsList = vals1 instanceof List; + boolean vals2IsList = vals2 instanceof List; + if (vals1IsList ^ vals2IsList) { + if (vals1IsList) { + if (((List) vals1).size() != 1 || !Objects.equals(((List) vals1).get(0), vals2)) { + throw notEqualValidationException(dim1Name, vals1, vals2); + } + } else { + if (((List) vals2).size() != 1 || !Objects.equals(((List) vals2).get(0), vals1)) { + throw notEqualValidationException(dim1Name, vals1, vals2); + } } } else { if (!Objects.equals(vals1, vals2)) { @@ -325,10 +321,13 @@ private static boolean isNullRow(@Nullable Object row) if (row == null) { return true; } - if (!(row instanceof Object[])) { + if (!(row instanceof List)) { return false; } - for (Object v : (Object[]) row) { + List rowAsList = (List) row; + //noinspection ForLoopReplaceableByForEach -- in order to not create a garbage iterator object + for (int i = 0, rowSize = rowAsList.size(); i < rowSize; i++) { + Object v = rowAsList.get(i); //noinspection VariableNotUsedInsideIf if (v != null) { return false; @@ -389,7 +388,7 @@ public MMappedIndex mapDir(File inDir) throws IOException Map metrics = Maps.newLinkedHashMap(); for (String metric : availableMetrics) { final String metricFilename = makeMetricFile(inDir, metric, BYTE_ORDER).getName(); - final MetricHolder holder = MetricHolder.fromByteBuffer(smooshedFiles.mapFile(metricFilename), smooshedFiles); + final MetricHolder holder = MetricHolder.fromByteBuffer(smooshedFiles.mapFile(metricFilename)); if (!metric.equals(holder.getName())) { throw new ISE("Metric[%s] loaded up metric[%s] from disk. File names do matter.", metric, holder.getName()); @@ -480,7 +479,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException ColumnBuilder builder = new ColumnBuilder() .setType(ValueType.STRING) .setHasMultipleValues(true) - .setDictionaryEncodedColumn( + .setDictionaryEncodedColumnSupplier( new DictionaryEncodedColumnSupplier( index.getDimValueLookup(dimension), null, @@ -511,8 +510,8 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException metric, new ColumnBuilder() .setType(ValueType.FLOAT) - .setGenericColumn( - new FloatGenericColumnSupplier( + .setNumericColumnSupplier( + new FloatNumericColumnSupplier( metricHolder.floatType, LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() ) @@ -524,11 +523,8 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException metric, new ColumnBuilder() .setType(ValueType.COMPLEX) - .setComplexColumn( - new ComplexColumnPartSupplier( - metricHolder.getTypeName(), - (GenericIndexed) metricHolder.complexType - ) + .setComplexColumnSupplier( + new ComplexColumnPartSupplier(metricHolder.getTypeName(), metricHolder.complexType) ) .build() ); @@ -547,8 +543,8 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException Column.TIME_COLUMN_NAME, new ColumnBuilder() .setType(ValueType.LONG) - .setGenericColumn( - new LongGenericColumnSupplier( + .setNumericColumnSupplier( + new LongNumericColumnSupplier( index.timestamps, LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() ) diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index 87af6a20d4e8..0f7bd0893341 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -54,12 +54,12 @@ import io.druid.segment.serde.ComplexColumnPartSerde; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; -import io.druid.segment.serde.DoubleGenericColumnPartSerde; -import io.druid.segment.serde.DoubleGenericColumnPartSerdeV2; -import io.druid.segment.serde.FloatGenericColumnPartSerde; -import io.druid.segment.serde.FloatGenericColumnPartSerdeV2; -import io.druid.segment.serde.LongGenericColumnPartSerde; -import io.druid.segment.serde.LongGenericColumnPartSerdeV2; +import io.druid.segment.serde.DoubleNumericColumnPartSerde; +import io.druid.segment.serde.DoubleNumericColumnPartSerdeV2; +import io.druid.segment.serde.FloatNumericColumnPartSerde; +import io.druid.segment.serde.FloatNumericColumnPartSerdeV2; +import io.druid.segment.serde.LongNumericColumnPartSerde; +import io.druid.segment.serde.LongNumericColumnPartSerdeV2; import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import org.apache.commons.io.FileUtils; @@ -184,8 +184,8 @@ private File makeIndexFiles( mergers ); closer.register(timeAndDimsIterator); - final GenericColumnSerializer timeWriter = setupTimeWriter(segmentWriteOutMedium, indexSpec); - final ArrayList metricWriters = + final ColumnSerializer timeWriter = setupTimeWriter(segmentWriteOutMedium, indexSpec); + final ArrayList metricWriters = setupMetricsWriters(segmentWriteOutMedium, mergedMetrics, metricsValueTypes, metricTypeNames, indexSpec); List rowNumConversions = mergeIndexesAndWriteColumns( adapters, @@ -320,7 +320,7 @@ private void makeMetricsColumns( final List mergedMetrics, final Map metricsValueTypes, final Map metricTypeNames, - final List metWriters, + final List metWriters, final IndexSpec indexSpec ) throws IOException { @@ -331,7 +331,7 @@ private void makeMetricsColumns( for (int i = 0; i < mergedMetrics.size(); ++i) { String metric = mergedMetrics.get(i); long metricStartTime = System.currentTimeMillis(); - GenericColumnSerializer writer = metWriters.get(i); + ColumnSerializer writer = metWriters.get(i); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); ValueType type = metricsValueTypes.get(metric); @@ -369,16 +369,16 @@ private void makeMetricsColumns( progress.stopSection(section); } - static ColumnPartSerde createLongColumnPartSerde(GenericColumnSerializer serializer, IndexSpec indexSpec) + static ColumnPartSerde createLongColumnPartSerde(ColumnSerializer serializer, IndexSpec indexSpec) { - // If using default values for null use LongGenericColumnPartSerde to allow rollback to previous versions. + // If using default values for null use LongNumericColumnPartSerde to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { - return LongGenericColumnPartSerde.serializerBuilder() + return LongNumericColumnPartSerde.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate(serializer) .build(); } else { - return LongGenericColumnPartSerdeV2.serializerBuilder() + return LongNumericColumnPartSerdeV2.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .withDelegate(serializer) @@ -386,16 +386,16 @@ static ColumnPartSerde createLongColumnPartSerde(GenericColumnSerializer seriali } } - static ColumnPartSerde createDoubleColumnPartSerde(GenericColumnSerializer serializer, IndexSpec indexSpec) + static ColumnPartSerde createDoubleColumnPartSerde(ColumnSerializer serializer, IndexSpec indexSpec) { - // If using default values for null use DoubleGenericColumnPartSerde to allow rollback to previous versions. + // If using default values for null use DoubleNumericColumnPartSerde to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { - return DoubleGenericColumnPartSerde.serializerBuilder() + return DoubleNumericColumnPartSerde.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate(serializer) .build(); } else { - return DoubleGenericColumnPartSerdeV2.serializerBuilder() + return DoubleNumericColumnPartSerdeV2.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .withDelegate(serializer) @@ -403,16 +403,16 @@ static ColumnPartSerde createDoubleColumnPartSerde(GenericColumnSerializer seria } } - static ColumnPartSerde createFloatColumnPartSerde(GenericColumnSerializer serializer, IndexSpec indexSpec) + static ColumnPartSerde createFloatColumnPartSerde(ColumnSerializer serializer, IndexSpec indexSpec) { - // If using default values for null use FloatGenericColumnPartSerde to allow rollback to previous versions. + // If using default values for null use FloatNumericColumnPartSerde to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { - return FloatGenericColumnPartSerde.serializerBuilder() + return FloatNumericColumnPartSerde.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate(serializer) .build(); } else { - return FloatGenericColumnPartSerdeV2.serializerBuilder() + return FloatNumericColumnPartSerdeV2.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .withDelegate(serializer) @@ -423,7 +423,7 @@ static ColumnPartSerde createFloatColumnPartSerde(GenericColumnSerializer serial private void makeTimeColumn( final FileSmoosher v9Smoosher, final ProgressIndicator progress, - final GenericColumnSerializer timeWriter, + final ColumnSerializer timeWriter, final IndexSpec indexSpec ) throws IOException { @@ -466,8 +466,8 @@ private List mergeIndexesAndWriteColumns( final List adapters, final ProgressIndicator progress, final TimeAndDimsIterator timeAndDimsIterator, - final GenericColumnSerializer timeWriter, - final ArrayList metricWriters, + final ColumnSerializer timeWriter, + final ArrayList metricWriters, final List mergers, final boolean fillRowNumConversions ) throws IOException @@ -557,10 +557,10 @@ private List mergeIndexesAndWriteColumns( return rowNumConversions; } - private GenericColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMedium, IndexSpec indexSpec) + private ColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMedium, IndexSpec indexSpec) throws IOException { - GenericColumnSerializer timeWriter = createLongColumnSerializer( + ColumnSerializer timeWriter = createLongColumnSerializer( segmentWriteOutMedium, "little_end_time", indexSpec @@ -570,7 +570,7 @@ private GenericColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWri return timeWriter; } - private ArrayList setupMetricsWriters( + private ArrayList setupMetricsWriters( final SegmentWriteOutMedium segmentWriteOutMedium, final List mergedMetrics, final Map metricsValueTypes, @@ -578,11 +578,11 @@ private ArrayList setupMetricsWriters( final IndexSpec indexSpec ) throws IOException { - ArrayList metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size()); + ArrayList metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size()); for (String metric : mergedMetrics) { ValueType type = metricsValueTypes.get(metric); - GenericColumnSerializer writer; + ColumnSerializer writer; switch (type) { case LONG: writer = createLongColumnSerializer(segmentWriteOutMedium, metric, indexSpec); @@ -611,7 +611,7 @@ private ArrayList setupMetricsWriters( return metWriters; } - static GenericColumnSerializer createLongColumnSerializer( + static ColumnSerializer createLongColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String columnName, IndexSpec indexSpec @@ -636,7 +636,7 @@ static GenericColumnSerializer createLongColumnSerializer( } } - static GenericColumnSerializer createDoubleColumnSerializer( + static ColumnSerializer createDoubleColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String columnName, IndexSpec indexSpec @@ -659,7 +659,7 @@ static GenericColumnSerializer createDoubleColumnSerializer( } } - static GenericColumnSerializer createFloatColumnSerializer( + static ColumnSerializer createFloatColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String columnName, IndexSpec indexSpec diff --git a/processing/src/main/java/io/druid/segment/LongColumnSelector.java b/processing/src/main/java/io/druid/segment/LongColumnSelector.java index 8a46b7c556dd..69a7f3f473e2 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSelector.java @@ -31,7 +31,7 @@ public interface LongColumnSelector extends ColumnValueSelector { /** - * @deprecated This method is marked as deprecated in LongColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in LongColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of LongColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -43,7 +43,7 @@ default float getFloat() } /** - * @deprecated This method is marked as deprecated in LongColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in LongColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of LongColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -55,7 +55,7 @@ default double getDouble() } /** - * @deprecated This method is marked as deprecated in LongColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in LongColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of LongColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -71,7 +71,7 @@ default Long getObject() } /** - * @deprecated This method is marked as deprecated in LongColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in LongColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of LongColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ diff --git a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java index 2c74fde458ff..17b65b60ff40 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java @@ -33,7 +33,7 @@ /** * Unsafe for concurrent use from multiple threads. */ -public class LongColumnSerializer implements GenericColumnSerializer +public class LongColumnSerializer implements ColumnSerializer { public static LongColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, @@ -81,7 +81,7 @@ public void open() throws IOException } @Override - public void serialize(ColumnValueSelector selector) throws IOException + public void serialize(ColumnValueSelector selector) throws IOException { writer.add(selector.getLong()); } diff --git a/processing/src/main/java/io/druid/segment/LongColumnSerializerV2.java b/processing/src/main/java/io/druid/segment/LongColumnSerializerV2.java index f5536dce0d8e..c68ff607d94a 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSerializerV2.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSerializerV2.java @@ -40,9 +40,9 @@ * Column Serializer for long column. * The column is serialized in two parts, first a bitmap indicating the nullability of row values * and second the actual row values. - * This class is Unsafe for concurrent use from multiple threads. + * This class is unsafe for concurrent use from multiple threads. */ -public class LongColumnSerializerV2 implements GenericColumnSerializer +public class LongColumnSerializerV2 implements ColumnSerializer { public static LongColumnSerializerV2 create( SegmentWriteOutMedium segmentWriteOutMedium, @@ -111,7 +111,7 @@ public void open() throws IOException } @Override - public void serialize(ColumnValueSelector selector) throws IOException + public void serialize(ColumnValueSelector selector) throws IOException { if (selector.isNull()) { nullRowsBitmap.add(rowCount); diff --git a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java index 4b7df0a20b47..e7be19cda81c 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java @@ -169,7 +169,7 @@ public int getUnsortedEncodedKeyComponentHashCode(@Nullable Long key) } @Override - public Object convertUnsortedEncodedKeyComponentToActualArrayOrList(Long key, boolean asList) + public Object convertUnsortedEncodedKeyComponentToActualList(Long key) { return key; } diff --git a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java index 4f3aa7406db4..5e51d85aead8 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java @@ -33,7 +33,7 @@ public class LongDimensionMergerV9 extends NumericDimensionMergerV9 } @Override - GenericColumnSerializer setupEncodedValueWriter() + ColumnSerializer setupEncodedValueWriter() { return IndexMergerV9.createLongColumnSerializer(segmentWriteOutMedium, dimensionName, indexSpec); } diff --git a/processing/src/main/java/io/druid/segment/LongWrappingDimensionSelector.java b/processing/src/main/java/io/druid/segment/LongWrappingDimensionSelector.java index b3d786a89f35..f28e736db4dc 100644 --- a/processing/src/main/java/io/druid/segment/LongWrappingDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/LongWrappingDimensionSelector.java @@ -21,21 +21,22 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.virtual.BaseSingleValueDimensionSelector; import javax.annotation.Nullable; public class LongWrappingDimensionSelector extends BaseSingleValueDimensionSelector { private final BaseLongColumnValueSelector selector; + @Nullable private final ExtractionFn extractionFn; - public LongWrappingDimensionSelector(BaseLongColumnValueSelector selector, ExtractionFn extractionFn) + public LongWrappingDimensionSelector(BaseLongColumnValueSelector selector, @Nullable ExtractionFn extractionFn) { this.selector = selector; this.extractionFn = extractionFn; } + @Nullable @Override protected String getValue() { @@ -46,19 +47,6 @@ protected String getValue() } } - @Nullable - @Override - public Object getObject() - { - return getValue(); - } - - @Override - public Class classOfObject() - { - return String.class; - } - @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/MetricHolder.java b/processing/src/main/java/io/druid/segment/MetricHolder.java index 1f2e97963de0..402124b286f5 100644 --- a/processing/src/main/java/io/druid/segment/MetricHolder.java +++ b/processing/src/main/java/io/druid/segment/MetricHolder.java @@ -21,11 +21,8 @@ import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.ISE; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.segment.data.CompressedColumnarFloatsSupplier; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.Indexed; -import io.druid.segment.data.ObjectStrategy; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; @@ -39,12 +36,7 @@ public class MetricHolder private static final byte[] version = new byte[]{0x0}; private static final SerializerUtils serializerUtils = new SerializerUtils(); - public static MetricHolder fromByteBuffer(ByteBuffer buf, SmooshedFileMapper mapper) - { - return fromByteBuffer(buf, null, mapper); - } - - public static MetricHolder fromByteBuffer(ByteBuffer buf, ObjectStrategy strategy, SmooshedFileMapper mapper) + public static MetricHolder fromByteBuffer(ByteBuffer buf) { final byte ver = buf.get(); if (version[0] != ver) { @@ -60,17 +52,13 @@ public static MetricHolder fromByteBuffer(ByteBuffer buf, ObjectStrategy strateg holder.floatType = CompressedColumnarFloatsSupplier.fromByteBuffer(buf, ByteOrder.nativeOrder()); break; case COMPLEX: - if (strategy != null) { - holder.complexType = GenericIndexed.read(buf, strategy, mapper); - } else { - final ComplexMetricSerde serdeForType = ComplexMetrics.getSerdeForType(holder.getTypeName()); - - if (serdeForType == null) { - throw new ISE("Unknown type[%s], cannot load.", holder.getTypeName()); - } + final ComplexMetricSerde serdeForType = ComplexMetrics.getSerdeForType(holder.getTypeName()); - holder.complexType = GenericIndexed.read(buf, serdeForType.getObjectStrategy()); + if (serdeForType == null) { + throw new ISE("Unknown type[%s], cannot load.", holder.getTypeName()); } + + holder.complexType = read(buf, serdeForType); break; case LONG: case DOUBLE: @@ -80,9 +68,10 @@ public static MetricHolder fromByteBuffer(ByteBuffer buf, ObjectStrategy strateg return holder; } - private final String name; - private final String typeName; - private final MetricType type; + private static GenericIndexed read(ByteBuffer buf, ComplexMetricSerde serde) + { + return GenericIndexed.read(buf, serde.getObjectStrategy()); + } public enum MetricType { @@ -104,8 +93,11 @@ static MetricType determineType(String typeName) } } + private final String name; + private final String typeName; + private final MetricType type; CompressedColumnarFloatsSupplier floatType = null; - Indexed complexType = null; + GenericIndexed complexType = null; private MetricHolder( String name, diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index b65f0418fe73..453f7d4cc7c8 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -63,7 +63,7 @@ public IndexedInts getRow(int offset) } @Override - public ValueMatcher makeValueMatcher(String value) + public ValueMatcher makeValueMatcher(@Nullable String value) { return BooleanValueMatcher.of(value == null); } @@ -102,7 +102,7 @@ public IdLookup idLookup() } @Override - public int lookupId(String name) + public int lookupId(@Nullable String name) { return NullHandling.isNullOrEquivalent(name) ? 0 : -1; } diff --git a/processing/src/main/java/io/druid/segment/NumericDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/NumericDimensionMergerV9.java index bb9c03b78df0..b70cac73b461 100644 --- a/processing/src/main/java/io/druid/segment/NumericDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/NumericDimensionMergerV9.java @@ -35,7 +35,7 @@ public abstract class NumericDimensionMergerV9 implements DimensionMergerV9 protected final IndexSpec indexSpec; protected final SegmentWriteOutMedium segmentWriteOutMedium; - protected final GenericColumnSerializer serializer; + protected final ColumnSerializer serializer; NumericDimensionMergerV9( String dimensionName, @@ -56,7 +56,7 @@ public abstract class NumericDimensionMergerV9 implements DimensionMergerV9 } } - abstract GenericColumnSerializer setupEncodedValueWriter(); + abstract ColumnSerializer setupEncodedValueWriter(); @Override public final void writeMergedValueDictionary(List adapters) diff --git a/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java index d1e377a13abd..425f8e13de60 100644 --- a/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java @@ -30,7 +30,7 @@ public abstract class ObjectColumnSelector implements ColumnValueSelector { /** - * @deprecated This method is marked as deprecated in ObjectColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in ObjectColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of ObjectColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -46,7 +46,7 @@ public float getFloat() } /** - * @deprecated This method is marked as deprecated in ObjectColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in ObjectColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of ObjectColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ @@ -62,7 +62,7 @@ public double getDouble() } /** - * @deprecated This method is marked as deprecated in ObjectColumnSelector to minimize the probability of accidential + * @deprecated This method is marked as deprecated in ObjectColumnSelector to minimize the probability of accidental * calling. "Polymorphism" of ObjectColumnSelector should be used only when operating on {@link ColumnValueSelector} * objects. */ diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java index 48400ca634eb..01a150d45313 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java @@ -85,7 +85,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi } if (dimension.equals(Column.TIME_COLUMN_NAME)) { - return new SingleScanTimeDimSelector(makeColumnValueSelector(dimension), extractionFn, descending); + return new SingleScanTimeDimensionSelector(makeColumnValueSelector(dimension), extractionFn, descending); } ValueType type = columnDesc.getCapabilities().getType(); @@ -124,7 +124,7 @@ public ColumnValueSelector makeColumnValueSelector(String columnName) } else if (capabilities.getType() == ValueType.COMPLEX) { column = holder.getComplexColumn(); } else if (capabilities.getType().isNumeric()) { - column = holder.getGenericColumn(); + column = holder.getNumericColumn(); } else { throw new ISE("Unknown column type: %s", capabilities.getType()); } diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java index 16ba427e9d0f..cffbdfa1c997 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java @@ -199,18 +199,18 @@ class RowIteratorImpl implements TransformableRowIterator private final SimpleAscendingOffset offset = new SimpleAscendingOffset(numRows); private final int maxValidOffset = numRows - 1; - private final ColumnValueSelector offsetTimestampSelector; - private final ColumnValueSelector[] offsetDimensionValueSelectors; - private final ColumnValueSelector[] offsetMetricSelectors; + private final ColumnValueSelector offsetTimestampSelector; + private final ColumnValueSelector[] offsetDimensionValueSelectors; + private final ColumnValueSelector[] offsetMetricSelectors; private final SettableLongColumnValueSelector rowTimestampSelector = new SettableLongColumnValueSelector(); - private final SettableColumnValueSelector[] rowDimensionValueSelectors; - private final SettableColumnValueSelector[] rowMetricSelectors; + private final SettableColumnValueSelector[] rowDimensionValueSelectors; + private final SettableColumnValueSelector[] rowMetricSelectors; private final RowPointer rowPointer; private final SettableLongColumnValueSelector markedTimestampSelector = new SettableLongColumnValueSelector(); - private final SettableColumnValueSelector[] markedDimensionValueSelectors; - private final SettableColumnValueSelector[] markedMetricSelectors; + private final SettableColumnValueSelector[] markedDimensionValueSelectors; + private final SettableColumnValueSelector[] markedMetricSelectors; private final TimeAndDimsPointer markedRowPointer; boolean first = true; @@ -247,7 +247,7 @@ class RowIteratorImpl implements TransformableRowIterator .toArray(SettableColumnValueSelector[]::new); rowMetricSelectors = metricNames .stream() - .map(metric -> input.getColumn(metric).makeSettableColumnValueSelector()) + .map(metric -> input.getColumn(metric).makeNewSettableColumnValueSelector()) .toArray(SettableColumnValueSelector[]::new); rowPointer = new RowPointer( @@ -266,7 +266,7 @@ class RowIteratorImpl implements TransformableRowIterator .toArray(SettableColumnValueSelector[]::new); markedMetricSelectors = metricNames .stream() - .map(metric -> input.getColumn(metric).makeSettableColumnValueSelector()) + .map(metric -> input.getColumn(metric).makeNewSettableColumnValueSelector()) .toArray(SettableColumnValueSelector[]::new); markedRowPointer = new TimeAndDimsPointer( markedTimestampSelector, diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index 5c5ca50b48e9..e861f12e14bb 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -39,8 +39,8 @@ import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.NumericColumn; import io.druid.segment.column.ComplexColumn; -import io.druid.segment.column.GenericColumn; import io.druid.segment.data.Indexed; import io.druid.segment.data.Offset; import io.druid.segment.data.ReadableOffset; @@ -120,7 +120,7 @@ public int getNumRows() @Override public DateTime getMinTime() { - try (final GenericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn()) { + try (final NumericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getNumericColumn()) { return DateTimes.utc(column.getLongSingleValueRow(0)); } } @@ -128,7 +128,7 @@ public DateTime getMinTime() @Override public DateTime getMaxTime() { - try (final GenericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn()) { + try (final NumericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getNumericColumn()) { return DateTimes.utc(column.getLongSingleValueRow(column.length() - 1)); } } @@ -371,7 +371,7 @@ public Sequence build() // Column caches shared amongst all cursors in this sequence. final Map columnCache = new HashMap<>(); - final GenericColumn timestamps = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn(); + final NumericColumn timestamps = index.getColumn(Column.TIME_COLUMN_NAME).getNumericColumn(); final Closer closer = Closer.create(); closer.register(timestamps); @@ -537,13 +537,13 @@ public void reset() public abstract static class TimestampCheckingOffset extends Offset { final Offset baseOffset; - final GenericColumn timestamps; + final NumericColumn timestamps; final long timeLimit; final boolean allWithinThreshold; TimestampCheckingOffset( Offset baseOffset, - GenericColumn timestamps, + NumericColumn timestamps, long timeLimit, boolean allWithinThreshold ) @@ -614,7 +614,7 @@ public static class AscendingTimestampCheckingOffset extends TimestampCheckingOf { AscendingTimestampCheckingOffset( Offset baseOffset, - GenericColumn timestamps, + NumericColumn timestamps, long timeLimit, boolean allWithinThreshold ) @@ -647,7 +647,7 @@ public static class DescendingTimestampCheckingOffset extends TimestampCheckingO { DescendingTimestampCheckingOffset( Offset baseOffset, - GenericColumn timestamps, + NumericColumn timestamps, long timeLimit, boolean allWithinThreshold ) diff --git a/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java b/processing/src/main/java/io/druid/segment/SingleScanTimeDimensionSelector.java similarity index 83% rename from processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java rename to processing/src/main/java/io/druid/segment/SingleScanTimeDimensionSelector.java index c04fb0c8b5ce..6d1b4c647d6e 100644 --- a/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java +++ b/processing/src/main/java/io/druid/segment/SingleScanTimeDimensionSelector.java @@ -19,6 +19,7 @@ package io.druid.segment; +import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.ValueMatcher; @@ -31,7 +32,13 @@ import java.util.List; import java.util.Objects; -public class SingleScanTimeDimSelector implements DimensionSelector +/** + * A special {@link DimensionSelector} for projected time columns + * - it assumes time values are scanned once and values are grouped together + * (i.e. we never revisit a timestamp we have seen before, unless it is the same as the last accessed one) + * - it also applies and caches extraction function values at the {@link DimensionSelector} level to speed things up + */ +public class SingleScanTimeDimensionSelector implements DimensionSelector { private final ExtractionFn extractionFn; private final BaseLongColumnValueSelector selector; @@ -43,17 +50,13 @@ public class SingleScanTimeDimSelector implements DimensionSelector private long currentTimestamp = Long.MIN_VALUE; private int index = -1; - - // Use a special DimSelector for projected time columns - // - it assumes time values are scanned once and values are grouped together - // (i.e. we never revisit a timestamp we have seen before, unless it is the same as the last accessed one) - // - it also applies and caches extraction function values at the DimSelector level to speed things up - public SingleScanTimeDimSelector(BaseLongColumnValueSelector selector, ExtractionFn extractionFn, boolean descending) + public SingleScanTimeDimensionSelector( + BaseLongColumnValueSelector selector, + @Nullable ExtractionFn extractionFn, + boolean descending + ) { - if (extractionFn == null) { - throw new UnsupportedOperationException("time dimension must provide an extraction function"); - } - + Preconditions.checkNotNull(extractionFn, "time dimension must provide an extraction function"); this.extractionFn = extractionFn; this.selector = selector; this.descending = descending; @@ -67,7 +70,7 @@ public IndexedInts getRow() } @Override - public ValueMatcher makeValueMatcher(final String value) + public ValueMatcher makeValueMatcher(final @Nullable String value) { return new ValueMatcher() { @@ -80,7 +83,7 @@ public boolean matches() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - inspector.visit("selector", SingleScanTimeDimSelector.this); + inspector.visit("selector", SingleScanTimeDimensionSelector.this); } }; } @@ -99,7 +102,7 @@ public boolean matches() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - inspector.visit("selector", SingleScanTimeDimSelector.this); + inspector.visit("selector", SingleScanTimeDimensionSelector.this); inspector.visit("predicate", predicate); } }; @@ -174,13 +177,13 @@ public IdLookup idLookup() @Nullable @Override - public Object getObject() + public String getObject() { return currentValue; } @Override - public Class classOfObject() + public Class classOfObject() { return String.class; } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 75ca57bf07fe..793bf43d6f12 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -50,7 +50,6 @@ import it.unimi.dsi.fastutil.objects.Object2IntSortedMap; import javax.annotation.Nullable; -import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.Iterator; @@ -633,10 +632,7 @@ public Object getObject() return null; } - return convertUnsortedEncodedKeyComponentToActualArrayOrList( - (int[]) dims[dimIndex], - DimensionIndexer.ARRAY - ); + return convertUnsortedEncodedKeyComponentToActualList((int[]) dims[dimIndex]); } @SuppressWarnings("deprecation") @@ -664,30 +660,22 @@ public ColumnValueSelector makeColumnValueSelector( return makeDimensionSelector(DefaultDimensionSpec.of(desc.getName()), currEntry, desc); } + @Nullable @Override - public Object convertUnsortedEncodedKeyComponentToActualArrayOrList(int[] key, boolean asList) + public Object convertUnsortedEncodedKeyComponentToActualList(int[] key) { - if (key == null || key.length == 0) { + if (key.length == 0) { return null; } if (key.length == 1) { - return getActualValue(key[0], false); + return NullHandling.nullToEmptyIfNeeded(getActualValue(key[0], false)); } else { - if (asList) { - List rowVals = new ArrayList<>(key.length); - for (int id : key) { - String val = getActualValue(id, false); - rowVals.add(NullHandling.nullToEmptyIfNeeded(val)); - } - return rowVals; - } else { - String[] rowArray = new String[key.length]; - for (int i = 0; i < key.length; i++) { - String val = getActualValue(key[i], false); - rowArray[i] = NullHandling.nullToEmptyIfNeeded(val); - } - return rowArray; + String[] rowArray = new String[key.length]; + for (int i = 0; i < key.length; i++) { + String val = getActualValue(key[i], false); + rowArray[i] = NullHandling.nullToEmptyIfNeeded(val); } + return Arrays.asList(rowArray); } } diff --git a/processing/src/main/java/io/druid/segment/TimeAndDimsPointer.java b/processing/src/main/java/io/druid/segment/TimeAndDimsPointer.java index bf0962cc7f1d..43b888a94ae5 100644 --- a/processing/src/main/java/io/druid/segment/TimeAndDimsPointer.java +++ b/processing/src/main/java/io/druid/segment/TimeAndDimsPointer.java @@ -28,6 +28,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * TimeAndDimsPointer is used in conjunction with {@link TimeAndDimsIterator}, it's an _immutable_ object that points to @@ -178,9 +179,9 @@ public int hashCode() throw new UnsupportedOperationException("Should not compute hashCode() on TimeAndDimsPointer"); } - Object[] getDimensionValuesForDebug() + List getDimensionValuesForDebug() { - return Arrays.stream(dimensionSelectors).map(ColumnValueSelector::getObject).toArray(); + return Arrays.stream(dimensionSelectors).map(ColumnValueSelector::getObject).collect(Collectors.toList()); } @Override @@ -198,9 +199,6 @@ Map getDimensionNamesToValuesForDebug() LinkedHashMap result = new LinkedHashMap<>(); for (int i = 0; i < getNumDimensions(); i++) { Object value = dimensionSelectors[i].getObject(); - if (value instanceof Object[]) { - value = Arrays.asList((Object[]) value); - } result.put(dimensionHandlers.get(i).getDimensionName(), value); } return result; diff --git a/processing/src/main/java/io/druid/segment/column/BaseColumn.java b/processing/src/main/java/io/druid/segment/column/BaseColumn.java index 7b874357af35..376eca7a90e9 100644 --- a/processing/src/main/java/io/druid/segment/column/BaseColumn.java +++ b/processing/src/main/java/io/druid/segment/column/BaseColumn.java @@ -26,5 +26,5 @@ public interface BaseColumn extends Closeable { - ColumnValueSelector makeColumnValueSelector(ReadableOffset offset); + ColumnValueSelector makeColumnValueSelector(ReadableOffset offset); } diff --git a/processing/src/main/java/io/druid/segment/column/Column.java b/processing/src/main/java/io/druid/segment/column/Column.java index bc40ac8488d0..9f6d737c553a 100644 --- a/processing/src/main/java/io/druid/segment/column/Column.java +++ b/processing/src/main/java/io/druid/segment/column/Column.java @@ -40,7 +40,7 @@ static boolean storeDoubleAsFloat() int getLength(); DictionaryEncodedColumn getDictionaryEncoding(); - GenericColumn getGenericColumn(); + NumericColumn getNumericColumn(); ComplexColumn getComplexColumn(); BitmapIndex getBitmapIndex(); SpatialIndex getSpatialIndex(); @@ -48,5 +48,5 @@ static boolean storeDoubleAsFloat() /** * Returns a new instance of a {@link SettableColumnValueSelector}, corresponding to the type of this column. */ - SettableColumnValueSelector makeSettableColumnValueSelector(); + SettableColumnValueSelector makeNewSettableColumnValueSelector(); } diff --git a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java index 1bf8af0afb04..9a0b58351a88 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java @@ -30,9 +30,8 @@ public class ColumnBuilder private ValueType type = null; private boolean hasMultipleValues = false; - private Supplier dictionaryEncodedColumn = null; - private Supplier genericColumn = null; - private Supplier complexColumn = null; + private Supplier columnSupplier = null; + private boolean dictionaryEncoded = false; private Supplier bitmapIndex = null; private Supplier spatialIndex = null; private SmooshedFileMapper fileMapper = null; @@ -60,21 +59,22 @@ public ColumnBuilder setHasMultipleValues(boolean hasMultipleValues) return this; } - public ColumnBuilder setDictionaryEncodedColumn(Supplier dictionaryEncodedColumn) + public ColumnBuilder setDictionaryEncodedColumnSupplier(Supplier> columnSupplier) { - this.dictionaryEncodedColumn = dictionaryEncodedColumn; + this.columnSupplier = columnSupplier; + this.dictionaryEncoded = true; return this; } - public ColumnBuilder setGenericColumn(Supplier genericColumn) + public ColumnBuilder setComplexColumnSupplier(Supplier columnSupplier) { - this.genericColumn = genericColumn; + this.columnSupplier = columnSupplier; return this; } - public ColumnBuilder setComplexColumn(Supplier complexColumn) + public ColumnBuilder setNumericColumnSupplier(Supplier columnSupplier) { - this.complexColumn = complexColumn; + this.columnSupplier = columnSupplier; return this; } @@ -97,13 +97,11 @@ public Column build() return new SimpleColumn( new ColumnCapabilitiesImpl() .setType(type) - .setDictionaryEncoded(dictionaryEncodedColumn != null) + .setDictionaryEncoded(dictionaryEncoded) .setHasBitmapIndexes(bitmapIndex != null) .setHasSpatialIndexes(spatialIndex != null) .setHasMultipleValues(hasMultipleValues), - dictionaryEncodedColumn, - genericColumn, - complexColumn, + columnSupplier, bitmapIndex, spatialIndex ); diff --git a/processing/src/main/java/io/druid/segment/column/ComplexColumn.java b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java index 0f5d718bf1c3..05d739ff83fc 100644 --- a/processing/src/main/java/io/druid/segment/column/ComplexColumn.java +++ b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java @@ -22,23 +22,42 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnValueSelector; import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ReadableOffset; import javax.annotation.Nullable; /** - */ -public interface ComplexColumn extends BaseColumn +*/ +public class ComplexColumn implements BaseColumn { - Class getClazz(); - String getTypeName(); - Object getRowValue(int rowNum); + private final GenericIndexed index; + private final String typeName; - @Override - void close(); + public ComplexColumn(String typeName, GenericIndexed index) + { + this.index = index; + this.typeName = typeName; + } + + public Class getClazz() + { + return index.getClazz(); + } + + public String getTypeName() + { + return typeName; + } + + @Nullable + public Object getRowValue(int rowNum) + { + return index.get(rowNum); + } @Override - default ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { return new ObjectColumnSelector() { @@ -62,4 +81,10 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) } }; } + + @Override + public void close() + { + // nothing to close + } } diff --git a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java index 72eef34e2a58..00c9ff9e53be 100644 --- a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java @@ -29,7 +29,7 @@ /** */ -public interface DictionaryEncodedColumn extends BaseColumn +public interface DictionaryEncodedColumn> extends BaseColumn { Class getClazz(); int length(); @@ -43,7 +43,7 @@ public interface DictionaryEncodedColumn extends DimensionSelector makeDimensionSelector(ReadableOffset offset, @Nullable ExtractionFn extractionFn); @Override - default ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + default ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { return makeDimensionSelector(offset, null); } diff --git a/processing/src/main/java/io/druid/segment/column/DoublesColumn.java b/processing/src/main/java/io/druid/segment/column/DoublesColumn.java index 6d00d2ff0684..652e68999808 100644 --- a/processing/src/main/java/io/druid/segment/column/DoublesColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DoublesColumn.java @@ -29,7 +29,7 @@ /** */ -public class DoublesColumn implements GenericColumn +public class DoublesColumn implements NumericColumn { /** * Factory method to create DoublesColumn. @@ -57,7 +57,7 @@ public int length() } @Override - public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { return column.makeColumnValueSelector(offset, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap()); } diff --git a/processing/src/main/java/io/druid/segment/column/DoublesColumnWithNulls.java b/processing/src/main/java/io/druid/segment/column/DoublesColumnWithNulls.java index 5b6d34c8651c..1897d4949f3d 100644 --- a/processing/src/main/java/io/druid/segment/column/DoublesColumnWithNulls.java +++ b/processing/src/main/java/io/druid/segment/column/DoublesColumnWithNulls.java @@ -39,7 +39,7 @@ class DoublesColumnWithNulls extends DoublesColumn } @Override - public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { return column.makeColumnValueSelector(offset, nullValueBitmap); } diff --git a/processing/src/main/java/io/druid/segment/column/FloatsColumn.java b/processing/src/main/java/io/druid/segment/column/FloatsColumn.java index 8826afb1dc00..18034ba8fbf6 100644 --- a/processing/src/main/java/io/druid/segment/column/FloatsColumn.java +++ b/processing/src/main/java/io/druid/segment/column/FloatsColumn.java @@ -28,7 +28,7 @@ /** */ -public class FloatsColumn implements GenericColumn +public class FloatsColumn implements NumericColumn { /** * Factory method to create FloatsColumn. @@ -56,7 +56,7 @@ public int length() } @Override - public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { return column.makeColumnValueSelector(offset, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap()); } diff --git a/processing/src/main/java/io/druid/segment/column/FloatsColumnWithNulls.java b/processing/src/main/java/io/druid/segment/column/FloatsColumnWithNulls.java index 0edb6a4576d5..3bda7b80b4b9 100644 --- a/processing/src/main/java/io/druid/segment/column/FloatsColumnWithNulls.java +++ b/processing/src/main/java/io/druid/segment/column/FloatsColumnWithNulls.java @@ -39,7 +39,7 @@ class FloatsColumnWithNulls extends FloatsColumn } @Override - public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { return column.makeColumnValueSelector(offset, nullValueBitmap); } diff --git a/processing/src/main/java/io/druid/segment/column/LongsColumn.java b/processing/src/main/java/io/druid/segment/column/LongsColumn.java index e41cfb1816c0..89d9ebf03ee3 100644 --- a/processing/src/main/java/io/druid/segment/column/LongsColumn.java +++ b/processing/src/main/java/io/druid/segment/column/LongsColumn.java @@ -28,7 +28,7 @@ /** */ -public class LongsColumn implements GenericColumn +public class LongsColumn implements NumericColumn { /** * Factory method to create LongsColumn. @@ -56,10 +56,9 @@ public int length() } @Override - public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { - return column.makeColumnValueSelector(offset, IndexIO.LEGACY_FACTORY.getBitmapFactory() - .makeEmptyImmutableBitmap()); + return column.makeColumnValueSelector(offset, IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap()); } @Override diff --git a/processing/src/main/java/io/druid/segment/column/LongsColumnWithNulls.java b/processing/src/main/java/io/druid/segment/column/LongsColumnWithNulls.java index 7e8e642b191c..d75905eea44b 100644 --- a/processing/src/main/java/io/druid/segment/column/LongsColumnWithNulls.java +++ b/processing/src/main/java/io/druid/segment/column/LongsColumnWithNulls.java @@ -39,7 +39,7 @@ class LongsColumnWithNulls extends LongsColumn } @Override - public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { return column.makeColumnValueSelector(offset, nullValueBitmap); } diff --git a/processing/src/main/java/io/druid/segment/column/GenericColumn.java b/processing/src/main/java/io/druid/segment/column/NumericColumn.java similarity index 94% rename from processing/src/main/java/io/druid/segment/column/GenericColumn.java rename to processing/src/main/java/io/druid/segment/column/NumericColumn.java index 5e09259a3442..6853259578e0 100644 --- a/processing/src/main/java/io/druid/segment/column/GenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/NumericColumn.java @@ -24,7 +24,7 @@ /** */ -public interface GenericColumn extends BaseColumn, HotLoopCallee +public interface NumericColumn extends BaseColumn, HotLoopCallee { int length(); diff --git a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java index 6876a123bbdf..c2812295c9ec 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java @@ -27,25 +27,19 @@ class SimpleColumn implements Column { private final ColumnCapabilities capabilities; - private final Supplier dictionaryEncodedColumn; - private final Supplier genericColumn; - private final Supplier complexColumn; + private final Supplier columnSupplier; private final Supplier bitmapIndex; private final Supplier spatialIndex; SimpleColumn( ColumnCapabilities capabilities, - Supplier dictionaryEncodedColumn, - Supplier genericColumn, - Supplier complexColumn, + Supplier columnSupplier, Supplier bitmapIndex, Supplier spatialIndex ) { this.capabilities = capabilities; - this.dictionaryEncodedColumn = dictionaryEncodedColumn; - this.genericColumn = genericColumn; - this.complexColumn = complexColumn; + this.columnSupplier = columnSupplier; this.bitmapIndex = bitmapIndex; this.spatialIndex = spatialIndex; } @@ -59,7 +53,7 @@ public ColumnCapabilities getCapabilities() @Override public int getLength() { - try (final GenericColumn column = genericColumn.get()) { + try (final NumericColumn column = (NumericColumn) columnSupplier.get()) { return column.length(); } } @@ -67,19 +61,19 @@ public int getLength() @Override public DictionaryEncodedColumn getDictionaryEncoding() { - return dictionaryEncodedColumn == null ? null : dictionaryEncodedColumn.get(); + return (DictionaryEncodedColumn) columnSupplier.get(); } @Override - public GenericColumn getGenericColumn() + public NumericColumn getNumericColumn() { - return genericColumn == null ? null : genericColumn.get(); + return (NumericColumn) columnSupplier.get(); } @Override public ComplexColumn getComplexColumn() { - return complexColumn == null ? null : complexColumn.get(); + return (ComplexColumn) columnSupplier.get(); } @Override @@ -95,8 +89,8 @@ public SpatialIndex getSpatialIndex() } @Override - public SettableColumnValueSelector makeSettableColumnValueSelector() + public SettableColumnValueSelector makeNewSettableColumnValueSelector() { - return getCapabilities().getType().makeSettableColumnValueSelector(); + return getCapabilities().getType().makeNewSettableColumnValueSelector(); } } diff --git a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/StringDictionaryEncodedColumn.java similarity index 89% rename from processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java rename to processing/src/main/java/io/druid/segment/column/StringDictionaryEncodedColumn.java index 3b3d8fe4f717..933ab53c6ae3 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/StringDictionaryEncodedColumn.java @@ -25,6 +25,7 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.AbstractDimensionSelector; import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.IdLookup; import io.druid.segment.data.CachingIndexed; @@ -43,15 +44,15 @@ /** */ -public class SimpleDictionaryEncodedColumn implements DictionaryEncodedColumn +public class StringDictionaryEncodedColumn implements DictionaryEncodedColumn { - private final ColumnarInts column; - private final ColumnarMultiInts multiValueColumn; + private final @Nullable ColumnarInts column; + private final @Nullable ColumnarMultiInts multiValueColumn; private final CachingIndexed cachedLookups; - public SimpleDictionaryEncodedColumn( - ColumnarInts singleValueColumn, - ColumnarMultiInts multiValueColumn, + public StringDictionaryEncodedColumn( + @Nullable ColumnarInts singleValueColumn, + @Nullable ColumnarMultiInts multiValueColumn, CachingIndexed cachedLookups ) { @@ -115,7 +116,8 @@ public HistoricalDimensionSelector makeDimensionSelector( @Nullable final ExtractionFn extractionFn ) { - abstract class QueryableDimensionSelector implements HistoricalDimensionSelector, IdLookup + abstract class QueryableDimensionSelector extends AbstractDimensionSelector + implements HistoricalDimensionSelector, IdLookup { @Override public int getValueCardinality() @@ -126,10 +128,8 @@ public int getValueCardinality() @Override public String lookupName(int id) { - final String value = SimpleDictionaryEncodedColumn.this.lookupName(id); - return extractionFn == null ? - value : - extractionFn.apply(value); + final String value = StringDictionaryEncodedColumn.this.lookupName(id); + return extractionFn == null ? value : extractionFn.apply(value); } @Override @@ -151,7 +151,7 @@ public int lookupId(String name) if (extractionFn != null) { throw new UnsupportedOperationException("cannot perform lookup when applying an extraction function"); } - return SimpleDictionaryEncodedColumn.this.lookupId(name); + return StringDictionaryEncodedColumn.this.lookupId(name); } } @@ -171,7 +171,7 @@ public IndexedInts getRow(int offset) } @Override - public ValueMatcher makeValueMatcher(String value) + public ValueMatcher makeValueMatcher(@Nullable String value) { return DimensionSelectorUtils.makeValueMatcherGeneric(this, value); } @@ -236,7 +236,7 @@ public int getRowValue(int offset) } @Override - public ValueMatcher makeValueMatcher(final String value) + public ValueMatcher makeValueMatcher(final @Nullable String value) { if (extractionFn == null) { final int valueId = lookupId(value); @@ -252,7 +252,7 @@ public boolean matches() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - inspector.visit("column", SimpleDictionaryEncodedColumn.this); + inspector.visit("column", StringDictionaryEncodedColumn.this); } }; } else { @@ -293,7 +293,7 @@ public boolean matches() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - inspector.visit("column", SimpleDictionaryEncodedColumn.this); + inspector.visit("column", StringDictionaryEncodedColumn.this); } }; } diff --git a/processing/src/main/java/io/druid/segment/column/ValueType.java b/processing/src/main/java/io/druid/segment/column/ValueType.java index d22a8e98d32e..d06657e3b063 100644 --- a/processing/src/main/java/io/druid/segment/column/ValueType.java +++ b/processing/src/main/java/io/druid/segment/column/ValueType.java @@ -34,6 +34,8 @@ import io.druid.segment.selector.settable.SettableLongColumnValueSelector; import io.druid.segment.selector.settable.SettableObjectColumnValueSelector; +import javax.annotation.Nullable; + /** * Should be the same as {@link io.druid.data.input.impl.DimensionSchema.ValueType}. * TODO merge them when druid-api is merged back into the main repo @@ -44,14 +46,14 @@ public enum ValueType @Override public DimensionSelector makeNumericWrappingDimensionSelector( ColumnValueSelector numericColumnValueSelector, - ExtractionFn extractionFn + @Nullable ExtractionFn extractionFn ) { return new FloatWrappingDimensionSelector(numericColumnValueSelector, extractionFn); } @Override - public SettableColumnValueSelector makeSettableColumnValueSelector() + public SettableColumnValueSelector makeNewSettableColumnValueSelector() { return new SettableFloatColumnValueSelector(); } @@ -60,14 +62,14 @@ public SettableColumnValueSelector makeSettableColumnValueSelector() @Override public DimensionSelector makeNumericWrappingDimensionSelector( ColumnValueSelector numericColumnValueSelector, - ExtractionFn extractionFn + @Nullable ExtractionFn extractionFn ) { return new DoubleWrappingDimensionSelector(numericColumnValueSelector, extractionFn); } @Override - public SettableColumnValueSelector makeSettableColumnValueSelector() + public SettableColumnValueSelector makeNewSettableColumnValueSelector() { return new SettableDoubleColumnValueSelector(); } @@ -76,28 +78,28 @@ public SettableColumnValueSelector makeSettableColumnValueSelector() @Override public DimensionSelector makeNumericWrappingDimensionSelector( ColumnValueSelector numericColumnValueSelector, - ExtractionFn extractionFn + @Nullable ExtractionFn extractionFn ) { return new LongWrappingDimensionSelector(numericColumnValueSelector, extractionFn); } @Override - public SettableColumnValueSelector makeSettableColumnValueSelector() + public SettableColumnValueSelector makeNewSettableColumnValueSelector() { return new SettableLongColumnValueSelector(); } }, STRING { @Override - public SettableColumnValueSelector makeSettableColumnValueSelector() + public SettableColumnValueSelector makeNewSettableColumnValueSelector() { return new SettableDimensionValueSelector(); } }, COMPLEX { @Override - public SettableColumnValueSelector makeSettableColumnValueSelector() + public SettableColumnValueSelector makeNewSettableColumnValueSelector() { return new SettableObjectColumnValueSelector(); } @@ -105,13 +107,13 @@ public SettableColumnValueSelector makeSettableColumnValueSelector() public DimensionSelector makeNumericWrappingDimensionSelector( ColumnValueSelector numericColumnValueSelector, - ExtractionFn extractionFn + @Nullable ExtractionFn extractionFn ) { throw new UnsupportedOperationException("Not a numeric value type: " + name()); } - public abstract SettableColumnValueSelector makeSettableColumnValueSelector(); + public abstract SettableColumnValueSelector makeNewSettableColumnValueSelector(); public boolean isNumeric() { diff --git a/processing/src/main/java/io/druid/segment/column/IndexedComplexColumn.java b/processing/src/main/java/io/druid/segment/column/package-info.java similarity index 58% rename from processing/src/main/java/io/druid/segment/column/IndexedComplexColumn.java rename to processing/src/main/java/io/druid/segment/column/package-info.java index daf0eb3a064f..5814052fda14 100644 --- a/processing/src/main/java/io/druid/segment/column/IndexedComplexColumn.java +++ b/processing/src/main/java/io/druid/segment/column/package-info.java @@ -17,43 +17,7 @@ * under the License. */ +@EverythingIsNonnullByDefault package io.druid.segment.column; -import io.druid.segment.data.Indexed; - -/** -*/ -public class IndexedComplexColumn implements ComplexColumn -{ - private final Indexed column; - private final String typeName; - - public IndexedComplexColumn(String typeName, Indexed column) - { - this.column = column; - this.typeName = typeName; - } - - @Override - public Class getClazz() - { - return column.getClazz(); - } - - @Override - public String getTypeName() - { - return typeName; - } - - @Override - public Object getRowValue(int rowNum) - { - return column.get(rowNum); - } - - @Override - public void close() - { - } -} +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/processing/src/main/java/io/druid/segment/data/BitmapValues.java b/processing/src/main/java/io/druid/segment/data/BitmapValues.java index 9d2528b91c5f..fa96c95312f4 100644 --- a/processing/src/main/java/io/druid/segment/data/BitmapValues.java +++ b/processing/src/main/java/io/druid/segment/data/BitmapValues.java @@ -23,7 +23,7 @@ import it.unimi.dsi.fastutil.ints.IntIterators; /** - * Doesn't extend {@link it.unimi.dsi.fastutil.ints.IntIterable} to avoid accidential for-each iteration with boxing. + * Doesn't extend {@link it.unimi.dsi.fastutil.ints.IntIterable} to avoid accidental for-each iteration with boxing. */ public interface BitmapValues { diff --git a/processing/src/main/java/io/druid/segment/data/CachingIndexed.java b/processing/src/main/java/io/druid/segment/data/CachingIndexed.java index 71043a0786ae..3ba2f64b914b 100644 --- a/processing/src/main/java/io/druid/segment/data/CachingIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/CachingIndexed.java @@ -23,6 +23,7 @@ import io.druid.java.util.common.logger.Logger; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import javax.annotation.Nullable; import java.io.Closeable; import java.util.Iterator; import java.util.LinkedHashMap; @@ -30,11 +31,12 @@ public class CachingIndexed implements Indexed, Closeable { - public static final int INITIAL_CACHE_CAPACITY = 16384; + private static final int INITIAL_CACHE_CAPACITY = 16384; private static final Logger log = new Logger(CachingIndexed.class); private final GenericIndexed.BufferIndexed delegate; + @Nullable private final SizedLRUMap cachedValues; /** @@ -88,7 +90,7 @@ public T get(int index) } @Override - public int indexOf(T value) + public int indexOf(@Nullable T value) { return delegate.indexOf(value); } @@ -120,7 +122,7 @@ private static class SizedLRUMap extends LinkedHashMap private final int maxBytes; private int numBytes = 0; - public SizedLRUMap(int initialCapacity, int maxBytes) + SizedLRUMap(int initialCapacity, int maxBytes) { super(initialCapacity, 0.75f, true); this.maxBytes = maxBytes; @@ -136,13 +138,14 @@ protected boolean removeEldestEntry(Map.Entry> eldest) return false; } - public void put(K key, V value, int size) + public void put(K key, @Nullable V value, int size) { final int totalSize = size + 48; // add approximate object overhead numBytes += totalSize; super.put(key, new Pair<>(totalSize, value)); } + @Nullable public V getValue(Object key) { final Pair sizeValuePair = super.get(key); diff --git a/processing/src/main/java/io/druid/segment/data/ColumnarInts.java b/processing/src/main/java/io/druid/segment/data/ColumnarInts.java index f60e3b2b8781..f111ef8b5138 100644 --- a/processing/src/main/java/io/druid/segment/data/ColumnarInts.java +++ b/processing/src/main/java/io/druid/segment/data/ColumnarInts.java @@ -23,8 +23,8 @@ /** * Resource that provides random access to a packed array of primitive int values. Typically backs up dictionary-encoded - * string columns (see {@link io.druid.segment.column.DictionaryEncodedColumn}), either directly for single-valued - * string columns, or indirectly as part of implementation of {@link ColumnarMultiInts}. + * columns (see {@link io.druid.segment.column.DictionaryEncodedColumn}), either directly for single-valued columns, or + * indirectly as part of implementation of {@link ColumnarMultiInts}. */ public interface ColumnarInts extends IndexedInts, Closeable { diff --git a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java index 044892d9f24a..fdd039267336 100644 --- a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java @@ -58,7 +58,7 @@ public Class getClazz() public WrappedImmutableConciseBitmap fromByteBuffer(ByteBuffer buffer, int numBytes) { buffer.limit(buffer.position() + numBytes); - return new WrappedImmutableConciseBitmap(new ImmutableConciseSet(buffer)); + return new WrappedImmutableConciseBitmap(new ImmutableConciseSet(buffer.asIntBuffer())); } @Override diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 92160cdec714..1b97251472f7 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -93,10 +93,10 @@ public class GenericIndexed implements Indexed, Serializer private static final SerializerUtils SERIALIZER_UTILS = new SerializerUtils(); - public static final ObjectStrategy STRING_STRATEGY = new CacheableObjectStrategy() + public static final ObjectStrategy STRING_STRATEGY = new ObjectStrategy() { @Override - public Class getClazz() + public Class getClazz() { return String.class; } @@ -109,7 +109,7 @@ public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) @Override @Nullable - public byte[] toBytes(String val) + public byte[] toBytes(@Nullable String val) { return StringUtils.toUtf8Nullable(NullHandling.nullToEmptyIfNeeded(val)); } diff --git a/processing/src/main/java/io/druid/segment/data/Indexed.java b/processing/src/main/java/io/druid/segment/data/Indexed.java index a10cfd226211..296d78ac1c94 100644 --- a/processing/src/main/java/io/druid/segment/data/Indexed.java +++ b/processing/src/main/java/io/druid/segment/data/Indexed.java @@ -23,6 +23,8 @@ import io.druid.query.monomorphicprocessing.CalledFromHotLoop; import io.druid.query.monomorphicprocessing.HotLoopCallee; +import javax.annotation.Nullable; + @PublicApi public interface Indexed extends Iterable, HotLoopCallee { @@ -31,6 +33,7 @@ public interface Indexed extends Iterable, HotLoopCallee int size(); @CalledFromHotLoop + @Nullable T get(int index); /** @@ -42,5 +45,5 @@ public interface Indexed extends Iterable, HotLoopCallee * * @return index of value, or a negative number */ - int indexOf(T value); + int indexOf(@Nullable T value); } diff --git a/processing/src/main/java/io/druid/segment/data/IndexedInts.java b/processing/src/main/java/io/druid/segment/data/IndexedInts.java index 06e940cb70c1..3341451f1bb3 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedInts.java @@ -28,7 +28,7 @@ /** * Get a int an index (array or list lookup abstraction without boxing). * - * Doesn't extend {@link Iterable} (or {@link it.unimi.dsi.fastutil.ints.IntIterable} to avoid accidential + * Doesn't extend {@link Iterable} (or {@link it.unimi.dsi.fastutil.ints.IntIterable} to avoid accidental * for-each iteration with boxing. */ @PublicApi diff --git a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java index 7577f2f5bdaf..dfb5bf6161be 100644 --- a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java @@ -42,7 +42,7 @@ public class RoaringBitmapSerdeFactory implements BitmapSerdeFactory @JsonCreator public RoaringBitmapSerdeFactory( - @JsonProperty("compressRunOnSerialization") Boolean compressRunOnSerialization + @JsonProperty("compressRunOnSerialization") @Nullable Boolean compressRunOnSerialization ) { this.compressRunOnSerialization = compressRunOnSerialization == null @@ -85,8 +85,9 @@ public ImmutableBitmap fromByteBuffer(ByteBuffer buffer, int numBytes) return new WrappedImmutableRoaringBitmap(new ImmutableRoaringBitmap(buffer)); } + @Nullable @Override - public byte[] toBytes(ImmutableBitmap val) + public byte[] toBytes(@Nullable ImmutableBitmap val) { if (val == null || val.size() == 0) { return new byte[]{}; diff --git a/processing/src/main/java/io/druid/segment/data/CacheableObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/package-info.java similarity index 78% rename from processing/src/main/java/io/druid/segment/data/CacheableObjectStrategy.java rename to processing/src/main/java/io/druid/segment/data/package-info.java index ef77d47f211b..0e54ffa0fba4 100644 --- a/processing/src/main/java/io/druid/segment/data/CacheableObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/package-info.java @@ -17,10 +17,7 @@ * under the License. */ +@EverythingIsNonnullByDefault package io.druid.segment.data; -/** - * Implementing CacheableObjectStrategy instead of ObjectSrategy indicates - * that a column scan may cache the results of {@link #fromByteBuffer}. - */ -public interface CacheableObjectStrategy extends ObjectStrategy {} +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index d82eda9ce1e1..60f87a27969f 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -188,6 +188,7 @@ public Class classOfObject() return extractor.extractedClass(); } + @Nullable @Override public Object getObject() { @@ -806,6 +807,7 @@ public List getDimensions() } } + @Nullable public DimensionDesc getDimension(String dimension) { synchronized (dimensionDescs) { @@ -989,7 +991,7 @@ public Iterator iterator() continue; } final DimensionIndexer indexer = dimensionDesc.getIndexer(); - Object rowVals = indexer.convertUnsortedEncodedKeyComponentToActualArrayOrList(dim, DimensionIndexer.LIST); + Object rowVals = indexer.convertUnsortedEncodedKeyComponentToActualList(dim); theVals.put(dimensionName, rowVals); } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java index 39119bbb2831..6f5e31bd41ce 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java @@ -26,7 +26,7 @@ import io.druid.segment.DimensionIndexer; import io.druid.segment.DimensionSelector; import io.druid.segment.DimensionSelectorUtils; -import io.druid.segment.SingleScanTimeDimSelector; +import io.druid.segment.SingleScanTimeDimensionSelector; import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; @@ -73,7 +73,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); if (dimension.equals(Column.TIME_COLUMN_NAME)) { - return new SingleScanTimeDimSelector(makeColumnValueSelector(dimension), extractionFn, descending); + return new SingleScanTimeDimensionSelector(makeColumnValueSelector(dimension), extractionFn, descending); } final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimensionSpec.getDimension()); diff --git a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java index ae2c25f5f611..f61066b83453 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java @@ -123,7 +123,7 @@ protected BufferAggregator[] initAggs( selectors.put( agg.getName(), - new OnheapIncrementalIndex.ObjectCachingColumnSelectorFactory(columnSelectorFactory, concurrentEventAdd) + new OnheapIncrementalIndex.CachingColumnSelectorFactory(columnSelectorFactory, concurrentEventAdd) ); if (i == 0) { diff --git a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java index 5404aa07147c..2870db62c49c 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java @@ -129,7 +129,7 @@ protected Aggregator[] initAggs( for (AggregatorFactory agg : metrics) { selectors.put( agg.getName(), - new ObjectCachingColumnSelectorFactory( + new CachingColumnSelectorFactory( makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics), concurrentEventAdd ) @@ -385,16 +385,17 @@ public void close() } } - // Caches references to selector objects for each column instead of creating a new object each time in order to save heap space. - // In general the selectorFactory need not to thread-safe. - // If required, set concurrentEventAdd to true to use concurrent hash map instead of vanilla hash map for thread-safe - // operations. - static class ObjectCachingColumnSelectorFactory implements ColumnSelectorFactory + /** + * Caches references to selector objects for each column instead of creating a new object each time in order to save + * heap space. In general the selectorFactory need not to thread-safe. If required, set concurrentEventAdd to true to + * use concurrent hash map instead of vanilla hash map for thread-safe operations. + */ + static class CachingColumnSelectorFactory implements ColumnSelectorFactory { private final Map> columnSelectorMap; private final ColumnSelectorFactory delegate; - public ObjectCachingColumnSelectorFactory(ColumnSelectorFactory delegate, boolean concurrentEventAdd) + public CachingColumnSelectorFactory(ColumnSelectorFactory delegate, boolean concurrentEventAdd) { this.delegate = delegate; diff --git a/processing/src/main/java/io/druid/segment/selector/settable/SettableColumnValueSelector.java b/processing/src/main/java/io/druid/segment/selector/settable/SettableColumnValueSelector.java index 8483bbaa8aca..da99ed7299fb 100644 --- a/processing/src/main/java/io/druid/segment/selector/settable/SettableColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/selector/settable/SettableColumnValueSelector.java @@ -27,7 +27,7 @@ */ public interface SettableColumnValueSelector extends ColumnValueSelector { - void setValueFrom(ColumnValueSelector selector); + void setValueFrom(ColumnValueSelector selector); @Override default void inspectRuntimeShape(RuntimeShapeInspector inspector) diff --git a/processing/src/main/java/io/druid/segment/selector/settable/SettableDimensionValueSelector.java b/processing/src/main/java/io/druid/segment/selector/settable/SettableDimensionValueSelector.java index ded605f60dac..b85491e9c156 100644 --- a/processing/src/main/java/io/druid/segment/selector/settable/SettableDimensionValueSelector.java +++ b/processing/src/main/java/io/druid/segment/selector/settable/SettableDimensionValueSelector.java @@ -33,14 +33,14 @@ /** * Settable implementation of {@link DimensionSelector}. */ -public class SettableDimensionValueSelector implements SettableColumnValueSelector, DimensionSelector +public class SettableDimensionValueSelector implements DimensionSelector, SettableColumnValueSelector { @Nullable private DimensionSelector keptSelector; private final ArrayBasedIndexedInts keptRow = new ArrayBasedIndexedInts(); @Override - public void setValueFrom(ColumnValueSelector selector) + public void setValueFrom(ColumnValueSelector selector) { DimensionSelector dimensionSelector = (DimensionSelector) selector; keptSelector = dimensionSelector; diff --git a/processing/src/main/java/io/druid/segment/selector/settable/SettableDoubleColumnValueSelector.java b/processing/src/main/java/io/druid/segment/selector/settable/SettableDoubleColumnValueSelector.java index 22d7abdbfa18..05c1051df77e 100644 --- a/processing/src/main/java/io/druid/segment/selector/settable/SettableDoubleColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/selector/settable/SettableDoubleColumnValueSelector.java @@ -29,7 +29,7 @@ public class SettableDoubleColumnValueSelector implements SettableColumnValueSel private double value; @Override - public void setValueFrom(ColumnValueSelector selector) + public void setValueFrom(ColumnValueSelector selector) { isNull = selector.isNull(); if (!isNull) { diff --git a/processing/src/main/java/io/druid/segment/selector/settable/SettableFloatColumnValueSelector.java b/processing/src/main/java/io/druid/segment/selector/settable/SettableFloatColumnValueSelector.java index 99b2fbf49495..500e071fea8a 100644 --- a/processing/src/main/java/io/druid/segment/selector/settable/SettableFloatColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/selector/settable/SettableFloatColumnValueSelector.java @@ -29,7 +29,7 @@ public class SettableFloatColumnValueSelector implements SettableColumnValueSele private float value; @Override - public void setValueFrom(ColumnValueSelector selector) + public void setValueFrom(ColumnValueSelector selector) { isNull = selector.isNull(); if (!isNull) { diff --git a/processing/src/main/java/io/druid/segment/selector/settable/SettableLongColumnValueSelector.java b/processing/src/main/java/io/druid/segment/selector/settable/SettableLongColumnValueSelector.java index d596e328c893..b60a3f96eb37 100644 --- a/processing/src/main/java/io/druid/segment/selector/settable/SettableLongColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/selector/settable/SettableLongColumnValueSelector.java @@ -29,7 +29,7 @@ public class SettableLongColumnValueSelector implements SettableColumnValueSelec private long value; @Override - public void setValueFrom(ColumnValueSelector selector) + public void setValueFrom(ColumnValueSelector selector) { isNull = selector.isNull(); if (!isNull) { diff --git a/processing/src/main/java/io/druid/segment/selector/settable/package-info.java b/processing/src/main/java/io/druid/segment/selector/settable/package-info.java new file mode 100644 index 000000000000..cd5a886eddf1 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/selector/settable/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.segment.selector.settable; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java index 6662e60269bc..6b3ebcad753e 100644 --- a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java @@ -24,6 +24,7 @@ import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; +import javax.annotation.Nullable; import java.nio.ByteBuffer; /** @@ -31,16 +32,17 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { @JsonSubTypes.Type(name = "complex", value = ComplexColumnPartSerde.class), - @JsonSubTypes.Type(name = "float", value = FloatGenericColumnPartSerde.class), - @JsonSubTypes.Type(name = "long", value = LongGenericColumnPartSerde.class), - @JsonSubTypes.Type(name = "double", value = DoubleGenericColumnPartSerde.class), + @JsonSubTypes.Type(name = "float", value = FloatNumericColumnPartSerde.class), + @JsonSubTypes.Type(name = "long", value = LongNumericColumnPartSerde.class), + @JsonSubTypes.Type(name = "double", value = DoubleNumericColumnPartSerde.class), @JsonSubTypes.Type(name = "stringDictionary", value = DictionaryEncodedColumnPartSerde.class), - @JsonSubTypes.Type(name = "floatV2", value = FloatGenericColumnPartSerdeV2.class), - @JsonSubTypes.Type(name = "longV2", value = LongGenericColumnPartSerdeV2.class), - @JsonSubTypes.Type(name = "doubleV2", value = DoubleGenericColumnPartSerdeV2.class), + @JsonSubTypes.Type(name = "floatV2", value = FloatNumericColumnPartSerdeV2.class), + @JsonSubTypes.Type(name = "longV2", value = LongNumericColumnPartSerdeV2.class), + @JsonSubTypes.Type(name = "doubleV2", value = DoubleNumericColumnPartSerdeV2.class), }) public interface ColumnPartSerde { + @Nullable Serializer getSerializer(); Deserializer getDeserializer(); diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java index 432e1bf7a390..78f8e524da2e 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.segment.GenericColumnSerializer; +import io.druid.segment.ColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; @@ -84,7 +84,7 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo public static class SerializerBuilder { private String typeName = null; - private GenericColumnSerializer delegate = null; + private ColumnSerializer delegate = null; public SerializerBuilder withTypeName(final String typeName) { @@ -92,7 +92,7 @@ public SerializerBuilder withTypeName(final String typeName) return this; } - public SerializerBuilder withDelegate(final GenericColumnSerializer delegate) + public SerializerBuilder withDelegate(final ColumnSerializer delegate) { this.delegate = delegate; return this; diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSupplier.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSupplier.java index 631f4447f732..ae6e4e55b04e 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSupplier.java @@ -21,17 +21,16 @@ import com.google.common.base.Supplier; import io.druid.segment.column.ComplexColumn; -import io.druid.segment.column.IndexedComplexColumn; import io.druid.segment.data.GenericIndexed; /** */ public class ComplexColumnPartSupplier implements Supplier { - private final GenericIndexed complexType; + private final GenericIndexed complexType; private final String typeName; - public ComplexColumnPartSupplier(final String typeName, final GenericIndexed complexType) + public ComplexColumnPartSupplier(final String typeName, final GenericIndexed complexType) { this.complexType = complexType; this.typeName = typeName; @@ -40,6 +39,6 @@ public ComplexColumnPartSupplier(final String typeName, final GenericIndexed com @Override public ComplexColumn get() { - return new IndexedComplexColumn(typeName, complexType); + return new ComplexColumn(typeName, complexType); } } diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java index 9135b12c19ff..c1ec5a1b0cf6 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java @@ -23,7 +23,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.ColumnValueSelector; -import io.druid.segment.GenericColumnSerializer; +import io.druid.segment.ColumnSerializer; import io.druid.segment.data.GenericIndexedWriter; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.writeout.SegmentWriteOutMedium; @@ -31,7 +31,7 @@ import java.io.IOException; import java.nio.channels.WritableByteChannel; -public class ComplexColumnSerializer implements GenericColumnSerializer +public class ComplexColumnSerializer implements ColumnSerializer { @PublicApi public static ComplexColumnSerializer create(SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy) @@ -60,7 +60,7 @@ public void open() throws IOException } @Override - public void serialize(ColumnValueSelector selector) throws IOException + public void serialize(ColumnValueSelector selector) throws IOException { writer.write(selector.getObject()); } diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java index b251456834a2..61cb11758658 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java @@ -22,11 +22,15 @@ import io.druid.data.input.InputRow; import io.druid.guice.annotations.ExtensionPoint; +import javax.annotation.Nullable; + /** */ @ExtensionPoint -public interface ComplexMetricExtractor +public interface ComplexMetricExtractor { - Class extractedClass(); - Object extractValue(InputRow inputRow, String metricName); + Class extractedClass(); + + @Nullable + T extractValue(InputRow inputRow, String metricName); } diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index 8698b36e1f53..872e46a7846a 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -21,8 +21,8 @@ import com.google.common.base.Function; import io.druid.guice.annotations.ExtensionPoint; +import io.druid.segment.ColumnSerializer; import io.druid.segment.writeout.SegmentWriteOutMedium; -import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.ObjectStrategy; import it.unimi.dsi.fastutil.bytes.ByteArrays; @@ -110,9 +110,9 @@ public Object fromBytes(byte[] data, int start, int numBytes) * For large column (i.e columns greater than Integer.MAX) use * (@link LargeColumnSupportedComplexColumnSerializer) * - * @return an instance of GenericColumnSerializer used for serialization. + * @return an instance of ColumnSerializer used for serialization. */ - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return ComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetrics.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetrics.java index 609482b581c5..eb88c2f808e5 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetrics.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetrics.java @@ -22,6 +22,7 @@ import com.google.common.collect.Maps; import io.druid.java.util.common.ISE; +import javax.annotation.Nullable; import java.util.Map; /** @@ -30,6 +31,7 @@ public class ComplexMetrics { private static final Map complexSerializers = Maps.newHashMap(); + @Nullable public static ComplexMetricSerde getSerdeForType(String type) { return complexSerializers.get(type); diff --git a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 64e6a190cd4c..98d57da1bf60 100644 --- a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -319,7 +319,9 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo rMultiValuedColumn, columnConfig.columnCacheSizeBytes() ); - builder.setHasMultipleValues(hasMultipleValues).setDictionaryEncodedColumn(dictionaryEncodedColumnSupplier); + builder + .setHasMultipleValues(hasMultipleValues) + .setDictionaryEncodedColumnSupplier(dictionaryEncodedColumnSupplier); if (!Feature.NO_BITMAP_INDEX.isSet(rFlags)) { GenericIndexed rBitmaps = GenericIndexed.read( diff --git a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnSupplier.java index 4323fcd09027..467976ea40dc 100644 --- a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnSupplier.java @@ -21,25 +21,27 @@ import com.google.common.base.Supplier; import io.druid.segment.column.DictionaryEncodedColumn; -import io.druid.segment.column.SimpleDictionaryEncodedColumn; +import io.druid.segment.column.StringDictionaryEncodedColumn; import io.druid.segment.data.CachingIndexed; import io.druid.segment.data.ColumnarInts; import io.druid.segment.data.ColumnarMultiInts; import io.druid.segment.data.GenericIndexed; +import javax.annotation.Nullable; + /** */ -public class DictionaryEncodedColumnSupplier implements Supplier +public class DictionaryEncodedColumnSupplier implements Supplier> { private final GenericIndexed dictionary; - private final Supplier singleValuedColumn; - private final Supplier multiValuedColumn; + private final @Nullable Supplier singleValuedColumn; + private final @Nullable Supplier multiValuedColumn; private final int lookupCacheSize; public DictionaryEncodedColumnSupplier( GenericIndexed dictionary, - Supplier singleValuedColumn, - Supplier multiValuedColumn, + @Nullable Supplier singleValuedColumn, + @Nullable Supplier multiValuedColumn, int lookupCacheSize ) { @@ -50,9 +52,9 @@ public DictionaryEncodedColumnSupplier( } @Override - public DictionaryEncodedColumn get() + public DictionaryEncodedColumn get() { - return new SimpleDictionaryEncodedColumn( + return new StringDictionaryEncodedColumn( singleValuedColumn != null ? singleValuedColumn.get() : null, multiValuedColumn != null ? multiValuedColumn.get() : null, new CachingIndexed<>(dictionary, lookupCacheSize) diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DoubleNumericColumnPartSerde.java similarity index 79% rename from processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java rename to processing/src/main/java/io/druid/segment/serde/DoubleNumericColumnPartSerde.java index 1047c018684e..92bffc6e42e4 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleNumericColumnPartSerde.java @@ -29,23 +29,25 @@ import io.druid.segment.data.ColumnarDoubles; import io.druid.segment.data.CompressedColumnarDoublesSuppliers; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.ByteOrder; -public class DoubleGenericColumnPartSerde implements ColumnPartSerde +public class DoubleNumericColumnPartSerde implements ColumnPartSerde { @JsonCreator - public static DoubleGenericColumnPartSerde getDoubleGenericColumnPartSerde( + public static DoubleNumericColumnPartSerde getDoubleGenericColumnPartSerde( @JsonProperty("byteOrder") ByteOrder byteOrder ) { - return new DoubleGenericColumnPartSerde(byteOrder, null); + return new DoubleNumericColumnPartSerde(byteOrder, null); } private final ByteOrder byteOrder; + @Nullable private final Serializer serializer; - private DoubleGenericColumnPartSerde(ByteOrder byteOrder, Serializer serializer) + private DoubleNumericColumnPartSerde(ByteOrder byteOrder, @Nullable Serializer serializer) { this.byteOrder = byteOrder; this.serializer = serializer; @@ -79,12 +81,13 @@ public SerializerBuilder withDelegate(final Serializer delegate) return this; } - public DoubleGenericColumnPartSerde build() + public DoubleNumericColumnPartSerde build() { - return new DoubleGenericColumnPartSerde(byteOrder, delegate); + return new DoubleNumericColumnPartSerde(byteOrder, delegate); } } + @Nullable @Override public Serializer getSerializer() { @@ -103,10 +106,13 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo buffer, byteOrder ); + DoubleNumericColumnSupplier columnSupplier = new DoubleNumericColumnSupplier( + column, + IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() + ); builder.setType(ValueType.DOUBLE) .setHasMultipleValues(false) - .setGenericColumn(new DoubleGenericColumnSupplier(column, IndexIO.LEGACY_FACTORY.getBitmapFactory() - .makeEmptyImmutableBitmap())); + .setNumericColumnSupplier(columnSupplier); } }; diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java similarity index 79% rename from processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java rename to processing/src/main/java/io/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java index f68bec1f108b..b3f85f2ea4ed 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleNumericColumnPartSerdeV2.java @@ -38,15 +38,15 @@ /** */ -public class DoubleGenericColumnPartSerdeV2 implements ColumnPartSerde +public class DoubleNumericColumnPartSerdeV2 implements ColumnPartSerde { @JsonCreator - public static DoubleGenericColumnPartSerdeV2 getDoubleGenericColumnPartSerde( + public static DoubleNumericColumnPartSerdeV2 getDoubleGenericColumnPartSerde( @JsonProperty("byteOrder") ByteOrder byteOrder, @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory ) { - return new DoubleGenericColumnPartSerdeV2( + return new DoubleNumericColumnPartSerdeV2( byteOrder, bitmapSerdeFactory != null ? bitmapSerdeFactory : new BitmapSerde.LegacyBitmapSerdeFactory(), null @@ -54,18 +54,19 @@ public static DoubleGenericColumnPartSerdeV2 getDoubleGenericColumnPartSerde( } private final ByteOrder byteOrder; - private Serializer serialize; + @Nullable + private Serializer serializer; private final BitmapSerdeFactory bitmapSerdeFactory; - public DoubleGenericColumnPartSerdeV2( + public DoubleNumericColumnPartSerdeV2( ByteOrder byteOrder, BitmapSerdeFactory bitmapSerdeFactory, - Serializer serialize + @Nullable Serializer serializer ) { this.byteOrder = byteOrder; this.bitmapSerdeFactory = bitmapSerdeFactory; - this.serialize = serialize; + this.serializer = serializer; } @JsonProperty @@ -109,33 +110,31 @@ public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFa return this; } - public DoubleGenericColumnPartSerdeV2 build() + public DoubleNumericColumnPartSerdeV2 build() { - return new DoubleGenericColumnPartSerdeV2( - byteOrder, - bitmapSerdeFactory, - new Serializer() - { - @Override - public long getSerializedSize() throws IOException - { - return delegate.getSerializedSize(); - } - - @Override - public void writeTo(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException - { - delegate.writeTo(channel, fileSmoosher); - } - } - ); + Serializer serializer = new Serializer() + { + @Override + public long getSerializedSize() throws IOException + { + return delegate.getSerializedSize(); + } + + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException + { + delegate.writeTo(channel, fileSmoosher); + } + }; + return new DoubleNumericColumnPartSerdeV2(byteOrder, bitmapSerdeFactory, serializer); } } + @Nullable @Override public Serializer getSerializer() { - return serialize; + return serializer; } @Override @@ -158,7 +157,7 @@ public Deserializer getDeserializer() } builder.setType(ValueType.DOUBLE) .setHasMultipleValues(false) - .setGenericColumn(new DoubleGenericColumnSupplier(column, bitmap)); + .setNumericColumnSupplier(new DoubleNumericColumnSupplier(column, bitmap)); }; } } diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/DoubleNumericColumnSupplier.java similarity index 86% rename from processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java rename to processing/src/main/java/io/druid/segment/serde/DoubleNumericColumnSupplier.java index 6c91590b6c2d..6d5976ecb468 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleNumericColumnSupplier.java @@ -22,22 +22,22 @@ import com.google.common.base.Supplier; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.column.DoublesColumn; -import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.NumericColumn; import io.druid.segment.data.ColumnarDoubles; -public class DoubleGenericColumnSupplier implements Supplier +public class DoubleNumericColumnSupplier implements Supplier { private final Supplier column; private final ImmutableBitmap nullValueBitmap; - DoubleGenericColumnSupplier(Supplier column, ImmutableBitmap nullValueBitmap) + DoubleNumericColumnSupplier(Supplier column, ImmutableBitmap nullValueBitmap) { this.column = column; this.nullValueBitmap = nullValueBitmap; } @Override - public GenericColumn get() + public NumericColumn get() { return DoublesColumn.create(column.get(), nullValueBitmap); } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/FloatNumericColumnPartSerde.java similarity index 78% rename from processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java rename to processing/src/main/java/io/druid/segment/serde/FloatNumericColumnPartSerde.java index 9d3810c53df9..cba08dfc5d4e 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatNumericColumnPartSerde.java @@ -27,25 +27,27 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.CompressedColumnarFloatsSupplier; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.ByteOrder; /** */ -public class FloatGenericColumnPartSerde implements ColumnPartSerde +public class FloatNumericColumnPartSerde implements ColumnPartSerde { @JsonCreator - public static FloatGenericColumnPartSerde createDeserializer( + public static FloatNumericColumnPartSerde createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder ) { - return new FloatGenericColumnPartSerde(byteOrder, null); + return new FloatNumericColumnPartSerde(byteOrder, null); } private final ByteOrder byteOrder; + @Nullable private final Serializer serializer; - private FloatGenericColumnPartSerde(ByteOrder byteOrder, Serializer serializer) + private FloatNumericColumnPartSerde(ByteOrder byteOrder, @Nullable Serializer serializer) { this.byteOrder = byteOrder; this.serializer = serializer; @@ -79,12 +81,13 @@ public SerializerBuilder withDelegate(final Serializer delegate) return this; } - public FloatGenericColumnPartSerde build() + public FloatNumericColumnPartSerde build() { - return new FloatGenericColumnPartSerde(byteOrder, delegate); + return new FloatNumericColumnPartSerde(byteOrder, delegate); } } + @Nullable @Override public Serializer getSerializer() { @@ -103,13 +106,13 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo buffer, byteOrder ); + FloatNumericColumnSupplier columnSupplier = new FloatNumericColumnSupplier( + column, + IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() + ); builder.setType(ValueType.FLOAT) .setHasMultipleValues(false) - .setGenericColumn(new FloatGenericColumnSupplier( - column, - IndexIO.LEGACY_FACTORY.getBitmapFactory() - .makeEmptyImmutableBitmap() - )); + .setNumericColumnSupplier(columnSupplier); } }; } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatNumericColumnPartSerdeV2.java similarity index 80% rename from processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java rename to processing/src/main/java/io/druid/segment/serde/FloatNumericColumnPartSerdeV2.java index d1fc5a496963..cb22747f57b3 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatNumericColumnPartSerdeV2.java @@ -35,15 +35,15 @@ /** */ -public class FloatGenericColumnPartSerdeV2 implements ColumnPartSerde +public class FloatNumericColumnPartSerdeV2 implements ColumnPartSerde { @JsonCreator - public static FloatGenericColumnPartSerdeV2 createDeserializer( + public static FloatNumericColumnPartSerdeV2 createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder, @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory ) { - return new FloatGenericColumnPartSerdeV2( + return new FloatNumericColumnPartSerdeV2( byteOrder, bitmapSerdeFactory != null ? bitmapSerdeFactory : new BitmapSerde.LegacyBitmapSerdeFactory(), null @@ -51,13 +51,14 @@ public static FloatGenericColumnPartSerdeV2 createDeserializer( } private final ByteOrder byteOrder; + @Nullable private Serializer serializer; private final BitmapSerdeFactory bitmapSerdeFactory; - private FloatGenericColumnPartSerdeV2( + private FloatNumericColumnPartSerdeV2( ByteOrder byteOrder, BitmapSerdeFactory bitmapSerdeFactory, - Serializer serializer + @Nullable Serializer serializer ) { this.byteOrder = byteOrder; @@ -106,28 +107,27 @@ public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFa return this; } - public FloatGenericColumnPartSerdeV2 build() + public FloatNumericColumnPartSerdeV2 build() { - return new FloatGenericColumnPartSerdeV2( - byteOrder, bitmapSerdeFactory, - new Serializer() - { - @Override - public long getSerializedSize() throws IOException - { - return delegate.getSerializedSize(); - } - - @Override - public void writeTo(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException - { - delegate.writeTo(channel, fileSmoosher); - } - } - ); + Serializer serializer = new Serializer() + { + @Override + public long getSerializedSize() throws IOException + { + return delegate.getSerializedSize(); + } + + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException + { + delegate.writeTo(channel, fileSmoosher); + } + }; + return new FloatNumericColumnPartSerdeV2(byteOrder, bitmapSerdeFactory, serializer); } } + @Nullable @Override public Serializer getSerializer() { @@ -153,7 +153,7 @@ public Deserializer getDeserializer() } builder.setType(ValueType.FLOAT) .setHasMultipleValues(false) - .setGenericColumn(new FloatGenericColumnSupplier(column, bitmap)); + .setNumericColumnSupplier(new FloatNumericColumnSupplier(column, bitmap)); }; } } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/FloatNumericColumnSupplier.java similarity index 88% rename from processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java rename to processing/src/main/java/io/druid/segment/serde/FloatNumericColumnSupplier.java index 9af3adf5971a..2d54af6b11dd 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatNumericColumnSupplier.java @@ -22,17 +22,17 @@ import com.google.common.base.Supplier; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.column.FloatsColumn; -import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.NumericColumn; import io.druid.segment.data.CompressedColumnarFloatsSupplier; /** */ -public class FloatGenericColumnSupplier implements Supplier +public class FloatNumericColumnSupplier implements Supplier { private final CompressedColumnarFloatsSupplier column; private final ImmutableBitmap nullValueBitmap; - public FloatGenericColumnSupplier( + public FloatNumericColumnSupplier( CompressedColumnarFloatsSupplier column, ImmutableBitmap nullValueBitmap ) @@ -42,7 +42,7 @@ public FloatGenericColumnSupplier( } @Override - public GenericColumn get() + public NumericColumn get() { return FloatsColumn.create(column.get(), nullValueBitmap); } diff --git a/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java b/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java index a0bf176a3688..a5e14827f980 100644 --- a/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java @@ -23,7 +23,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.ColumnValueSelector; -import io.druid.segment.GenericColumnSerializer; +import io.druid.segment.ColumnSerializer; import io.druid.segment.data.GenericIndexedWriter; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.writeout.SegmentWriteOutMedium; @@ -31,33 +31,38 @@ import java.io.IOException; import java.nio.channels.WritableByteChannel; -public class LargeColumnSupportedComplexColumnSerializer implements GenericColumnSerializer +public class LargeColumnSupportedComplexColumnSerializer implements ColumnSerializer { @PublicApi - public static LargeColumnSupportedComplexColumnSerializer create( + public static LargeColumnSupportedComplexColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy ) { - return new LargeColumnSupportedComplexColumnSerializer(segmentWriteOutMedium, filenameBase, strategy); + return new LargeColumnSupportedComplexColumnSerializer<>(segmentWriteOutMedium, filenameBase, strategy); } - public static LargeColumnSupportedComplexColumnSerializer createWithColumnSize( + public static LargeColumnSupportedComplexColumnSerializer createWithColumnSize( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy, int columnSize ) { - return new LargeColumnSupportedComplexColumnSerializer(segmentWriteOutMedium, filenameBase, strategy, columnSize); + return new LargeColumnSupportedComplexColumnSerializer<>( + segmentWriteOutMedium, + filenameBase, + strategy, + columnSize + ); } private final SegmentWriteOutMedium segmentWriteOutMedium; private final String filenameBase; private final ObjectStrategy strategy; private final int columnSize; - private GenericIndexedWriter writer; + private GenericIndexedWriter writer; private LargeColumnSupportedComplexColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, @@ -95,7 +100,7 @@ public void open() throws IOException } @Override - public void serialize(ColumnValueSelector selector) throws IOException + public void serialize(ColumnValueSelector selector) throws IOException { writer.write(selector.getObject()); } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/LongNumericColumnPartSerde.java similarity index 78% rename from processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java rename to processing/src/main/java/io/druid/segment/serde/LongNumericColumnPartSerde.java index bc926713b9bd..af4067f129e0 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/LongNumericColumnPartSerde.java @@ -27,25 +27,27 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.CompressedColumnarLongsSupplier; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.ByteOrder; /** */ -public class LongGenericColumnPartSerde implements ColumnPartSerde +public class LongNumericColumnPartSerde implements ColumnPartSerde { @JsonCreator - public static LongGenericColumnPartSerde createDeserializer( + public static LongNumericColumnPartSerde createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder ) { - return new LongGenericColumnPartSerde(byteOrder, null); + return new LongNumericColumnPartSerde(byteOrder, null); } private final ByteOrder byteOrder; + @Nullable private final Serializer serializer; - private LongGenericColumnPartSerde(ByteOrder byteOrder, Serializer serializer) + private LongNumericColumnPartSerde(ByteOrder byteOrder, @Nullable Serializer serializer) { this.byteOrder = byteOrder; this.serializer = serializer; @@ -79,12 +81,13 @@ public SerializerBuilder withDelegate(final Serializer delegate) return this; } - public LongGenericColumnPartSerde build() + public LongNumericColumnPartSerde build() { - return new LongGenericColumnPartSerde(byteOrder, delegate); + return new LongNumericColumnPartSerde(byteOrder, delegate); } } + @Nullable @Override public Serializer getSerializer() { @@ -103,13 +106,13 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo buffer, byteOrder ); + LongNumericColumnSupplier columnSupplier = new LongNumericColumnSupplier( + column, + IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() + ); builder.setType(ValueType.LONG) .setHasMultipleValues(false) - .setGenericColumn(new LongGenericColumnSupplier( - column, - IndexIO.LEGACY_FACTORY.getBitmapFactory() - .makeEmptyImmutableBitmap() - )); + .setNumericColumnSupplier(columnSupplier); } }; } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongNumericColumnPartSerdeV2.java similarity index 79% rename from processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java rename to processing/src/main/java/io/druid/segment/serde/LongNumericColumnPartSerdeV2.java index a49c6e56b966..d21b8805ac7e 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/LongNumericColumnPartSerdeV2.java @@ -35,15 +35,15 @@ /** */ -public class LongGenericColumnPartSerdeV2 implements ColumnPartSerde +public class LongNumericColumnPartSerdeV2 implements ColumnPartSerde { @JsonCreator - public static LongGenericColumnPartSerdeV2 createDeserializer( + public static LongNumericColumnPartSerdeV2 createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder, @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory ) { - return new LongGenericColumnPartSerdeV2( + return new LongNumericColumnPartSerdeV2( byteOrder, bitmapSerdeFactory != null ? bitmapSerdeFactory : new BitmapSerde.LegacyBitmapSerdeFactory(), null @@ -51,12 +51,14 @@ public static LongGenericColumnPartSerdeV2 createDeserializer( } private final ByteOrder byteOrder; + @Nullable private Serializer serializer; private final BitmapSerdeFactory bitmapSerdeFactory; - private LongGenericColumnPartSerdeV2( + private LongNumericColumnPartSerdeV2( ByteOrder byteOrder, - BitmapSerdeFactory bitmapSerdeFactory, Serializer serializer + BitmapSerdeFactory bitmapSerdeFactory, + @Nullable Serializer serializer ) { this.byteOrder = byteOrder; @@ -105,28 +107,27 @@ public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFa return this; } - public LongGenericColumnPartSerdeV2 build() + public LongNumericColumnPartSerdeV2 build() { - return new LongGenericColumnPartSerdeV2( - byteOrder, bitmapSerdeFactory, - new Serializer() - { - @Override - public long getSerializedSize() throws IOException - { - return delegate.getSerializedSize(); - } - - @Override - public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException - { - delegate.writeTo(channel, smoosher); - } - } - ); + Serializer serializer = new Serializer() + { + @Override + public long getSerializedSize() throws IOException + { + return delegate.getSerializedSize(); + } + + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + { + delegate.writeTo(channel, smoosher); + } + }; + return new LongNumericColumnPartSerdeV2(byteOrder, bitmapSerdeFactory, serializer); } } + @Nullable @Override public Serializer getSerializer() { @@ -152,7 +153,7 @@ public Deserializer getDeserializer() } builder.setType(ValueType.LONG) .setHasMultipleValues(false) - .setGenericColumn(new LongGenericColumnSupplier(column, bitmap)); + .setNumericColumnSupplier(new LongNumericColumnSupplier(column, bitmap)); }; } } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/LongNumericColumnSupplier.java similarity index 87% rename from processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java rename to processing/src/main/java/io/druid/segment/serde/LongNumericColumnSupplier.java index c70d4e4dc15f..615a691e18f4 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/LongNumericColumnSupplier.java @@ -21,25 +21,25 @@ import com.google.common.base.Supplier; import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.NumericColumn; import io.druid.segment.column.LongsColumn; import io.druid.segment.data.CompressedColumnarLongsSupplier; /** */ -public class LongGenericColumnSupplier implements Supplier +public class LongNumericColumnSupplier implements Supplier { private final CompressedColumnarLongsSupplier column; private final ImmutableBitmap nullValueBitmap; - public LongGenericColumnSupplier(CompressedColumnarLongsSupplier column, ImmutableBitmap nullValueBitmap) + public LongNumericColumnSupplier(CompressedColumnarLongsSupplier column, ImmutableBitmap nullValueBitmap) { this.column = column; this.nullValueBitmap = nullValueBitmap; } @Override - public GenericColumn get() + public NumericColumn get() { return LongsColumn.create(column.get(), nullValueBitmap); } diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 3aae3b797d7c..62631640838b 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -33,6 +33,7 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseSingleValueDimensionSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.ConstantColumnValueSelector; diff --git a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java index 2b2db902d9ea..1d89396f8917 100644 --- a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java @@ -44,7 +44,9 @@ public class SingleStringInputCachingExpressionColumnValueSelector implements Co private final DimensionSelector selector; private final Expr expression; private final Expr.ObjectBinding bindings; + @Nullable private final ExprEval[] arrayEvalCache; + @Nullable private final LruEvalCache lruEvalCache; public SingleStringInputCachingExpressionColumnValueSelector( diff --git a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputDimensionSelector.java b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputDimensionSelector.java index 8f070435cac5..8e8673cc0d56 100644 --- a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputDimensionSelector.java @@ -139,7 +139,7 @@ public String lookupName(final int id) @Override public boolean nameLookupPossibleInAdvance() { - return true; + return selector.nameLookupPossibleInAdvance(); } @Nullable @@ -153,7 +153,7 @@ public IdLookup idLookup() @Override public Object getObject() { - return defaultGetObject(); + return lookupName(getRow().get(0)); } @Override diff --git a/processing/src/main/java/io/druid/segment/virtual/package-info.java b/processing/src/main/java/io/druid/segment/virtual/package-info.java new file mode 100644 index 000000000000..8c0f70f313df --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@EverythingIsNonnullByDefault +package io.druid.segment.virtual; + +import io.druid.annotations.EverythingIsNonnullByDefault; diff --git a/processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java b/processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java index 8b7b10b1c4a8..a3a6d26ef24a 100644 --- a/processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java +++ b/processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java @@ -73,7 +73,7 @@ protected static ImmutableConciseSet makeOffheapConcise(ImmutableConciseSet conc conciseCount++; final ByteBuffer buf = ByteBuffer.allocateDirect(bytes.length).put(bytes); buf.rewind(); - return new ImmutableConciseSet(buf); + return new ImmutableConciseSet(buf.asIntBuffer()); } protected static ImmutableRoaringBitmap writeImmutable(MutableRoaringBitmap r, ByteBuffer buf) throws IOException diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 584f08a2b810..f37caeeaaa55 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -39,6 +39,7 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.query.ordering.StringComparators; import io.druid.query.search.ContainsSearchQuerySpec; +import io.druid.segment.AbstractDimensionSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; @@ -95,7 +96,7 @@ public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) if (dimensionName.equals("dim")) { return dimensionSpec.decorate( - new DimensionSelector() + new AbstractDimensionSelector() { @Override public IndexedInts getRow() @@ -167,13 +168,6 @@ public int lookupId(String name) }; } - @Nullable - @Override - public Object getObject() - { - return defaultGetObject(); - } - @Override public Class classOfObject() { diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index 1249049e1df2..1d9872e646ed 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -44,6 +44,7 @@ import io.druid.query.extraction.RegexDimExtractionFn; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.AbstractDimensionSelector; import io.druid.segment.DimensionSelector; import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.IdLookup; @@ -58,7 +59,7 @@ public class CardinalityAggregatorTest { - public static class TestDimensionSelector implements DimensionSelector + public static class TestDimensionSelector extends AbstractDimensionSelector { private final List column; private final Map ids; @@ -192,13 +193,6 @@ public int lookupId(String s) }; } - @Nullable - @Override - public Object getObject() - { - return defaultGetObject(); - } - @Override public Class classOfObject() { diff --git a/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactoryTest.java b/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactoryTest.java index f183f49fcf9b..ce913f709329 100644 --- a/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactoryTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactoryTest.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; -import io.druid.hll.HLLCV0; +import io.druid.hll.VersionZeroHyperLogLogCollector; import io.druid.hll.HyperLogLogCollector; import io.druid.java.util.common.StringUtils; import io.druid.query.aggregation.AggregatorFactory; @@ -47,7 +47,7 @@ public class HyperUniquesAggregatorFactoryTest public void testDeserializeV0() { Object v0 = aggregatorFactory.deserialize(V0_BASE64); - Assert.assertEquals("deserialized value is HLLCV0", HLLCV0.class, v0.getClass()); + Assert.assertEquals("deserialized value is VersionZeroHyperLogLogCollector", VersionZeroHyperLogLogCollector.class, v0.getClass()); } @Test diff --git a/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java b/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java index 85aa6beb7555..bbeec61beeb6 100644 --- a/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java +++ b/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java @@ -22,7 +22,7 @@ import com.google.common.base.Predicate; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.DimensionSelector; +import io.druid.segment.AbstractDimensionSelector; import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.IdLookup; import io.druid.segment.data.IndexedInts; @@ -35,7 +35,7 @@ * encoding 0 -> a, 1 -> b, ... * row -> [c,e,g] */ -class TestDimensionSelector implements DimensionSelector +class TestDimensionSelector extends AbstractDimensionSelector { public static final TestDimensionSelector instance = new TestDimensionSelector(); @@ -94,13 +94,6 @@ public int lookupId(String name) }; } - @Nullable - @Override - public Object getObject() - { - return defaultGetObject(); - } - @Override public Class classOfObject() { diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index a334a7646544..7fdbe5b9bd9d 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -46,7 +46,7 @@ import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.DictionaryEncodedColumn; -import io.druid.segment.column.SimpleDictionaryEncodedColumn; +import io.druid.segment.column.StringDictionaryEncodedColumn; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.BitmapValues; import io.druid.segment.data.CompressionFactory; @@ -725,12 +725,12 @@ private void assertDimCompression(QueryableIndex index, CompressionStrategy expe DictionaryEncodedColumn encodedColumn = index.getColumn("dim2").getDictionaryEncoding(); Object obj; if (encodedColumn.hasMultipleValues()) { - Field field = SimpleDictionaryEncodedColumn.class.getDeclaredField("multiValueColumn"); + Field field = StringDictionaryEncodedColumn.class.getDeclaredField("multiValueColumn"); field.setAccessible(true); obj = field.get(encodedColumn); } else { - Field field = SimpleDictionaryEncodedColumn.class.getDeclaredField("column"); + Field field = StringDictionaryEncodedColumn.class.getDeclaredField("column"); field.setAccessible(true); obj = field.get(encodedColumn); diff --git a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java index 3ddc0d7633b2..567ba71a1dd6 100644 --- a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java +++ b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java @@ -24,8 +24,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogCollector; +import io.druid.segment.ColumnSerializer; import io.druid.segment.writeout.SegmentWriteOutMedium; -import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -109,7 +109,7 @@ public void deserializeColumn( column = GenericIndexed.read(byteBuffer, getObjectStrategy(), columnBuilder.getFileMapper()); } - columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); + columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); } @Override @@ -152,7 +152,7 @@ public int compare(HyperLogLogCollector o1, HyperLogLogCollector o2) } @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String metric) + public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String metric) { return LargeColumnSupportedComplexColumnSerializer.createWithColumnSize( segmentWriteOutMedium, diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionColumnValueSelectorTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionColumnValueSelectorTest.java index 8e8e8c1349e4..4e5d29e346c4 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionColumnValueSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionColumnValueSelectorTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Supplier; import io.druid.common.guava.SettableSupplier; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.BaseSingleValueDimensionSelector; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; import io.druid.segment.TestObjectColumnSelector; diff --git a/services/src/main/java/io/druid/cli/DumpSegment.java b/services/src/main/java/io/druid/cli/DumpSegment.java index 3d82fbf8d3ce..d1a1c3ed2365 100644 --- a/services/src/main/java/io/druid/cli/DumpSegment.java +++ b/services/src/main/java/io/druid/cli/DumpSegment.java @@ -49,7 +49,6 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.Granularities; -import io.druid.java.util.common.guava.Accumulator; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; @@ -65,10 +64,9 @@ import io.druid.query.metadata.metadata.ListColumnIncluderator; import io.druid.query.metadata.metadata.SegmentAnalysis; import io.druid.query.metadata.metadata.SegmentMetadataQuery; -import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.query.spec.SpecificSegmentSpec; import io.druid.segment.BaseObjectColumnValueSelector; -import io.druid.segment.ColumnValueSelector; +import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.Cursor; import io.druid.segment.IndexIO; import io.druid.segment.QueryableIndex; @@ -87,16 +85,15 @@ import org.joda.time.chrono.ISOChronology; import org.roaringbitmap.IntIterator; -import javax.annotation.Nullable; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @Command( name = "dump-segment", @@ -275,13 +272,11 @@ public Object apply(final OutputStream out) @Override public Object apply(Cursor cursor) { - final List selectors = Lists.newArrayList(); - - for (String columnName : columnNames) { - ColumnValueSelector selector = - cursor.getColumnSelectorFactory().makeColumnValueSelector(columnName); - selectors.add(new ListObjectSelector(selector)); - } + ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory(); + final List selectors = columnNames + .stream() + .map(columnSelectorFactory::makeColumnValueSelector) + .collect(Collectors.toList()); while (!cursor.isDone()) { final Map row = Maps.newLinkedHashMap(); @@ -481,84 +476,17 @@ public int columnCacheSizeBytes() private static Sequence executeQuery(final Injector injector, final QueryableIndex index, final Query query) { final QueryRunnerFactoryConglomerate conglomerate = injector.getInstance(QueryRunnerFactoryConglomerate.class); - final QueryRunnerFactory factory = conglomerate.findFactory(query); + final QueryRunnerFactory> factory = conglomerate.findFactory(query); final QueryRunner runner = factory.createRunner(new QueryableIndexSegment("segment", index)); - final Sequence results = factory.getToolchest().mergeResults( - factory.mergeRunners(MoreExecutors.sameThreadExecutor(), ImmutableList.of(runner)) - ).run(QueryPlus.wrap(query), Maps.newHashMap()); - return (Sequence) results; + return factory + .getToolchest() + .mergeResults(factory.mergeRunners(MoreExecutors.sameThreadExecutor(), ImmutableList.of(runner))) + .run(QueryPlus.wrap(query), Maps.newHashMap()); } private static void evaluateSequenceForSideEffects(final Sequence sequence) { - sequence.accumulate( - null, - new Accumulator() - { - @Override - public Object accumulate(Object accumulated, T in) - { - return null; - } - } - ); + sequence.accumulate(null, (accumulated, in) -> null); } - private static class ListObjectSelector implements ColumnValueSelector - { - private final ColumnValueSelector delegate; - - private ListObjectSelector(ColumnValueSelector delegate) - { - this.delegate = delegate; - } - - @Override - public double getDouble() - { - return delegate.getDouble(); - } - - @Override - public float getFloat() - { - return delegate.getFloat(); - } - - @Override - public long getLong() - { - return delegate.getLong(); - } - - @Nullable - @Override - public Object getObject() - { - Object object = delegate.getObject(); - if (object instanceof String[]) { - return Arrays.asList((String[]) object); - } else { - return object; - } - } - - @Override - public Class classOfObject() - { - return Object.class; - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("delegate", delegate); - } - - @Override - public boolean isNull() - { - return delegate.isNull(); - } - } } diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index d558f777b425..a9372226a472 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import io.druid.hll.HLLCV1; +import io.druid.hll.VersionOneHyperLogLogCollector; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.StringUtils; @@ -557,12 +557,12 @@ public void testSelectStar() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, VersionOneHyperLogLogCollector.class.getName()} ) ); } @@ -589,7 +589,7 @@ public void testSelectStarOnForbiddenTable() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "forbidden", "abcd", 9999.0f, 0.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-01"), 1L, "forbidden", "abcd", 9999.0f, 0.0, VersionOneHyperLogLogCollector.class.getName()} ) ); } @@ -644,8 +644,8 @@ public void testSelectStarWithLimit() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2.0f, 2.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", "", 2.0f, 2.0, VersionOneHyperLogLogCollector.class.getName()} ) ); } @@ -693,8 +693,8 @@ public void testSelectStarWithLimitTimeDescending() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6d, HLLCV1.class.getName()}, - new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5d, HLLCV1.class.getName()} + new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6d, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5d, VersionOneHyperLogLogCollector.class.getName()} ) ); } @@ -733,12 +733,12 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, VersionOneHyperLogLogCollector.class.getName()} ) ); } @@ -1898,9 +1898,9 @@ public void testSelectStarWithDimFilter() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0d, HLLCV1.class.getName()}, - new Object[]{T("2001-01-01"), 1L, "1", "a", 4.0f, 4.0d, HLLCV1.class.getName()}, - new Object[]{T("2001-01-02"), 1L, "def", "abc", 5.0f, 5.0d, HLLCV1.class.getName()} + new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0d, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2001-01-01"), 1L, "1", "a", 4.0f, 4.0d, VersionOneHyperLogLogCollector.class.getName()}, + new Object[]{T("2001-01-02"), 1L, "def", "abc", 5.0f, 5.0d, VersionOneHyperLogLogCollector.class.getName()} ) ); } From ff5141d3657f006fa763c8f7c6301ddd392aa0af Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 6 Jul 2018 18:12:01 +0200 Subject: [PATCH 02/29] Fixes --- .../druid/benchmark/BoundFilterBenchmark.java | 16 +---- .../DimensionPredicateFilterBenchmark.java | 4 +- .../ExpressionSelectorBenchmark.java | 4 +- .../benchmark/FilterPartitionBenchmark.java | 8 +-- .../druid/benchmark/LikeFilterBenchmark.java | 4 +- .../benchmark/query/TimeseriesBenchmark.java | 6 +- .../indexing/kafka/KafkaIndexTaskTest.java | 2 +- .../indexing/common/task/CompactionTask.java | 8 +-- .../common/task/CompactionTaskTest.java | 12 ++-- .../IngestSegmentFirehoseFactoryTest.java | 4 +- .../SimpleDoubleAggregatorFactory.java | 4 +- .../first/DoubleFirstAggregatorFactory.java | 10 +-- .../first/FloatFirstAggregatorFactory.java | 8 +-- .../first/LongFirstAggregatorFactory.java | 8 +-- .../last/DoubleLastAggregatorFactory.java | 10 +-- .../last/FloatLastAggregatorFactory.java | 8 +-- .../last/LongLastAggregatorFactory.java | 8 +-- .../query/filter/BitmapIndexSelector.java | 9 ++- .../io/druid/query/groupby/GroupByQuery.java | 6 +- .../RowBasedColumnSelectorFactory.java | 8 +-- .../druid/query/metadata/SegmentAnalyzer.java | 26 +++---- .../io/druid/query/scan/ScanQueryEngine.java | 8 +-- .../io/druid/query/search/AutoStrategy.java | 8 +-- .../query/search/UseIndexesStrategy.java | 12 ++-- .../druid/query/select/SelectQueryEngine.java | 4 +- .../TimeBoundaryQueryRunnerFactory.java | 4 +- .../io/druid/query/topn/TopNQueryEngine.java | 4 +- .../java/io/druid/segment/AbstractIndex.java | 4 +- .../java/io/druid/segment/ColumnSelector.java | 4 +- .../ColumnSelectorBitmapIndexSelector.java | 60 +++++++++------- .../io/druid/segment/DimensionIndexer.java | 4 +- .../druid/segment/DoubleDimensionIndexer.java | 4 +- .../druid/segment/FloatDimensionIndexer.java | 4 +- .../main/java/io/druid/segment/IndexIO.java | 12 ++-- .../java/io/druid/segment/IndexMergerV9.java | 4 +- .../io/druid/segment/IndexableAdapter.java | 6 +- .../druid/segment/LongDimensionIndexer.java | 4 +- .../QueryableIndexColumnSelectorFactory.java | 60 ++++++++-------- .../QueryableIndexIndexableAdapter.java | 69 ++++++++----------- .../segment/QueryableIndexStorageAdapter.java | 60 +++++++++------- .../segment/RowFilteringIndexAdapter.java | 4 +- .../druid/segment/SimpleQueryableIndex.java | 18 ++--- .../druid/segment/StringDimensionIndexer.java | 12 +++- .../java/io/druid/segment/VirtualColumns.java | 4 +- .../druid/segment/column/ColumnBuilder.java | 4 +- .../segment/column/ColumnDescriptor.java | 2 +- .../column/{Column.java => ColumnHolder.java} | 11 +-- ...pleColumn.java => SimpleColumnHolder.java} | 30 ++++---- .../druid/segment/data/CloseableIndexed.java | 26 +++++++ .../io/druid/segment/data/GenericIndexed.java | 8 ++- .../java/io/druid/segment/filter/Filters.java | 57 +++++++++------ .../io/druid/segment/filter/LikeFilter.java | 18 +++-- .../segment/incremental/IncrementalIndex.java | 4 +- .../incremental/IncrementalIndexAdapter.java | 4 +- ...IncrementalIndexColumnSelectorFactory.java | 6 +- .../IncrementalIndexStorageAdapter.java | 4 +- .../transform/ExpressionTransform.java | 4 +- .../druid/segment/transform/Transformer.java | 8 +-- .../segment/virtual/ExpressionSelectors.java | 6 +- .../io/druid/query/DoubleStorageTest.java | 10 +-- .../first/DoubleFirstAggregationTest.java | 4 +- .../first/FloatFirstAggregationTest.java | 4 +- .../first/LongFirstAggregationTest.java | 4 +- .../last/DoubleLastAggregationTest.java | 4 +- .../last/FloatLastAggregationTest.java | 4 +- .../last/LongLastAggregationTest.java | 4 +- .../filter/GetDimensionRangeSetTest.java | 6 +- .../query/filter/IntervalDimFilterTest.java | 34 ++++----- ...roupByLimitPushDownMultiNodeMergeTest.java | 4 +- .../query/groupby/GroupByQueryRunnerTest.java | 18 ++--- .../groupby/GroupByQueryRunnerTestHelper.java | 4 +- .../druid/query/scan/ScanQueryRunnerTest.java | 6 +- .../query/search/SearchQueryRunnerTest.java | 12 ++-- .../query/select/SelectQueryRunnerTest.java | 34 ++++----- .../druid/query/topn/TopNQueryRunnerTest.java | 10 +-- .../java/io/druid/segment/EmptyIndexTest.java | 4 +- .../segment/IndexMergerNullHandlingTest.java | 13 ++-- .../io/druid/segment/IndexMergerTestBase.java | 50 +++++++------- .../filter/ExtractionDimFilterTest.java | 59 +++++++++++++++- .../segment/filter/TimeFilteringTest.java | 48 ++++++------- ...nSupportedComplexColumnSerializerTest.java | 6 +- .../firehose/IngestSegmentFirehose.java | 4 +- .../main/java/io/druid/cli/DumpSegment.java | 10 +-- .../sql/calcite/expression/Expressions.java | 8 +-- .../MoveTimeFiltersToIntervals.java | 4 +- .../io/druid/sql/calcite/rel/DruidQuery.java | 4 +- .../io/druid/sql/calcite/rel/QueryMaker.java | 4 +- .../druid/sql/calcite/table/RowSignature.java | 4 +- .../druid/sql/calcite/CalciteQueryTest.java | 4 +- .../calcite/filtration/FiltrationTest.java | 6 +- 90 files changed, 617 insertions(+), 511 deletions(-) rename processing/src/main/java/io/druid/segment/column/{Column.java => ColumnHolder.java} (91%) rename processing/src/main/java/io/druid/segment/column/{SimpleColumn.java => SimpleColumnHolder.java} (81%) create mode 100644 processing/src/main/java/io/druid/segment/data/CloseableIndexed.java diff --git a/benchmarks/src/main/java/io/druid/benchmark/BoundFilterBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/BoundFilterBenchmark.java index 17e48fed0c8e..c68af7b81f1c 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/BoundFilterBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/BoundFilterBenchmark.java @@ -33,8 +33,8 @@ import io.druid.query.ordering.StringComparators; import io.druid.segment.column.BitmapIndex; import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.Indexed; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.filter.BoundFilter; import io.druid.segment.serde.BitmapIndexColumnPartSupplier; @@ -158,17 +158,7 @@ public void setup() final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null); final List ints = generateInts(); final GenericIndexed dictionary = GenericIndexed.fromIterable( - FluentIterable.from(ints) - .transform( - new Function() - { - @Override - public String apply(Integer i) - { - return i.toString(); - } - } - ), + FluentIterable.from(ints).transform(i -> i.toString()), GenericIndexed.STRING_STRATEGY ); final BitmapIndex bitmapIndex = new BitmapIndexColumnPartSupplier( @@ -194,7 +184,7 @@ public ImmutableBitmap apply(Integer i) selector = new BitmapIndexSelector() { @Override - public Indexed getDimensionValues(String dimension) + public CloseableIndexed getDimensionValues(String dimension) { return dictionary; } diff --git a/benchmarks/src/main/java/io/druid/benchmark/DimensionPredicateFilterBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/DimensionPredicateFilterBenchmark.java index f1be5083f168..767eb309a726 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/DimensionPredicateFilterBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/DimensionPredicateFilterBenchmark.java @@ -35,8 +35,8 @@ import io.druid.query.filter.DruidPredicateFactory; import io.druid.segment.column.BitmapIndex; import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.Indexed; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.filter.DimensionPredicateFilter; import io.druid.segment.serde.BitmapIndexColumnPartSupplier; @@ -155,7 +155,7 @@ public ImmutableBitmap apply(Integer i) selector = new BitmapIndexSelector() { @Override - public Indexed getDimensionValues(String dimension) + public CloseableIndexed getDimensionValues(String dimension) { return dictionary; } diff --git a/benchmarks/src/main/java/io/druid/benchmark/ExpressionSelectorBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/ExpressionSelectorBenchmark.java index 8f5130acf989..4daef198bc11 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/ExpressionSelectorBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/ExpressionSelectorBenchmark.java @@ -37,7 +37,7 @@ import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexStorageAdapter; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; import io.druid.timeline.DataSegment; @@ -175,7 +175,7 @@ public void timeFloorUsingExtractionFn(Blackhole blackhole) .getColumnSelectorFactory() .makeDimensionSelector( new ExtractionDimensionSpec( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, "v", new TimeFormatExtractionFn(null, null, null, Granularities.HOUR, true) ) diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java index 11ffcf007cb6..ea2540b3b894 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java @@ -61,7 +61,7 @@ import io.druid.segment.QueryableIndexStorageAdapter; import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnConfig; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.AndFilter; @@ -186,7 +186,7 @@ public void setup() throws IOException Interval interval = schemaInfo.getDataInterval(); timeFilterNone = new BoundFilter(new BoundDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, String.valueOf(Long.MAX_VALUE), String.valueOf(Long.MAX_VALUE), true, @@ -198,7 +198,7 @@ public void setup() throws IOException long halfEnd = (interval.getEndMillis() + interval.getStartMillis()) / 2; timeFilterHalf = new BoundFilter(new BoundDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, String.valueOf(interval.getStartMillis()), String.valueOf(halfEnd), true, @@ -209,7 +209,7 @@ public void setup() throws IOException )); timeFilterAll = new BoundFilter(new BoundDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, String.valueOf(interval.getStartMillis()), String.valueOf(interval.getEndMillis()), true, diff --git a/benchmarks/src/main/java/io/druid/benchmark/LikeFilterBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/LikeFilterBenchmark.java index 0847b165b2dc..03998db2ba09 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/LikeFilterBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/LikeFilterBenchmark.java @@ -35,8 +35,8 @@ import io.druid.query.ordering.StringComparators; import io.druid.segment.column.BitmapIndex; import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.Indexed; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.serde.BitmapIndexColumnPartSupplier; import org.openjdk.jmh.annotations.Benchmark; @@ -155,7 +155,7 @@ public ImmutableBitmap apply(Integer i) selector = new BitmapIndexSelector() { @Override - public Indexed getDimensionValues(String dimension) + public CloseableIndexed getDimensionValues(String dimension) { return dictionary; } diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/TimeseriesBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/TimeseriesBenchmark.java index e39e7c242ebe..cd26cc9cbc9a 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/TimeseriesBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/TimeseriesBenchmark.java @@ -67,7 +67,7 @@ import io.druid.segment.IndexSpec; import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexSegment; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnConfig; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.serde.ComplexMetrics; @@ -179,7 +179,7 @@ private void setupQueries() List queryAggs = new ArrayList<>(); LongSumAggregatorFactory lsaf = new LongSumAggregatorFactory("sumLongSequential", "sumLongSequential"); - BoundDimFilter timeFilter = new BoundDimFilter(Column.TIME_COLUMN_NAME, "200000", "300000", false, false, null, null, + BoundDimFilter timeFilter = new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "200000", "300000", false, false, null, null, StringComparators.NUMERIC); queryAggs.add(new FilteredAggregatorFactory(lsaf, timeFilter)); @@ -199,7 +199,7 @@ private void setupQueries() List queryAggs = new ArrayList<>(); LongSumAggregatorFactory lsaf = new LongSumAggregatorFactory("sumLongSequential", "sumLongSequential"); - BoundDimFilter timeFilter = new BoundDimFilter(Column.TIME_COLUMN_NAME, "200000", "300000", false, false, null, null, + BoundDimFilter timeFilter = new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "200000", "300000", false, false, null, null, StringComparators.ALPHANUMERIC); queryAggs.add(new FilteredAggregatorFactory(lsaf, timeFilter)); diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java index f76788d4984b..1e31a84aea3e 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -2186,7 +2186,7 @@ private List readSegmentColumn(final String column, final SegmentDescrip ); IndexIO indexIO = new TestUtils().getTestIndexIO(); QueryableIndex index = indexIO.loadIndex(outputLocation); - DictionaryEncodedColumn theColumn = index.getColumn(column).getDictionaryEncoding(); + DictionaryEncodedColumn theColumn = (DictionaryEncodedColumn) index.getColumn(column).getColumn(); List values = Lists.newArrayList(); for (int i = 0; i < theColumn.length(); i++) { int id = theColumn.getSingleValueRow(i); diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java index 6eff77b2480d..9f58c1b517f0 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java @@ -61,7 +61,7 @@ import io.druid.segment.DimensionHandler; import io.druid.segment.IndexIO; import io.druid.segment.QueryableIndex; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.segment.indexing.DataSchema; import io.druid.segment.indexing.granularity.ArbitraryGranularitySpec; @@ -396,7 +396,7 @@ private static DimensionsSpec createDimensionsSpec(List dimensionHandlerMap = queryableIndex.getDimensionHandlers(); for (String dimension : queryableIndex.getAvailableDimensions()) { - final Column column = Preconditions.checkNotNull( + final ColumnHolder columnHolder = Preconditions.checkNotNull( queryableIndex.getColumn(dimension), "Cannot find column for dimension[%s]", dimension @@ -413,10 +413,10 @@ private static DimensionsSpec createDimensionsSpec(List(); AGGREGATORS = new HashMap<>(); - DIMENSIONS.put(Column.TIME_COLUMN_NAME, new LongDimensionSchema(Column.TIME_COLUMN_NAME)); + DIMENSIONS.put(ColumnHolder.TIME_COLUMN_NAME, new LongDimensionSchema(ColumnHolder.TIME_COLUMN_NAME)); DIMENSIONS.put(TIMESTAMP_COLUMN, new LongDimensionSchema(TIMESTAMP_COLUMN)); for (int i = 0; i < 5; i++) { final StringDimensionSchema schema = new StringDimensionSchema( @@ -620,10 +620,10 @@ private static class TestIndexIO extends IndexIO for (Entry entry : segmentFileMap.entrySet()) { final DataSegment segment = entry.getKey(); final List columnNames = new ArrayList<>(segment.getDimensions().size() + segment.getMetrics().size()); - columnNames.add(Column.TIME_COLUMN_NAME); + columnNames.add(ColumnHolder.TIME_COLUMN_NAME); columnNames.addAll(segment.getDimensions()); columnNames.addAll(segment.getMetrics()); - final Map columnMap = new HashMap<>(columnNames.size()); + final Map columnMap = new HashMap<>(columnNames.size()); final List aggregatorFactories = new ArrayList<>(segment.getMetrics().size()); for (String columnName : columnNames) { @@ -691,7 +691,7 @@ Map getQueryableIndexMap() } } - private static Column createColumn(DimensionSchema dimensionSchema) + private static ColumnHolder createColumn(DimensionSchema dimensionSchema) { return new ColumnBuilder() .setType(IncrementalIndex.TYPE_MAP.get(dimensionSchema.getValueType())) @@ -700,7 +700,7 @@ private static Column createColumn(DimensionSchema dimensionSchema) .build(); } - private static Column createColumn(AggregatorFactory aggregatorFactory) + private static ColumnHolder createColumn(AggregatorFactory aggregatorFactory) { return new ColumnBuilder() .setType(ValueType.fromString(aggregatorFactory.getTypeName())) diff --git a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java index 89ecf1e6bcbe..8d97add97c51 100644 --- a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java @@ -71,7 +71,7 @@ import io.druid.segment.IndexMergerV9; import io.druid.segment.IndexSpec; import io.druid.segment.TestHelper; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.loading.DataSegmentArchiver; @@ -550,7 +550,7 @@ public void testTransformSpec() throws IOException Assert.assertEquals(MAX_SHARD_NUMBER.longValue(), segmentSet.size()); Integer rowcount = 0; final TransformSpec transformSpec = new TransformSpec( - new SelectorDimFilter(Column.TIME_COLUMN_NAME, "1", null), + new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "1", null), ImmutableList.of( new ExpressionTransform(METRIC_FLOAT_NAME, METRIC_FLOAT_NAME + " * 10", ExprMacroTable.nil()) ) diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index add62c8a645c..992843c2a6f6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -26,7 +26,7 @@ import io.druid.math.expr.Parser; import io.druid.segment.BaseDoubleColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import javax.annotation.Nullable; import java.util.Collections; @@ -55,7 +55,7 @@ public SimpleDoubleAggregatorFactory( this.name = name; this.fieldName = fieldName; this.expression = expression; - this.storeDoubleAsFloat = Column.storeDoubleAsFloat(); + this.storeDoubleAsFloat = ColumnHolder.storeDoubleAsFloat(); Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); Preconditions.checkArgument( fieldName == null ^ expression == null, diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index bc725cc0769a..f124fd886f55 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -35,7 +35,7 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseObjectColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.nio.ByteBuffer; import java.util.Arrays; @@ -71,14 +71,14 @@ public DoubleFirstAggregatorFactory( this.name = name; this.fieldName = fieldName; - this.storeDoubleAsFloat = Column.storeDoubleAsFloat(); + this.storeDoubleAsFloat = ColumnHolder.storeDoubleAsFloat(); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { return new DoubleFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -87,7 +87,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { return new DoubleFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -195,7 +195,7 @@ public String getFieldName() @Override public List requiredFields() { - return Arrays.asList(Column.TIME_COLUMN_NAME, fieldName); + return Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, fieldName); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index fe66d280ca5a..f3ae774f7095 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -35,7 +35,7 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseObjectColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.nio.ByteBuffer; import java.util.Arrays; @@ -76,7 +76,7 @@ public FloatFirstAggregatorFactory( public Aggregator factorize(ColumnSelectorFactory metricFactory) { return new FloatFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -85,7 +85,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { return new FloatFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -193,7 +193,7 @@ public String getFieldName() @Override public List requiredFields() { - return Arrays.asList(Column.TIME_COLUMN_NAME, fieldName); + return Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, fieldName); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index a2780e94e6a4..479182f0430e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -34,7 +34,7 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseObjectColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.nio.ByteBuffer; import java.util.Arrays; @@ -69,7 +69,7 @@ public LongFirstAggregatorFactory( public Aggregator factorize(ColumnSelectorFactory metricFactory) { return new LongFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -78,7 +78,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { return new LongFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -186,7 +186,7 @@ public String getFieldName() @Override public List requiredFields() { - return Arrays.asList(Column.TIME_COLUMN_NAME, fieldName); + return Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, fieldName); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index ed011684f06f..ee048630c9ee 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -35,7 +35,7 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseObjectColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.nio.ByteBuffer; import java.util.Arrays; @@ -61,14 +61,14 @@ public DoubleLastAggregatorFactory( Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); this.name = name; this.fieldName = fieldName; - this.storeDoubleAsFloat = Column.storeDoubleAsFloat(); + this.storeDoubleAsFloat = ColumnHolder.storeDoubleAsFloat(); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { return new DoubleLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -77,7 +77,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { return new DoubleLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -185,7 +185,7 @@ public String getFieldName() @Override public List requiredFields() { - return Arrays.asList(Column.TIME_COLUMN_NAME, fieldName); + return Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, fieldName); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 9dce9c8949ab..f6a8ce3b126b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -35,7 +35,7 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseObjectColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.nio.ByteBuffer; import java.util.Arrays; @@ -66,7 +66,7 @@ public FloatLastAggregatorFactory( public Aggregator factorize(ColumnSelectorFactory metricFactory) { return new FloatLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -75,7 +75,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { return new FloatLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -183,7 +183,7 @@ public String getFieldName() @Override public List requiredFields() { - return Arrays.asList(Column.TIME_COLUMN_NAME, fieldName); + return Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, fieldName); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 4bee9e9a6659..f185583df3c0 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -35,7 +35,7 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseObjectColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.nio.ByteBuffer; import java.util.Arrays; @@ -65,7 +65,7 @@ public LongLastAggregatorFactory( public Aggregator factorize(ColumnSelectorFactory metricFactory) { return new LongLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -74,7 +74,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { return new LongLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), metricFactory.makeColumnValueSelector(fieldName) ); } @@ -182,7 +182,7 @@ public String getFieldName() @Override public List requiredFields() { - return Arrays.asList(Column.TIME_COLUMN_NAME, fieldName); + return Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, fieldName); } @Override diff --git a/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java index 776480068c3e..be5b20496c4a 100644 --- a/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java @@ -19,17 +19,22 @@ package io.druid.query.filter; +import com.google.errorprone.annotations.MustBeClosed; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.spatial.ImmutableRTree; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; + +import javax.annotation.Nullable; /** */ public interface BitmapIndexSelector { - Indexed getDimensionValues(String dimension); + @MustBeClosed + @Nullable + CloseableIndexed getDimensionValues(String dimension); boolean hasMultipleValues(String dimension); int getNumRows(); BitmapFactory getBitmapFactory(); diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index c2855b4d7bc7..dd07506fb956 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -60,7 +60,7 @@ import io.druid.query.spec.QuerySegmentSpec; import io.druid.segment.VirtualColumn; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import org.joda.time.Interval; @@ -676,10 +676,10 @@ private static void verifyOutputNames( } } - if (outputNames.contains(Column.TIME_COLUMN_NAME)) { + if (outputNames.contains(ColumnHolder.TIME_COLUMN_NAME)) { throw new IAE( "'%s' cannot be used as an output name for dimensions, aggregators, or post-aggregators.", - Column.TIME_COLUMN_NAME + ColumnHolder.TIME_COLUMN_NAME ); } } diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java index a66a766fdf39..1472c56f14e5 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java @@ -35,7 +35,7 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.IdLookup; import io.druid.segment.LongColumnSelector; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ValueType; @@ -90,7 +90,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi final String dimension = dimensionSpec.getDimension(); final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); - if (Column.TIME_COLUMN_NAME.equals(dimensionSpec.getDimension())) { + if (ColumnHolder.TIME_COLUMN_NAME.equals(dimensionSpec.getDimension())) { if (extractionFn == null) { throw new UnsupportedOperationException("time dimension must provide an extraction function"); } @@ -298,7 +298,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ColumnValueSelector makeColumnValueSelector(String columnName) { - if (columnName.equals(Column.TIME_COLUMN_NAME)) { + if (columnName.equals(ColumnHolder.TIME_COLUMN_NAME)) { class TimeLongColumnSelector implements LongColumnSelector { @Override @@ -380,7 +380,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ColumnCapabilities getColumnCapabilities(String columnName) { - if (Column.TIME_COLUMN_NAME.equals(columnName)) { + if (ColumnHolder.TIME_COLUMN_NAME.equals(columnName)) { // TIME_COLUMN_NAME is handled specially; override the provided rowSignature. return new ColumnCapabilitiesImpl().setType(ValueType.LONG); } else { diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java index 196bb42cd3b0..ec5bae29cf1f 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java @@ -40,9 +40,9 @@ import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumns; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ComplexColumn; import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; @@ -99,9 +99,9 @@ public Map analyze(Segment segment) Map columns = Maps.newTreeMap(); for (String columnName : columnNames) { - final Column column = index == null ? null : index.getColumn(columnName); - final ColumnCapabilities capabilities = column != null - ? column.getCapabilities() + final ColumnHolder columnHolder = index == null ? null : index.getColumn(columnName); + final ColumnCapabilities capabilities = columnHolder != null + ? columnHolder.getCapabilities() : storageAdapter.getColumnCapabilities(columnName); final ColumnAnalysis analysis; @@ -118,13 +118,13 @@ public Map analyze(Segment segment) break; case STRING: if (index != null) { - analysis = analyzeStringColumn(capabilities, column); + analysis = analyzeStringColumn(capabilities, columnHolder); } else { analysis = analyzeStringColumn(capabilities, storageAdapter, columnName); } break; case COMPLEX: - analysis = analyzeComplexColumn(capabilities, column, storageAdapter.getColumnTypeName(columnName)); + analysis = analyzeComplexColumn(capabilities, columnHolder, storageAdapter.getColumnTypeName(columnName)); break; default: log.warn("Unknown column type[%s].", type); @@ -135,12 +135,12 @@ public Map analyze(Segment segment) } // Add time column too - ColumnCapabilities timeCapabilities = storageAdapter.getColumnCapabilities(Column.TIME_COLUMN_NAME); + ColumnCapabilities timeCapabilities = storageAdapter.getColumnCapabilities(ColumnHolder.TIME_COLUMN_NAME); if (timeCapabilities == null) { timeCapabilities = new ColumnCapabilitiesImpl().setType(ValueType.LONG).setHasMultipleValues(false); } columns.put( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, analyzeNumericColumn(timeCapabilities, length, NUM_BYTES_IN_TIMESTAMP) ); @@ -191,7 +191,7 @@ private ColumnAnalysis analyzeNumericColumn( private ColumnAnalysis analyzeStringColumn( final ColumnCapabilities capabilities, - final Column column + final ColumnHolder columnHolder ) { long size = 0; @@ -203,7 +203,7 @@ private ColumnAnalysis analyzeStringColumn( return ColumnAnalysis.error("string_no_bitmap"); } - final BitmapIndex bitmapIndex = column.getBitmapIndex(); + final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); final int cardinality = bitmapIndex.getCardinality(); if (analyzingSize()) { @@ -310,11 +310,11 @@ public Long accumulate(Long accumulated, Cursor cursor) private ColumnAnalysis analyzeComplexColumn( @Nullable final ColumnCapabilities capabilities, - @Nullable final Column column, + @Nullable final ColumnHolder columnHolder, final String typeName ) { - try (final ComplexColumn complexColumn = column != null ? column.getComplexColumn() : null) { + try (final ComplexColumn complexColumn = columnHolder != null ? (ComplexColumn) columnHolder.getColumn() : null) { final boolean hasMultipleValues = capabilities != null && capabilities.hasMultipleValues(); long size = 0; @@ -329,7 +329,7 @@ private ColumnAnalysis analyzeComplexColumn( return new ColumnAnalysis(typeName, hasMultipleValues, 0, null, null, null, null); } - final int length = column.getLength(); + final int length = columnHolder.getLength(); for (int i = 0; i < length; ++i) { size += inputSizeFn.apply(complexColumn.getRowValue(i)); } diff --git a/processing/src/main/java/io/druid/query/scan/ScanQueryEngine.java b/processing/src/main/java/io/druid/query/scan/ScanQueryEngine.java index 693d2262f66a..1b6b4b2a6f2f 100644 --- a/processing/src/main/java/io/druid/query/scan/ScanQueryEngine.java +++ b/processing/src/main/java/io/druid/query/scan/ScanQueryEngine.java @@ -36,7 +36,7 @@ import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumn; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.filter.Filters; import org.joda.time.Interval; @@ -94,7 +94,7 @@ public Sequence process( } else { final Set availableColumns = Sets.newLinkedHashSet( Iterables.concat( - Collections.singleton(legacy ? LEGACY_TIMESTAMP_KEY : Column.TIME_COLUMN_NAME), + Collections.singleton(legacy ? LEGACY_TIMESTAMP_KEY : ColumnHolder.TIME_COLUMN_NAME), Iterables.transform( Arrays.asList(query.getVirtualColumns().getVirtualColumns()), VirtualColumn::getOutputName @@ -107,7 +107,7 @@ public Sequence process( allColumns.addAll(availableColumns); if (legacy) { - allColumns.remove(Column.TIME_COLUMN_NAME); + allColumns.remove(ColumnHolder.TIME_COLUMN_NAME); } } @@ -145,7 +145,7 @@ public Iterator make() if (legacy && column.equals(LEGACY_TIMESTAMP_KEY)) { selector = cursor.getColumnSelectorFactory() - .makeColumnValueSelector(Column.TIME_COLUMN_NAME); + .makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME); } else { selector = cursor.getColumnSelectorFactory().makeColumnValueSelector(column); } diff --git a/processing/src/main/java/io/druid/query/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/AutoStrategy.java index 665669e5476c..878237ac9bbf 100644 --- a/processing/src/main/java/io/druid/query/search/AutoStrategy.java +++ b/processing/src/main/java/io/druid/query/search/AutoStrategy.java @@ -27,7 +27,7 @@ import io.druid.segment.Segment; import io.druid.segment.VirtualColumns; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.util.List; @@ -114,9 +114,9 @@ private static long computeTotalCard(final QueryableIndex index, final Iterable< { long totalCard = 0; for (DimensionSpec dimension : dimensionSpecs) { - final Column column = index.getColumn(dimension.getDimension()); - if (column != null) { - final BitmapIndex bitmapIndex = column.getBitmapIndex(); + final ColumnHolder columnHolder = index.getColumn(dimension.getDimension()); + if (columnHolder != null) { + final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); if (bitmapIndex != null) { totalCard += bitmapIndex.getCardinality(); } diff --git a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java index 5c3212e51e25..1c43bc46993c 100644 --- a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java @@ -39,7 +39,7 @@ import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumns; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.NumericColumn; import it.unimi.dsi.fastutil.objects.Object2IntRBTreeMap; @@ -165,8 +165,8 @@ static ImmutableBitmap makeTimeFilteredBitmap( final ImmutableBitmap timeFilteredBitmap; if (!interval.contains(segment.getDataInterval())) { final MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); - final Column timeColumn = index.getColumn(Column.TIME_COLUMN_NAME); - try (final NumericColumn timeValues = timeColumn.getNumericColumn()) { + final ColumnHolder timeColumnHolder = index.getColumn(ColumnHolder.TIME_COLUMN_NAME); + try (final NumericColumn timeValues = (NumericColumn) timeColumnHolder.getColumn()) { int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true)); int endIndex = Math.min( @@ -246,12 +246,12 @@ public Object2IntRBTreeMap execute(int limit) final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); for (DimensionSpec dimension : dimsToSearch) { - final Column column = index.getColumn(dimension.getDimension()); - if (column == null) { + final ColumnHolder columnHolder = index.getColumn(dimension.getDimension()); + if (columnHolder == null) { continue; } - final BitmapIndex bitmapIndex = column.getBitmapIndex(); + final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); Preconditions.checkArgument(bitmapIndex != null, "Dimension [%s] should support bitmap index", dimension.getDimension() ); diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java index 029ea5749391..de52ddd06e6f 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java @@ -46,7 +46,7 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; @@ -235,7 +235,7 @@ public Result apply(Cursor cursor) ); final BaseLongColumnValueSelector timestampColumnSelector = - cursor.getColumnSelectorFactory().makeColumnValueSelector(Column.TIME_COLUMN_NAME); + cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME); final List> selectorPlusList = Arrays.asList( DimensionHandlerUtils.createColumnSelectorPluses( diff --git a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java index 51f4a34e5c67..ea8de9f7d669 100644 --- a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java @@ -40,7 +40,7 @@ import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.filter.Filters; import org.joda.time.DateTime; @@ -102,7 +102,7 @@ public Result apply(Cursor cursor) return null; } final BaseLongColumnValueSelector timestampColumnSelector = - cursor.getColumnSelectorFactory().makeColumnValueSelector(Column.TIME_COLUMN_NAME); + cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME); final DateTime timestamp = DateTimes.utc(timestampColumnSelector.getLong()); return new Result<>(adapter.getInterval().getStart(), timestamp); } diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java b/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java index bc573b9dea90..8fbd82b2c6e6 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java @@ -33,7 +33,7 @@ import io.druid.segment.Cursor; import io.druid.segment.SegmentMissingException; import io.druid.segment.StorageAdapter; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; import io.druid.segment.filter.Filters; @@ -131,7 +131,7 @@ private TopNMapFn getMapFn( // TimeExtractionTopNAlgorithm can work on any single-value dimension of type long. // Once we have arbitrary dimension types following check should be replaced by checking // that the column is of type long and single-value. - dimension.equals(Column.TIME_COLUMN_NAME) + dimension.equals(ColumnHolder.TIME_COLUMN_NAME) ) { // A special TimeExtractionTopNAlgorithm is required, since DimExtractionTopNAlgorithm // currently relies on the dimension cardinality to support lexicographic sorting diff --git a/processing/src/main/java/io/druid/segment/AbstractIndex.java b/processing/src/main/java/io/druid/segment/AbstractIndex.java index 474ece9df8cc..55df7144061e 100644 --- a/processing/src/main/java/io/druid/segment/AbstractIndex.java +++ b/processing/src/main/java/io/druid/segment/AbstractIndex.java @@ -21,7 +21,7 @@ import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.util.ArrayList; import java.util.List; @@ -47,7 +47,7 @@ public String toString() null ).toList(); List columnNames = new ArrayList<>(); - columnNames.add(Column.TIME_COLUMN_NAME); + columnNames.add(ColumnHolder.TIME_COLUMN_NAME); columnNames.addAll(getColumnNames()); for (Cursor cursor : cursors) { ColumnSelectorFactory columnSelectorFactory = cursor.getColumnSelectorFactory(); diff --git a/processing/src/main/java/io/druid/segment/ColumnSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelector.java index 570e71cd88d6..47fc0089a775 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelector.java @@ -19,7 +19,7 @@ package io.druid.segment; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import javax.annotation.Nullable; import java.util.List; @@ -31,5 +31,5 @@ public interface ColumnSelector List getColumnNames(); @Nullable - Column getColumn(String columnName); + ColumnHolder getColumn(String columnName); } diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index f85835e043d2..a9dd71717528 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -25,15 +25,18 @@ import io.druid.common.config.NullHandling; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.column.BaseColumn; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.NumericColumn; import io.druid.segment.column.ValueType; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.IndexedIterable; import io.druid.segment.filter.Filters; +import javax.annotation.Nullable; +import java.io.IOException; import java.util.Iterator; /** @@ -55,20 +58,25 @@ public ColumnSelectorBitmapIndexSelector( this.index = index; } + @Nullable @Override - public Indexed getDimensionValues(String dimension) + public CloseableIndexed getDimensionValues(String dimension) { if (isVirtualColumn(dimension)) { // Virtual columns don't have dictionaries or indexes. return null; } - final Column columnDesc = index.getColumn(dimension); - if (columnDesc == null || !columnDesc.getCapabilities().isDictionaryEncoded()) { + final ColumnHolder columnHolder = index.getColumn(dimension); + if (columnHolder == null) { return null; } - final DictionaryEncodedColumn column = columnDesc.getDictionaryEncoding(); - return new Indexed() + BaseColumn col = columnHolder.getColumn(); + if (!(col instanceof DictionaryEncodedColumn)) { + return null; + } + final DictionaryEncodedColumn column = (DictionaryEncodedColumn) col; + return new CloseableIndexed() { @Override public Class getClazz() @@ -105,6 +113,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("column", column); } + + @Override + public void close() throws IOException + { + column.close(); + } }; } @@ -115,14 +129,14 @@ public boolean hasMultipleValues(final String dimension) return virtualColumns.getVirtualColumn(dimension).capabilities(dimension).hasMultipleValues(); } - final Column column = index.getColumn(dimension); - return column != null && column.getCapabilities().hasMultipleValues(); + final ColumnHolder columnHolder = index.getColumn(dimension); + return columnHolder != null && columnHolder.getCapabilities().hasMultipleValues(); } @Override public int getNumRows() { - try (final NumericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getNumericColumn()) { + try (final NumericColumn column = (NumericColumn) index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { return column.length(); } } @@ -141,8 +155,8 @@ public BitmapIndex getBitmapIndex(String dimension) return null; } - final Column column = index.getColumn(dimension); - if (column == null || !columnSupportsFiltering(column)) { + final ColumnHolder columnHolder = index.getColumn(dimension); + if (columnHolder == null || !columnSupportsFiltering(columnHolder)) { // for missing columns and columns with types that do not support filtering, // treat the column as if it were a String column full of nulls. // Create a BitmapIndex so that filters applied to null columns can use @@ -196,8 +210,8 @@ public ImmutableBitmap getBitmap(int idx) } } }; - } else if (column.getCapabilities().hasBitmapIndexes()) { - return column.getBitmapIndex(); + } else if (columnHolder.getCapabilities().hasBitmapIndexes()) { + return columnHolder.getBitmapIndex(); } else { return null; } @@ -211,8 +225,8 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) return null; } - final Column column = index.getColumn(dimension); - if (column == null || !columnSupportsFiltering(column)) { + final ColumnHolder columnHolder = index.getColumn(dimension); + if (columnHolder == null || !columnSupportsFiltering(columnHolder)) { if (NullHandling.isNullOrEquivalent(value)) { return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows()); } else { @@ -220,11 +234,11 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) } } - if (!column.getCapabilities().hasBitmapIndexes()) { + if (!columnHolder.getCapabilities().hasBitmapIndexes()) { return null; } - final BitmapIndex bitmapIndex = column.getBitmapIndex(); + final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); return bitmapIndex.getBitmap(bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(value))); } @@ -235,12 +249,12 @@ public ImmutableRTree getSpatialIndex(String dimension) return ImmutableRTree.empty(); } - final Column column = index.getColumn(dimension); - if (column == null || !column.getCapabilities().hasSpatialIndexes()) { + final ColumnHolder columnHolder = index.getColumn(dimension); + if (columnHolder == null || !columnHolder.getCapabilities().hasSpatialIndexes()) { return ImmutableRTree.empty(); } - return column.getSpatialIndex().getRTree(); + return columnHolder.getSpatialIndex().getRTree(); } private boolean isVirtualColumn(final String columnName) @@ -248,9 +262,9 @@ private boolean isVirtualColumn(final String columnName) return virtualColumns.getVirtualColumn(columnName) != null; } - private static boolean columnSupportsFiltering(Column column) + private static boolean columnSupportsFiltering(ColumnHolder columnHolder) { - ValueType columnType = column.getCapabilities().getType(); + ValueType columnType = columnHolder.getCapabilities().getType(); return Filters.FILTERABLE_TYPES.contains(columnType); } } diff --git a/processing/src/main/java/io/druid/segment/DimensionIndexer.java b/processing/src/main/java/io/druid/segment/DimensionIndexer.java index 0a51bc4b6703..37eccbca9838 100644 --- a/processing/src/main/java/io/druid/segment/DimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DimensionIndexer.java @@ -22,7 +22,7 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; import io.druid.query.dimension.DimensionSpec; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexRowHolder; @@ -160,7 +160,7 @@ public interface DimensionIndexer * * @return Sorted index of actual values */ - Indexed getSortedIndexedValues(); + CloseableIndexed getSortedIndexedValues(); /** diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java index bca270ee373f..80d226820087 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java @@ -25,7 +25,7 @@ import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexRowHolder; @@ -62,7 +62,7 @@ public Double getUnsortedEncodedValueFromSorted(Double sortedIntermediateValue) } @Override - public Indexed getSortedIndexedValues() + public CloseableIndexed getSortedIndexedValues() { throw new UnsupportedOperationException("Numeric columns do not support value dictionaries."); } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java index db3d8c7778c8..d2fa8d5637fd 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java @@ -25,7 +25,7 @@ import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexRowHolder; @@ -63,7 +63,7 @@ public Float getUnsortedEncodedValueFromSorted(Float sortedIntermediateValue) } @Override - public Indexed getSortedIndexedValues() + public CloseableIndexed getSortedIndexedValues() { throw new UnsupportedOperationException("Numeric columns do not support value dictionaries."); } diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 4665a4fb6073..806754ee21ab 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -45,7 +45,7 @@ import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.logger.Logger; import io.druid.java.util.emitter.EmittingLogger; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnConfig; @@ -473,7 +473,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException { MMappedIndex index = legacyHandler.mapDir(inDir); - Map columns = Maps.newHashMap(); + Map columns = Maps.newHashMap(); for (String dimension : index.getAvailableDimensions()) { ColumnBuilder builder = new ColumnBuilder() @@ -540,7 +540,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException } columns.put( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, new ColumnBuilder() .setType(ValueType.LONG) .setNumericColumnSupplier( @@ -632,7 +632,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException } } - Map columns = Maps.newHashMap(); + Map columns = Maps.newHashMap(); for (String columnName : cols) { if (Strings.isNullOrEmpty(columnName)) { @@ -642,7 +642,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException columns.put(columnName, deserializeColumn(mapper, smooshedFiles.mapFile(columnName), smooshedFiles)); } - columns.put(Column.TIME_COLUMN_NAME, deserializeColumn(mapper, smooshedFiles.mapFile("__time"), smooshedFiles)); + columns.put(ColumnHolder.TIME_COLUMN_NAME, deserializeColumn(mapper, smooshedFiles.mapFile("__time"), smooshedFiles)); final QueryableIndex index = new SimpleQueryableIndex( dataInterval, @@ -658,7 +658,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException return index; } - private Column deserializeColumn(ObjectMapper mapper, ByteBuffer byteBuffer, SmooshedFileMapper smooshedFiles) + private ColumnHolder deserializeColumn(ObjectMapper mapper, ByteBuffer byteBuffer, SmooshedFileMapper smooshedFiles) throws IOException { ColumnDescriptor serde = mapper.readValue( diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index 0f7bd0893341..a606798f6988 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -41,7 +41,7 @@ import io.druid.java.util.common.io.smoosh.SmooshedWriter; import io.druid.java.util.common.logger.Logger; import io.druid.query.aggregation.AggregatorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ColumnDescriptor; @@ -436,7 +436,7 @@ private void makeTimeColumn( .setValueType(ValueType.LONG) .addSerde(createLongColumnPartSerde(timeWriter, indexSpec)) .build(); - makeColumn(v9Smoosher, Column.TIME_COLUMN_NAME, serdeficator); + makeColumn(v9Smoosher, ColumnHolder.TIME_COLUMN_NAME, serdeficator); log.info("Completed time column in %,d millis.", System.currentTimeMillis() - startTime); progress.stopSection(section); } diff --git a/processing/src/main/java/io/druid/segment/IndexableAdapter.java b/processing/src/main/java/io/druid/segment/IndexableAdapter.java index 8274ac5f89e3..cdfb2ca0113d 100644 --- a/processing/src/main/java/io/druid/segment/IndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/IndexableAdapter.java @@ -19,9 +19,10 @@ package io.druid.segment; +import com.google.errorprone.annotations.MustBeClosed; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.data.BitmapValues; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; import org.joda.time.Interval; import javax.annotation.Nullable; @@ -40,8 +41,9 @@ public interface IndexableAdapter List getMetricNames(); + @MustBeClosed @Nullable - > Indexed getDimValueLookup(String dimension); + > CloseableIndexed getDimValueLookup(String dimension); TransformableRowIterator getRows(); diff --git a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java index e7be19cda81c..ebc530fc6bde 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java @@ -25,7 +25,7 @@ import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexRowHolder; @@ -63,7 +63,7 @@ public Long getUnsortedEncodedValueFromSorted(Long sortedIntermediateValue) } @Override - public Indexed getSortedIndexedValues() + public CloseableIndexed getSortedIndexedValues() { throw new UnsupportedOperationException("Numeric columns do not support value dictionaries."); } diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java index 01a150d45313..726ffc28a7aa 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java @@ -19,13 +19,12 @@ package io.druid.segment; -import io.druid.java.util.common.ISE; import io.druid.java.util.common.io.Closer; import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.column.BaseColumn; -import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.ValueType; import io.druid.segment.data.ReadableOffset; @@ -79,28 +78,34 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi final String dimension = dimensionSpec.getDimension(); final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); - final Column columnDesc = index.getColumn(dimension); - if (columnDesc == null) { + final ColumnHolder columnHolder = index.getColumn(dimension); + if (columnHolder == null) { return DimensionSelectorUtils.constantSelector(null, extractionFn); } - if (dimension.equals(Column.TIME_COLUMN_NAME)) { + if (dimension.equals(ColumnHolder.TIME_COLUMN_NAME)) { return new SingleScanTimeDimensionSelector(makeColumnValueSelector(dimension), extractionFn, descending); } - ValueType type = columnDesc.getCapabilities().getType(); + ValueType type = columnHolder.getCapabilities().getType(); if (type.isNumeric()) { return type.makeNumericWrappingDimensionSelector(makeColumnValueSelector(dimension), extractionFn); } - @SuppressWarnings("unchecked") - DictionaryEncodedColumn column = (DictionaryEncodedColumn) - columnCache.computeIfAbsent(dimension, d -> closer.register(columnDesc.getDictionaryEncoding())); + BaseColumn column = columnCache.computeIfAbsent(dimension, d -> { + BaseColumn col = columnHolder.getColumn(); + if (col instanceof DictionaryEncodedColumn) { + return closer.register(col); + } else { + return null; + } + }); - if (column == null) { - return DimensionSelectorUtils.constantSelector(null, extractionFn); + if (column instanceof DictionaryEncodedColumn) { + //noinspection unchecked + return ((DictionaryEncodedColumn) column).makeDimensionSelector(offset, extractionFn); } else { - return column.makeDimensionSelector(offset, extractionFn); + return DimensionSelectorUtils.constantSelector(null, extractionFn); } } @@ -111,31 +116,20 @@ public ColumnValueSelector makeColumnValueSelector(String columnName) return virtualColumns.makeColumnValueSelector(columnName, this); } - BaseColumn column = columnCache.get(columnName); - - if (column == null) { - Column holder = index.getColumn(columnName); - + BaseColumn column = columnCache.computeIfAbsent(columnName, name -> { + ColumnHolder holder = index.getColumn(name); if (holder != null) { - final ColumnCapabilities capabilities = holder.getCapabilities(); - - if (capabilities.isDictionaryEncoded()) { - column = holder.getDictionaryEncoding(); - } else if (capabilities.getType() == ValueType.COMPLEX) { - column = holder.getComplexColumn(); - } else if (capabilities.getType().isNumeric()) { - column = holder.getNumericColumn(); - } else { - throw new ISE("Unknown column type: %s", capabilities.getType()); - } - closer.register(column); - columnCache.put(columnName, column); + return closer.register(holder.getColumn()); } else { - return NilColumnValueSelector.instance(); + return null; } - } + }); - return column.makeColumnValueSelector(offset); + if (column != null) { + return column.makeColumnValueSelector(offset); + } else { + return NilColumnValueSelector.instance(); + } } @Override diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java index cffbdfa1c997..a2b81cb12cc1 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java @@ -25,24 +25,24 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.io.Closer; -import io.druid.java.util.common.logger.Logger; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BaseColumn; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ComplexColumn; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.ValueType; import io.druid.segment.data.BitmapValues; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.ImmutableBitmapValues; -import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedIterable; import io.druid.segment.selector.settable.SettableColumnValueSelector; import io.druid.segment.selector.settable.SettableLongColumnValueSelector; import org.joda.time.Interval; import javax.annotation.Nullable; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -55,8 +55,6 @@ */ public class QueryableIndexIndexableAdapter implements IndexableAdapter { - private static final Logger log = new Logger(QueryableIndexIndexableAdapter.class); - private final int numRows; private final QueryableIndex input; private final ImmutableList availableDimensions; @@ -66,25 +64,7 @@ public QueryableIndexIndexableAdapter(QueryableIndex input) { this.input = input; numRows = input.getNumRows(); - - // It appears possible that the dimensions have some columns listed which do not have a DictionaryEncodedColumn - // This breaks current logic, but should be fine going forward. This is a work-around to make things work - // in the current state. This code shouldn't be needed once github tracker issue #55 is finished. - ImmutableList.Builder availableDimensions = ImmutableList.builder(); - for (String dim : input.getAvailableDimensions()) { - final Column col = input.getColumn(dim); - - if (col == null) { - log.warn("Wtf!? column[%s] didn't exist!?!?!?", dim); - } else { - if (col.getDictionaryEncoding() == null) { - log.info("No dictionary on dimension[%s]", dim); - } - availableDimensions.add(dim); - } - } - this.availableDimensions = availableDimensions.build(); - + availableDimensions = ImmutableList.copyOf(input.getAvailableDimensions()); this.metadata = input.getMetadata(); } @@ -116,21 +96,24 @@ public List getMetricNames() @Nullable @Override - public > Indexed getDimValueLookup(String dimension) + public > CloseableIndexed getDimValueLookup(String dimension) { - final Column column = input.getColumn(dimension); + final ColumnHolder columnHolder = input.getColumn(dimension); - if (column == null) { + if (columnHolder == null) { return null; } - final DictionaryEncodedColumn dict = column.getDictionaryEncoding(); + final BaseColumn col = columnHolder.getColumn(); - if (dict == null) { + if (!(col instanceof DictionaryEncodedColumn)) { return null; } - return new Indexed() + @SuppressWarnings("unchecked") + DictionaryEncodedColumn dict = (DictionaryEncodedColumn) col; + + return new CloseableIndexed() { @Override public Class getClazz() @@ -167,6 +150,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("dict", dict); } + + @Override + public void close() throws IOException + { + dict.close(); + } }; } @@ -227,7 +216,7 @@ class RowIteratorImpl implements TransformableRowIterator columnCache ); - offsetTimestampSelector = columnSelectorFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME); + offsetTimestampSelector = columnSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME); final List dimensionHandlers = new ArrayList<>(input.getDimensionHandlers().values()); @@ -370,9 +359,9 @@ void resetToMemoizedOffset() @Override public String getMetricType(String metric) { - final Column column = input.getColumn(metric); + final ColumnHolder columnHolder = input.getColumn(metric); - final ValueType type = column.getCapabilities().getType(); + final ValueType type = columnHolder.getCapabilities().getType(); switch (type) { case FLOAT: return "float"; @@ -381,7 +370,7 @@ public String getMetricType(String metric) case DOUBLE: return "double"; case COMPLEX: { - try (ComplexColumn complexColumn = column.getComplexColumn()) { + try (ComplexColumn complexColumn = (ComplexColumn) columnHolder.getColumn()) { return complexColumn.getTypeName(); } } @@ -399,12 +388,12 @@ public ColumnCapabilities getCapabilities(String column) @Override public BitmapValues getBitmapValues(String dimension, int dictId) { - final Column column = input.getColumn(dimension); - if (column == null) { + final ColumnHolder columnHolder = input.getColumn(dimension); + if (columnHolder == null) { return BitmapValues.EMPTY; } - final BitmapIndex bitmaps = column.getBitmapIndex(); + final BitmapIndex bitmaps = columnHolder.getBitmapIndex(); if (bitmaps == null) { return BitmapValues.EMPTY; } @@ -419,13 +408,13 @@ public BitmapValues getBitmapValues(String dimension, int dictId) @VisibleForTesting BitmapValues getBitmapIndex(String dimension, String value) { - final Column column = input.getColumn(dimension); + final ColumnHolder columnHolder = input.getColumn(dimension); - if (column == null) { + if (columnHolder == null) { return BitmapValues.EMPTY; } - final BitmapIndex bitmaps = column.getBitmapIndex(); + final BitmapIndex bitmaps = columnHolder.getBitmapIndex(); if (bitmaps == null) { return BitmapValues.EMPTY; } diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index e861f12e14bb..491e23b02304 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -37,8 +37,9 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BaseColumn; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.NumericColumn; import io.druid.segment.column.ComplexColumn; import io.druid.segment.data.Indexed; @@ -50,6 +51,8 @@ import org.joda.time.Interval; import javax.annotation.Nullable; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -97,18 +100,16 @@ public Iterable getAvailableMetrics() @Override public int getDimensionCardinality(String dimension) { - if (dimension == null) { - return 0; - } - - Column column = index.getColumn(dimension); - if (column == null) { - return 0; + ColumnHolder columnHolder = index.getColumn(dimension); + try (BaseColumn col = columnHolder.getColumn()) { + if (!(col instanceof DictionaryEncodedColumn)) { + return Integer.MAX_VALUE; + } + return ((DictionaryEncodedColumn) col).getCardinality(); } - if (!column.getCapabilities().isDictionaryEncoded()) { - return Integer.MAX_VALUE; + catch (IOException e) { + throw new UncheckedIOException(e); } - return column.getDictionaryEncoding().getCardinality(); } @Override @@ -120,7 +121,7 @@ public int getNumRows() @Override public DateTime getMinTime() { - try (final NumericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getNumericColumn()) { + try (final NumericColumn column = (NumericColumn) index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { return DateTimes.utc(column.getLongSingleValueRow(0)); } } @@ -128,7 +129,7 @@ public DateTime getMinTime() @Override public DateTime getMaxTime() { - try (final NumericColumn column = index.getColumn(Column.TIME_COLUMN_NAME).getNumericColumn()) { + try (final NumericColumn column = (NumericColumn) index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { return DateTimes.utc(column.getLongSingleValueRow(column.length() - 1)); } } @@ -137,9 +138,9 @@ public DateTime getMaxTime() @Nullable public Comparable getMinValue(String dimension) { - Column column = index.getColumn(dimension); - if (column != null && column.getCapabilities().hasBitmapIndexes()) { - BitmapIndex bitmap = column.getBitmapIndex(); + ColumnHolder columnHolder = index.getColumn(dimension); + if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { + BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(0) : null; } return null; @@ -149,9 +150,9 @@ public Comparable getMinValue(String dimension) @Nullable public Comparable getMaxValue(String dimension) { - Column column = index.getColumn(dimension); - if (column != null && column.getCapabilities().hasBitmapIndexes()) { - BitmapIndex bitmap = column.getBitmapIndex(); + ColumnHolder columnHolder = index.getColumn(dimension); + if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { + BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(bitmap.getCardinality() - 1) : null; } return null; @@ -173,9 +174,16 @@ public ColumnCapabilities getColumnCapabilities(String column) @Override public String getColumnTypeName(String columnName) { - final Column column = index.getColumn(columnName); - try (final ComplexColumn complexColumn = column.getComplexColumn()) { - return complexColumn != null ? complexColumn.getTypeName() : column.getCapabilities().getType().toString(); + final ColumnHolder columnHolder = index.getColumn(columnName); + try (final BaseColumn col = columnHolder.getColumn()) { + if (col instanceof ComplexColumn) { + return ((ComplexColumn) col).getTypeName(); + } else { + return columnHolder.getCapabilities().getType().toString(); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); } } @@ -318,11 +326,11 @@ public Sequence makeCursors( @Nullable static ColumnCapabilities getColumnCapabilities(ColumnSelector index, String columnName) { - Column columnObj = index.getColumn(columnName); - if (columnObj == null) { + ColumnHolder columnHolder = index.getColumn(columnName); + if (columnHolder == null) { return null; } - return columnObj.getCapabilities(); + return columnHolder.getCapabilities(); } private static class CursorSequenceBuilder @@ -371,7 +379,7 @@ public Sequence build() // Column caches shared amongst all cursors in this sequence. final Map columnCache = new HashMap<>(); - final NumericColumn timestamps = index.getColumn(Column.TIME_COLUMN_NAME).getNumericColumn(); + final NumericColumn timestamps = (NumericColumn) index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getColumn(); final Closer closer = Closer.create(); closer.register(timestamps); diff --git a/processing/src/main/java/io/druid/segment/RowFilteringIndexAdapter.java b/processing/src/main/java/io/druid/segment/RowFilteringIndexAdapter.java index e93753a1af4b..f3d636c63f42 100644 --- a/processing/src/main/java/io/druid/segment/RowFilteringIndexAdapter.java +++ b/processing/src/main/java/io/druid/segment/RowFilteringIndexAdapter.java @@ -21,7 +21,7 @@ import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.data.BitmapValues; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; import org.joda.time.Interval; import java.util.List; @@ -65,7 +65,7 @@ public List getMetricNames() } @Override - public > Indexed getDimValueLookup(String dimension) + public > CloseableIndexed getDimValueLookup(String dimension) { return baseAdapter.getDimValueLookup(dimension); } diff --git a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java index 74ef74e06067..bd4543503692 100644 --- a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java @@ -25,7 +25,7 @@ import com.google.common.collect.Maps; import io.druid.collections.bitmap.BitmapFactory; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.data.Indexed; import org.joda.time.Interval; @@ -42,7 +42,7 @@ public class SimpleQueryableIndex extends AbstractIndex implements QueryableInde private final List columnNames; private final Indexed availableDimensions; private final BitmapFactory bitmapFactory; - private final Map columns; + private final Map columns; private final SmooshedFileMapper fileMapper; @Nullable private final Metadata metadata; @@ -51,16 +51,16 @@ public class SimpleQueryableIndex extends AbstractIndex implements QueryableInde public SimpleQueryableIndex( Interval dataInterval, Indexed dimNames, BitmapFactory bitmapFactory, - Map columns, + Map columns, SmooshedFileMapper fileMapper, @Nullable Metadata metadata ) { - Preconditions.checkNotNull(columns.get(Column.TIME_COLUMN_NAME)); + Preconditions.checkNotNull(columns.get(ColumnHolder.TIME_COLUMN_NAME)); this.dataInterval = Preconditions.checkNotNull(dataInterval, "dataInterval"); ImmutableList.Builder columnNamesBuilder = ImmutableList.builder(); for (String column : columns.keySet()) { - if (!Column.TIME_COLUMN_NAME.equals(column)) { + if (!ColumnHolder.TIME_COLUMN_NAME.equals(column)) { columnNamesBuilder.add(column); } } @@ -80,7 +80,7 @@ public SimpleQueryableIndex( List columnNames, Indexed availableDimensions, BitmapFactory bitmapFactory, - Map columns, + Map columns, SmooshedFileMapper fileMapper, @Nullable Metadata metadata, Map dimensionHandlers @@ -105,7 +105,7 @@ public Interval getDataInterval() @Override public int getNumRows() { - return columns.get(Column.TIME_COLUMN_NAME).getLength(); + return columns.get(ColumnHolder.TIME_COLUMN_NAME).getLength(); } @Override @@ -134,13 +134,13 @@ public BitmapFactory getBitmapFactoryForDimensions() @Nullable @Override - public Column getColumn(String columnName) + public ColumnHolder getColumn(String columnName) { return columns.get(columnName); } @VisibleForTesting - public Map getColumns() + public Map getColumns() { return columns; } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 793bf43d6f12..31eca6d61c76 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -35,7 +35,7 @@ import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.ArrayBasedIndexedInts; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.IndexedIterable; import io.druid.segment.filter.BooleanValueMatcher; @@ -324,9 +324,9 @@ public Integer getUnsortedEncodedValueFromSorted(Integer sortedIntermediateValue } @Override - public Indexed getSortedIndexedValues() + public CloseableIndexed getSortedIndexedValues() { - return new Indexed() + return new CloseableIndexed() { @Override public Class getClazz() @@ -364,6 +364,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { // nothing to inspect } + + @Override + public void close() + { + // nothing to close + } }; } diff --git a/processing/src/main/java/io/druid/segment/VirtualColumns.java b/processing/src/main/java/io/druid/segment/VirtualColumns.java index 189f3cd44585..db9235f3e2e2 100644 --- a/processing/src/main/java/io/druid/segment/VirtualColumns.java +++ b/processing/src/main/java/io/druid/segment/VirtualColumns.java @@ -33,7 +33,7 @@ import io.druid.java.util.common.Pair; import io.druid.query.cache.CacheKeyBuilder; import io.druid.query.dimension.DimensionSpec; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.virtual.VirtualizedColumnSelectorFactory; @@ -89,7 +89,7 @@ public static VirtualColumns create(List virtualColumns) throw new IAE("Empty or null virtualColumn name"); } - if (vc.getOutputName().equals(Column.TIME_COLUMN_NAME)) { + if (vc.getOutputName().equals(ColumnHolder.TIME_COLUMN_NAME)) { throw new IAE("virtualColumn name[%s] not allowed", vc.getOutputName()); } diff --git a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java index 9a0b58351a88..bb2b73986ec6 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java @@ -90,11 +90,11 @@ public ColumnBuilder setSpatialIndex(Supplier spatialIndex) return this; } - public Column build() + public ColumnHolder build() { Preconditions.checkState(type != null, "Type must be set."); - return new SimpleColumn( + return new SimpleColumnHolder( new ColumnCapabilitiesImpl() .setType(type) .setDictionaryEncoded(dictionaryEncoded) diff --git a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java index 0793c40f249c..ff485e6d7d93 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java @@ -95,7 +95,7 @@ public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws I } } - public Column read(ByteBuffer buffer, ColumnConfig columnConfig, SmooshedFileMapper smooshedFiles) + public ColumnHolder read(ByteBuffer buffer, ColumnConfig columnConfig, SmooshedFileMapper smooshedFiles) { final ColumnBuilder builder = new ColumnBuilder() .setType(valueType) diff --git a/processing/src/main/java/io/druid/segment/column/Column.java b/processing/src/main/java/io/druid/segment/column/ColumnHolder.java similarity index 91% rename from processing/src/main/java/io/druid/segment/column/Column.java rename to processing/src/main/java/io/druid/segment/column/ColumnHolder.java index 9f6d737c553a..b374f65851fe 100644 --- a/processing/src/main/java/io/druid/segment/column/Column.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnHolder.java @@ -22,9 +22,11 @@ import io.druid.java.util.common.StringUtils; import io.druid.segment.selector.settable.SettableColumnValueSelector; +import javax.annotation.Nullable; + /** */ -public interface Column +public interface ColumnHolder { String TIME_COLUMN_NAME = "__time"; String DOUBLE_STORAGE_TYPE_PROPERTY = "druid.indexing.doubleStorage"; @@ -38,11 +40,10 @@ static boolean storeDoubleAsFloat() ColumnCapabilities getCapabilities(); int getLength(); - DictionaryEncodedColumn getDictionaryEncoding(); - - NumericColumn getNumericColumn(); - ComplexColumn getComplexColumn(); + BaseColumn getColumn(); + @Nullable BitmapIndex getBitmapIndex(); + @Nullable SpatialIndex getSpatialIndex(); /** diff --git a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleColumnHolder.java similarity index 81% rename from processing/src/main/java/io/druid/segment/column/SimpleColumn.java rename to processing/src/main/java/io/druid/segment/column/SimpleColumnHolder.java index c2812295c9ec..3d6afb073ba0 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleColumnHolder.java @@ -22,20 +22,24 @@ import com.google.common.base.Supplier; import io.druid.segment.selector.settable.SettableColumnValueSelector; +import javax.annotation.Nullable; + /** */ -class SimpleColumn implements Column +class SimpleColumnHolder implements ColumnHolder { private final ColumnCapabilities capabilities; private final Supplier columnSupplier; + @Nullable private final Supplier bitmapIndex; + @Nullable private final Supplier spatialIndex; - SimpleColumn( + SimpleColumnHolder( ColumnCapabilities capabilities, Supplier columnSupplier, - Supplier bitmapIndex, - Supplier spatialIndex + @Nullable Supplier bitmapIndex, + @Nullable Supplier spatialIndex ) { this.capabilities = capabilities; @@ -59,29 +63,19 @@ public int getLength() } @Override - public DictionaryEncodedColumn getDictionaryEncoding() - { - return (DictionaryEncodedColumn) columnSupplier.get(); - } - - @Override - public NumericColumn getNumericColumn() - { - return (NumericColumn) columnSupplier.get(); - } - - @Override - public ComplexColumn getComplexColumn() + public BaseColumn getColumn() { - return (ComplexColumn) columnSupplier.get(); + return columnSupplier.get(); } + @Nullable @Override public BitmapIndex getBitmapIndex() { return bitmapIndex == null ? null : bitmapIndex.get(); } + @Nullable @Override public SpatialIndex getSpatialIndex() { diff --git a/processing/src/main/java/io/druid/segment/data/CloseableIndexed.java b/processing/src/main/java/io/druid/segment/data/CloseableIndexed.java new file mode 100644 index 000000000000..fc226c206434 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/data/CloseableIndexed.java @@ -0,0 +1,26 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.data; + +import java.io.Closeable; + +public interface CloseableIndexed extends Indexed, Closeable +{ +} diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 1b97251472f7..4b7bf5f918df 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -76,7 +76,7 @@ * value files are identified as: StringUtils.format("%s_value_%d", columnName, fileNumber) * number of value files == numElements/numberOfElementsPerValueFile */ -public class GenericIndexed implements Indexed, Serializer +public class GenericIndexed implements CloseableIndexed, Serializer { static final byte VERSION_ONE = 0x1; static final byte VERSION_TWO = 0x2; @@ -472,6 +472,12 @@ public Iterator iterator() } } + @Override + public void close() throws IOException + { + // nothing to close + } + /////////////// // VERSION ONE /////////////// diff --git a/processing/src/main/java/io/druid/segment/filter/Filters.java b/processing/src/main/java/io/druid/segment/filter/Filters.java index 415ab5bdc7dc..39aac4bf090f 100644 --- a/processing/src/main/java/io/druid/segment/filter/Filters.java +++ b/processing/src/main/java/io/druid/segment/filter/Filters.java @@ -47,15 +47,18 @@ import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.IntIteratorUtils; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.Indexed; import it.unimi.dsi.fastutil.ints.IntIterable; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntList; import javax.annotation.Nullable; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -276,14 +279,18 @@ public static Iterable matchPredicateNoUnion( Preconditions.checkNotNull(predicate, "predicate"); // Missing dimension -> match all rows if the predicate matches null; match no rows otherwise - final Indexed dimValues = selector.getDimensionValues(dimension); - if (dimValues == null || dimValues.size() == 0) { - return ImmutableList.of(predicate.apply(null) ? allTrue(selector) : allFalse(selector)); - } + try (final CloseableIndexed dimValues = selector.getDimensionValues(dimension)) { + if (dimValues == null || dimValues.size() == 0) { + return ImmutableList.of(predicate.apply(null) ? allTrue(selector) : allFalse(selector)); + } - // Apply predicate to all dimension values and union the matching bitmaps - final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension); - return makePredicateQualifyingBitmapIterable(bitmapIndex, predicate, dimValues); + // Apply predicate to all dimension values and union the matching bitmaps + final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension); + return makePredicateQualifyingBitmapIterable(bitmapIndex, predicate, dimValues); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } /** @@ -308,18 +315,24 @@ public static double estimateSelectivity( Preconditions.checkNotNull(predicate, "predicate"); // Missing dimension -> match all rows if the predicate matches null; match no rows otherwise - final Indexed dimValues = indexSelector.getDimensionValues(dimension); - if (dimValues == null || dimValues.size() == 0) { - return predicate.apply(null) ? 1. : 0.; - } + try (final CloseableIndexed dimValues = indexSelector.getDimensionValues(dimension)) { + if (dimValues == null || dimValues.size() == 0) { + return predicate.apply(null) ? 1. : 0.; + } - // Apply predicate to all dimension values and union the matching bitmaps - final BitmapIndex bitmapIndex = indexSelector.getBitmapIndex(dimension); - return estimateSelectivity( - bitmapIndex, - IntIteratorUtils.toIntList(makePredicateQualifyingIndexIterable(bitmapIndex, predicate, dimValues).iterator()), - indexSelector.getNumRows() - ); + // Apply predicate to all dimension values and union the matching bitmaps + final BitmapIndex bitmapIndex = indexSelector.getBitmapIndex(dimension); + return estimateSelectivity( + bitmapIndex, + IntIteratorUtils.toIntList( + makePredicateQualifyingIndexIterable(bitmapIndex, predicate, dimValues).iterator() + ), + indexSelector.getNumRows() + ); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } /** @@ -440,9 +453,9 @@ static boolean supportsSelectivityEstimation( ) { if (filter.supportsBitmapIndex(indexSelector)) { - final Column column = columnSelector.getColumn(dimension); - if (column != null) { - return !column.getCapabilities().hasMultipleValues(); + final ColumnHolder columnHolder = columnSelector.getColumn(dimension); + if (columnHolder != null) { + return !columnHolder.getCapabilities().hasMultipleValues(); } } return false; diff --git a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java index a5984f31b5c8..3e370f0c9cd1 100644 --- a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java @@ -31,10 +31,13 @@ import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.column.BitmapIndex; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.Indexed; import it.unimi.dsi.fastutil.ints.IntIterable; import it.unimi.dsi.fastutil.ints.IntIterator; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.NoSuchElementException; public class LikeFilter implements Filter @@ -101,14 +104,15 @@ private Iterable getBitmapIterable(final BitmapIndexSelector se } // search for start, end indexes in the bitmaps; then include all matching bitmaps between those points - final Indexed dimValues = selector.getDimensionValues(dimension); + try (final CloseableIndexed dimValues = selector.getDimensionValues(dimension)) { - // Union bitmaps for all matching dimension values in range. - // Use lazy iterator to allow unioning bitmaps one by one and avoid materializing all of them at once. - return Filters.bitmapsFromIndexes( - getDimValueIndexIterableForPrefixMatch(bitmapIndex, dimValues), - bitmapIndex - ); + // Union bitmaps for all matching dimension values in range. + // Use lazy iterator to allow unioning bitmaps one by one and avoid materializing all of them at once. + return Filters.bitmapsFromIndexes(getDimValueIndexIterableForPrefixMatch(bitmapIndex, dimValues), bitmapIndex); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } else { // fallback return Filters.matchPredicateNoUnion( diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 60f87a27969f..e1dc71f79257 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -65,7 +65,7 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ValueType; @@ -319,7 +319,7 @@ protected IncrementalIndex( //__time capabilities ColumnCapabilitiesImpl timeCapabilities = new ColumnCapabilitiesImpl(); timeCapabilities.setType(ValueType.LONG); - columnCapabilities.put(Column.TIME_COLUMN_NAME, timeCapabilities); + columnCapabilities.put(ColumnHolder.TIME_COLUMN_NAME, timeCapabilities); // This should really be more generic List spatialDimensions = dimensionsSpec.getSpatialDimensions(); diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java index 78c1206706c9..e0d7dc5d9497 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java @@ -28,7 +28,7 @@ import io.druid.segment.TransformableRowIterator; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.data.BitmapValues; -import io.druid.segment.data.Indexed; +import io.druid.segment.data.CloseableIndexed; import it.unimi.dsi.fastutil.ints.IntIterator; import org.joda.time.Interval; @@ -143,7 +143,7 @@ public List getMetricNames() @Nullable @Override - public > Indexed getDimValueLookup(String dimension) + public > CloseableIndexed getDimValueLookup(String dimension) { final DimensionAccessor accessor = accessors.get(dimension); if (accessor == null) { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java index 6f5e31bd41ce..5fe2c8e28fc9 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java @@ -28,7 +28,7 @@ import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.SingleScanTimeDimensionSelector; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import javax.annotation.Nullable; @@ -72,7 +72,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi final String dimension = dimensionSpec.getDimension(); final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); - if (dimension.equals(Column.TIME_COLUMN_NAME)) { + if (dimension.equals(ColumnHolder.TIME_COLUMN_NAME)) { return new SingleScanTimeDimensionSelector(makeColumnValueSelector(dimension), extractionFn, descending); } @@ -105,7 +105,7 @@ public ColumnValueSelector makeColumnValueSelector(String columnName) return virtualColumns.makeColumnValueSelector(columnName, this); } - if (columnName.equals(Column.TIME_COLUMN_NAME)) { + if (columnName.equals(ColumnHolder.TIME_COLUMN_NAME)) { return rowHolder; } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 35a375c72e55..a572eddf4f8c 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -36,7 +36,7 @@ import io.druid.segment.Metadata; import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.data.Indexed; import io.druid.segment.data.ListIndexed; @@ -85,7 +85,7 @@ public Iterable getAvailableMetrics() @Override public int getDimensionCardinality(String dimension) { - if (dimension.equals(Column.TIME_COLUMN_NAME)) { + if (dimension.equals(ColumnHolder.TIME_COLUMN_NAME)) { return Integer.MAX_VALUE; } diff --git a/processing/src/main/java/io/druid/segment/transform/ExpressionTransform.java b/processing/src/main/java/io/druid/segment/transform/ExpressionTransform.java index d9a35b5c17e3..b2e8c78bff95 100644 --- a/processing/src/main/java/io/druid/segment/transform/ExpressionTransform.java +++ b/processing/src/main/java/io/druid/segment/transform/ExpressionTransform.java @@ -27,7 +27,7 @@ import io.druid.math.expr.Expr; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.util.Objects; @@ -87,7 +87,7 @@ public Object eval(final Row row) private static Object getValueFromRow(final Row row, final String column) { - if (column.equals(Column.TIME_COLUMN_NAME)) { + if (column.equals(ColumnHolder.TIME_COLUMN_NAME)) { return row.getTimestampFromEpoch(); } else { return row.getRaw(column); diff --git a/processing/src/main/java/io/druid/segment/transform/Transformer.java b/processing/src/main/java/io/druid/segment/transform/Transformer.java index 5f475b56e363..b3291a2f27f9 100644 --- a/processing/src/main/java/io/druid/segment/transform/Transformer.java +++ b/processing/src/main/java/io/druid/segment/transform/Transformer.java @@ -25,7 +25,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.query.filter.ValueMatcher; import io.druid.query.groupby.RowBasedColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import org.joda.time.DateTime; @@ -113,9 +113,9 @@ public List getDimensions() @Override public long getTimestampFromEpoch() { - final RowFunction transform = transforms.get(Column.TIME_COLUMN_NAME); + final RowFunction transform = transforms.get(ColumnHolder.TIME_COLUMN_NAME); if (transform != null) { - return Rows.objectToNumber(Column.TIME_COLUMN_NAME, transform.eval(row)).longValue(); + return Rows.objectToNumber(ColumnHolder.TIME_COLUMN_NAME, transform.eval(row)).longValue(); } else { return row.getTimestampFromEpoch(); } @@ -124,7 +124,7 @@ public long getTimestampFromEpoch() @Override public DateTime getTimestamp() { - final RowFunction transform = transforms.get(Column.TIME_COLUMN_NAME); + final RowFunction transform = transforms.get(ColumnHolder.TIME_COLUMN_NAME); if (transform != null) { return DateTimes.utc(getTimestampFromEpoch()); } else { diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 62631640838b..c038aedbe97f 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -41,7 +41,7 @@ import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.NilColumnValueSelector; import io.druid.segment.NullDimensionSelector; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; @@ -139,11 +139,11 @@ public static ColumnValueSelector makeExprEvalSelector( final String column = Iterables.getOnlyElement(columns); final ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(column); - if (column.equals(Column.TIME_COLUMN_NAME)) { + if (column.equals(ColumnHolder.TIME_COLUMN_NAME)) { // Optimization for expressions that hit the __time column and nothing else. // May be worth applying this optimization to all long columns? return new SingleLongInputCachingExpressionColumnValueSelector( - columnSelectorFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), expression ); } else if (capabilities != null diff --git a/processing/src/test/java/io/druid/query/DoubleStorageTest.java b/processing/src/test/java/io/druid/query/DoubleStorageTest.java index f5b71c2f0b38..3b4f84d7919b 100644 --- a/processing/src/test/java/io/druid/query/DoubleStorageTest.java +++ b/processing/src/test/java/io/druid/query/DoubleStorageTest.java @@ -53,7 +53,7 @@ import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexSegment; import io.druid.segment.TestHelper; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; @@ -306,8 +306,8 @@ public void testSelectValues() private static QueryableIndex buildIndex(String storeDoubleAsFloat) throws IOException { - String oldValue = System.getProperty(Column.DOUBLE_STORAGE_TYPE_PROPERTY); - System.setProperty(Column.DOUBLE_STORAGE_TYPE_PROPERTY, storeDoubleAsFloat); + String oldValue = System.getProperty(ColumnHolder.DOUBLE_STORAGE_TYPE_PROPERTY); + System.setProperty(ColumnHolder.DOUBLE_STORAGE_TYPE_PROPERTY, storeDoubleAsFloat); final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() .withMinTimestamp(DateTimes.of("2011-01-13T00:00:00.000Z").getMillis()) .withDimensionsSpec(ROW_PARSER) @@ -332,9 +332,9 @@ private static QueryableIndex buildIndex(String storeDoubleAsFloat) throws IOExc }); if (oldValue == null) { - System.clearProperty(Column.DOUBLE_STORAGE_TYPE_PROPERTY); + System.clearProperty(ColumnHolder.DOUBLE_STORAGE_TYPE_PROPERTY); } else { - System.setProperty(Column.DOUBLE_STORAGE_TYPE_PROPERTY, oldValue); + System.setProperty(ColumnHolder.DOUBLE_STORAGE_TYPE_PROPERTY, oldValue); } File someTmpFile = File.createTempFile("billy", "yay"); someTmpFile.delete(); diff --git a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java index 8aba7090aba7..b8a313347c22 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -27,7 +27,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; @@ -62,7 +62,7 @@ public void setup() valueSelector = new TestDoubleColumnSelectorImpl(doubleValues); objectSelector = new TestObjectColumnSelector<>(pairs); colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); - EasyMock.expect(colSelectorFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME)).andReturn(timeSelector); + EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); EasyMock.replay(colSelectorFactory); diff --git a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java index ff0f00f5eb0b..9962ac376072 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -27,7 +27,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; @@ -62,7 +62,7 @@ public void setup() valueSelector = new TestFloatColumnSelector(floats); objectSelector = new TestObjectColumnSelector<>(pairs); colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); - EasyMock.expect(colSelectorFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME)).andReturn(timeSelector); + EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); EasyMock.replay(colSelectorFactory); diff --git a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java index 36a8b5cb3f6a..6b28dca02489 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -26,7 +26,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; @@ -61,7 +61,7 @@ public void setup() valueSelector = new TestLongColumnSelector(longValues); objectSelector = new TestObjectColumnSelector<>(pairs); colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); - EasyMock.expect(colSelectorFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME)).andReturn(timeSelector); + EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); EasyMock.replay(colSelectorFactory); diff --git a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java index c5453e18171a..0def959c500e 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -27,7 +27,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; @@ -62,7 +62,7 @@ public void setup() valueSelector = new TestDoubleColumnSelectorImpl(doubles); objectSelector = new TestObjectColumnSelector<>(pairs); colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); - EasyMock.expect(colSelectorFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME)).andReturn(timeSelector); + EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); EasyMock.replay(colSelectorFactory); diff --git a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java index 1938f8387339..484488dd128c 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -27,7 +27,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; @@ -62,7 +62,7 @@ public void setup() valueSelector = new TestFloatColumnSelector(floats); objectSelector = new TestObjectColumnSelector<>(pairs); colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); - EasyMock.expect(colSelectorFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME)).andReturn(timeSelector); + EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); EasyMock.replay(colSelectorFactory); diff --git a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java index 25d9160e15f7..4400842722ce 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java @@ -26,7 +26,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; @@ -61,7 +61,7 @@ public void setup() valueSelector = new TestLongColumnSelector(longValues); objectSelector = new TestObjectColumnSelector<>(pairs); colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); - EasyMock.expect(colSelectorFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME)).andReturn(timeSelector); + EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); EasyMock.replay(colSelectorFactory); diff --git a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java index 1ba026c01b9b..82c69d34ec86 100644 --- a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java +++ b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java @@ -28,7 +28,7 @@ import io.druid.query.extraction.IdentityExtractionFn; import io.druid.query.ordering.StringComparators; import io.druid.query.search.ContainsSearchQuerySpec; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.junit.Assert; import org.junit.Test; @@ -64,7 +64,7 @@ public class GetDimensionRangeSetTest private final DimFilter other3 = new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("a", true), null); private final DimFilter interval1 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -113,7 +113,7 @@ public void testSimpleFilter() Assert.assertNull(other2.getDimensionRangeSet("someOtherDim")); Assert.assertNull(other3.getDimensionRangeSet("dim")); - Assert.assertNull(interval1.getDimensionRangeSet(Column.TIME_COLUMN_NAME)); + Assert.assertNull(interval1.getDimensionRangeSet(ColumnHolder.TIME_COLUMN_NAME)); Assert.assertNull(interval2.getDimensionRangeSet("dim1")); } diff --git a/processing/src/test/java/io/druid/query/filter/IntervalDimFilterTest.java b/processing/src/test/java/io/druid/query/filter/IntervalDimFilterTest.java index f7d409c2514a..21aec3c6cd05 100644 --- a/processing/src/test/java/io/druid/query/filter/IntervalDimFilterTest.java +++ b/processing/src/test/java/io/druid/query/filter/IntervalDimFilterTest.java @@ -27,7 +27,7 @@ import io.druid.guice.annotations.Json; import io.druid.java.util.common.Intervals; import io.druid.query.extraction.RegexDimExtractionFn; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -50,7 +50,7 @@ public void setUp() public void testSerde() throws IOException { DimFilter intervalFilter = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -62,7 +62,7 @@ public void testSerde() throws IOException Assert.assertEquals(intervalFilter, actualFilter); intervalFilter = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -79,7 +79,7 @@ public void testSerde() throws IOException public void testGetCacheKey() { DimFilter intervalFilter1 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -88,7 +88,7 @@ public void testGetCacheKey() ); DimFilter intervalFilter2 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1976-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -99,7 +99,7 @@ public void testGetCacheKey() RegexDimExtractionFn regexFn = new RegexDimExtractionFn(".*", false, null); DimFilter intervalFilter3 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -107,7 +107,7 @@ public void testGetCacheKey() regexFn ); DimFilter intervalFilter4 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1976-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -123,7 +123,7 @@ public void testHashCode() RegexDimExtractionFn regexFn = new RegexDimExtractionFn(".*", false, null); DimFilter intervalFilter1 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -132,7 +132,7 @@ public void testHashCode() ); DimFilter intervalFilter2 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -141,7 +141,7 @@ public void testHashCode() ); DimFilter intervalFilter3 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1977-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -153,7 +153,7 @@ public void testHashCode() Assert.assertNotEquals(intervalFilter1.hashCode(), intervalFilter3.hashCode()); DimFilter intervalFilter4 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1977-01-01T00:00:00.004Z"), @@ -181,7 +181,7 @@ public void testEquals() RegexDimExtractionFn regexFn = new RegexDimExtractionFn(".*", false, null); DimFilter intervalFilter1 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -190,7 +190,7 @@ public void testEquals() ); DimFilter intervalFilter2 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -199,7 +199,7 @@ public void testEquals() ); DimFilter intervalFilter3 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1977-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") @@ -211,7 +211,7 @@ public void testEquals() Assert.assertNotEquals(intervalFilter1, intervalFilter3); DimFilter intervalFilter4 = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1977-01-01T00:00:00.004Z"), @@ -236,13 +236,13 @@ public void testEquals() public void testGetRequiredColumns() { DimFilter intervalFilter = new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.004Z"), Intervals.of("1975-01-01T00:00:00.001Z/1980-01-01T00:00:00.004Z") ), null ); - Assert.assertEquals(intervalFilter.getRequiredColumns(), Sets.newHashSet(Column.TIME_COLUMN_NAME)); + Assert.assertEquals(intervalFilter.getRequiredColumns(), Sets.newHashSet(ColumnHolder.TIME_COLUMN_NAME)); } } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java index ba04778d2d8a..11359ad90f4e 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java @@ -80,7 +80,7 @@ import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexSegment; import io.druid.segment.Segment; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnConfig; import io.druid.segment.column.ValueType; import io.druid.segment.incremental.IncrementalIndex; @@ -621,7 +621,7 @@ public Sequence run(QueryPlus queryPlus, Map responseC .setDimensions(Lists.newArrayList( new DefaultDimensionSpec("dimA", "dimA"), new ExtractionDimensionSpec( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, "hour", ValueType.LONG, new TimeFormatExtractionFn( diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index e35f8b7f2bb3..8e45d0f497c1 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -123,7 +123,7 @@ import io.druid.query.search.ContainsSearchQuerySpec; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.segment.TestHelper; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; import org.joda.time.DateTime; @@ -4789,8 +4789,8 @@ public void testGroupByTimeExtractionNamedUnderUnderTime() Lists.newArrayList( new DefaultDimensionSpec("market", "market"), new ExtractionDimensionSpec( - Column.TIME_COLUMN_NAME, - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, new TimeFormatExtractionFn("EEEE", null, null, null, false) ) ) @@ -5630,8 +5630,8 @@ public void testSubqueryWithOuterTimeFilter() .setGranularity(QueryRunnerTestHelper.dayGran) .build(); - final DimFilter fridayFilter = new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Friday", new TimeFormatExtractionFn("EEEE", null, null, null, false)); - final DimFilter firstDaysFilter = new InDimFilter(Column.TIME_COLUMN_NAME, ImmutableList.of("1", "2", "3"), new TimeFormatExtractionFn("d", null, null, null, false)); + final DimFilter fridayFilter = new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Friday", new TimeFormatExtractionFn("EEEE", null, null, null, false)); + final DimFilter firstDaysFilter = new InDimFilter(ColumnHolder.TIME_COLUMN_NAME, ImmutableList.of("1", "2", "3"), new TimeFormatExtractionFn("d", null, null, null, false)); final GroupByQuery query = GroupByQuery .builder() .setDataSource(subquery) @@ -6223,7 +6223,7 @@ public void testGroupByTimeExtraction() Lists.newArrayList( new DefaultDimensionSpec("market", "market"), new ExtractionDimensionSpec( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, "dayOfWeek", new TimeFormatExtractionFn("EEEE", null, null, null, false) ) @@ -6479,7 +6479,7 @@ public ExtractionType getExtractionType() Lists.newArrayList( new DefaultDimensionSpec("market", "market"), new ExtractionDimensionSpec( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, "dayOfWeek", new CascadeExtractionFn( new ExtractionFn[]{ @@ -8112,7 +8112,7 @@ public void testGroupByNumericStringsAsNumeric() Lists.newArrayList( new DefaultDimensionSpec("qualityLong", "ql_alias"), new DefaultDimensionSpec("qualityFloat", "qf_alias"), - new DefaultDimensionSpec(Column.TIME_COLUMN_NAME, "time_alias") + new DefaultDimensionSpec(ColumnHolder.TIME_COLUMN_NAME, "time_alias") ) ) .setDimFilter(new SelectorDimFilter("quality", "entertainment", null)) @@ -8672,7 +8672,7 @@ public void testGroupByNestedDoubleTimeExtractionFnWithLongOutputTypes() Lists.newArrayList( new DefaultDimensionSpec("quality", "alias"), new ExtractionDimensionSpec( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, "time_day", ValueType.LONG, new TimeFormatExtractionFn(null, null, null, Granularities.DAY, true) diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java index 17a21612604c..7948efe26126 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTestHelper.java @@ -32,7 +32,7 @@ import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.joda.time.DateTime; import org.joda.time.chrono.ISOChronology; @@ -76,7 +76,7 @@ public static Row createExpectedRow(final DateTime timestamp, Object... vals) public static List createExpectedRows(String[] columnNames, Object[]... values) { - int timeIndex = Arrays.asList(columnNames).indexOf(Column.TIME_COLUMN_NAME); + int timeIndex = Arrays.asList(columnNames).indexOf(ColumnHolder.TIME_COLUMN_NAME); Preconditions.checkArgument(timeIndex >= 0); List expected = Lists.newArrayList(); diff --git a/processing/src/test/java/io/druid/query/scan/ScanQueryRunnerTest.java b/processing/src/test/java/io/druid/query/scan/ScanQueryRunnerTest.java index ac742f5015e2..29809fbbde76 100644 --- a/processing/src/test/java/io/druid/query/scan/ScanQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/scan/ScanQueryRunnerTest.java @@ -46,7 +46,7 @@ import io.druid.query.spec.LegacySegmentSpec; import io.druid.query.spec.QuerySegmentSpec; import io.druid.segment.VirtualColumn; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; import org.joda.time.DateTime; @@ -235,7 +235,7 @@ public void testSelectWithUnderscoreUnderscoreTime() { ScanQuery query = newTestQuery() .intervals(I_0112_0114) - .columns(Column.TIME_COLUMN_NAME, QueryRunnerTestHelper.marketDimension, QueryRunnerTestHelper.indexMetric) + .columns(ColumnHolder.TIME_COLUMN_NAME, QueryRunnerTestHelper.marketDimension, QueryRunnerTestHelper.indexMetric) .build(); HashMap context = new HashMap(); @@ -629,7 +629,7 @@ private Object toTimestamp(final String value) private String getTimestampName() { - return legacy ? "timestamp" : Column.TIME_COLUMN_NAME; + return legacy ? "timestamp" : ColumnHolder.TIME_COLUMN_NAME; } private List toExpected( diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index f5da46c5fe78..f9dc6c662599 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -54,7 +54,7 @@ import io.druid.segment.QueryableIndexSegment; import io.druid.segment.TestHelper; import io.druid.segment.TestIndex; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; @@ -595,7 +595,7 @@ public void testSearchOnTime() .intervals(QueryRunnerTestHelper.fullOnInterval) .query("Friday") .dimensions(new ExtractionDimensionSpec( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, "__time2", new TimeFormatExtractionFn( "EEEE", @@ -619,7 +619,7 @@ public void testSearchOnLongColumn() SearchQuery searchQuery = Druids.newSearchQueryBuilder() .dimensions( new DefaultDimensionSpec( - Column.TIME_COLUMN_NAME, Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, ColumnHolder.TIME_COLUMN_NAME, ValueType.LONG ) ) @@ -630,7 +630,7 @@ public void testSearchOnLongColumn() .build(); List expectedHits = Lists.newLinkedList(); - expectedHits.add(new SearchHit(Column.TIME_COLUMN_NAME, "1297123200000", 13)); + expectedHits.add(new SearchHit(ColumnHolder.TIME_COLUMN_NAME, "1297123200000", 13)); checkSearchQuery(searchQuery, expectedHits); } @@ -643,7 +643,7 @@ public void testSearchOnLongColumnWithExFn() SearchQuery searchQuery = Druids.newSearchQueryBuilder() .dimensions( new ExtractionDimensionSpec( - Column.TIME_COLUMN_NAME, Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, ColumnHolder.TIME_COLUMN_NAME, jsExtractionFn ) ) @@ -654,7 +654,7 @@ public void testSearchOnLongColumnWithExFn() .build(); List expectedHits = Lists.newLinkedList(); - expectedHits.add(new SearchHit(Column.TIME_COLUMN_NAME, "super-1297123200000", 13)); + expectedHits.add(new SearchHit(ColumnHolder.TIME_COLUMN_NAME, "super-1297123200000", 13)); checkSearchQuery(searchQuery, expectedHits); } diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index 3a95457a1721..dd578ac8fb02 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -54,7 +54,7 @@ import io.druid.query.ordering.StringComparators; import io.druid.query.spec.LegacySegmentSpec; import io.druid.query.spec.QuerySegmentSpec; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; import org.joda.time.DateTime; @@ -684,12 +684,12 @@ public void testFullOnSelectWithLongAndFloat() { List dimSpecs = Arrays.asList( new DefaultDimensionSpec(QueryRunnerTestHelper.indexMetric, "floatIndex", ValueType.FLOAT), - new DefaultDimensionSpec(Column.TIME_COLUMN_NAME, "longTime", ValueType.LONG) + new DefaultDimensionSpec(ColumnHolder.TIME_COLUMN_NAME, "longTime", ValueType.LONG) ); SelectQuery query = newTestQuery() .dimensionSpecs(dimSpecs) - .metrics(Arrays.asList(Column.TIME_COLUMN_NAME, "index")) + .metrics(Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, "index")) .intervals(I_0112_0114) .build(); @@ -712,7 +712,7 @@ public void testFullOnSelectWithLongAndFloat() .put("longTime", 1294790400000L) .put("floatIndex", 100.0f) .put(QueryRunnerTestHelper.indexMetric, 100.000000F) - .put(Column.TIME_COLUMN_NAME, 1294790400000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) .build() ), new EventHolder( @@ -723,7 +723,7 @@ public void testFullOnSelectWithLongAndFloat() .put("longTime", 1294790400000L) .put("floatIndex", 100.0f) .put(QueryRunnerTestHelper.indexMetric, 100.000000F) - .put(Column.TIME_COLUMN_NAME, 1294790400000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) .build() ), new EventHolder( @@ -734,7 +734,7 @@ public void testFullOnSelectWithLongAndFloat() .put("longTime", 1294790400000L) .put("floatIndex", 100.0f) .put(QueryRunnerTestHelper.indexMetric, 100.000000F) - .put(Column.TIME_COLUMN_NAME, 1294790400000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) .build() ) ) @@ -758,7 +758,7 @@ public void testFullOnSelectWithLongAndFloat() .put("longTime", 1294876800000L) .put("floatIndex", 1564.6177f) .put(QueryRunnerTestHelper.indexMetric, 1564.6177f) - .put(Column.TIME_COLUMN_NAME, 1294876800000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) .build() ), new EventHolder( @@ -769,7 +769,7 @@ public void testFullOnSelectWithLongAndFloat() .put("longTime", 1294876800000L) .put("floatIndex", 826.0602f) .put(QueryRunnerTestHelper.indexMetric, 826.0602f) - .put(Column.TIME_COLUMN_NAME, 1294876800000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) .build() ), new EventHolder( @@ -780,7 +780,7 @@ public void testFullOnSelectWithLongAndFloat() .put("longTime", 1294876800000L) .put("floatIndex", 1689.0128f) .put(QueryRunnerTestHelper.indexMetric, 1689.0128f) - .put(Column.TIME_COLUMN_NAME, 1294876800000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) .build() ) ) @@ -799,12 +799,12 @@ public void testFullOnSelectWithLongAndFloatWithExFn() List dimSpecs = Arrays.asList( new ExtractionDimensionSpec(QueryRunnerTestHelper.indexMetric, "floatIndex", jsExtractionFn), - new ExtractionDimensionSpec(Column.TIME_COLUMN_NAME, "longTime", jsExtractionFn) + new ExtractionDimensionSpec(ColumnHolder.TIME_COLUMN_NAME, "longTime", jsExtractionFn) ); SelectQuery query = newTestQuery() .dimensionSpecs(dimSpecs) - .metrics(Arrays.asList(Column.TIME_COLUMN_NAME, "index")) + .metrics(Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, "index")) .intervals(I_0112_0114) .build(); @@ -827,7 +827,7 @@ public void testFullOnSelectWithLongAndFloatWithExFn() .put("longTime", "super-1294790400000") .put("floatIndex", "super-100") .put(QueryRunnerTestHelper.indexMetric, 100.000000F) - .put(Column.TIME_COLUMN_NAME, 1294790400000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) .build() ), new EventHolder( @@ -838,7 +838,7 @@ public void testFullOnSelectWithLongAndFloatWithExFn() .put("longTime", "super-1294790400000") .put("floatIndex", "super-100") .put(QueryRunnerTestHelper.indexMetric, 100.000000F) - .put(Column.TIME_COLUMN_NAME, 1294790400000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) .build() ), new EventHolder( @@ -849,7 +849,7 @@ public void testFullOnSelectWithLongAndFloatWithExFn() .put("longTime", "super-1294790400000") .put("floatIndex", "super-100") .put(QueryRunnerTestHelper.indexMetric, 100.000000F) - .put(Column.TIME_COLUMN_NAME, 1294790400000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) .build() ) ) @@ -873,7 +873,7 @@ public void testFullOnSelectWithLongAndFloatWithExFn() .put("longTime", "super-1294876800000") .put("floatIndex", "super-1564.617729") .put(QueryRunnerTestHelper.indexMetric, 1564.6177f) - .put(Column.TIME_COLUMN_NAME, 1294876800000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) .build() ), new EventHolder( @@ -884,7 +884,7 @@ public void testFullOnSelectWithLongAndFloatWithExFn() .put("longTime", "super-1294876800000") .put("floatIndex", "super-826.060182") .put(QueryRunnerTestHelper.indexMetric, 826.0602f) - .put(Column.TIME_COLUMN_NAME, 1294876800000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) .build() ), new EventHolder( @@ -895,7 +895,7 @@ public void testFullOnSelectWithLongAndFloatWithExFn() .put("longTime", "super-1294876800000") .put("floatIndex", "super-1689.012875") .put(QueryRunnerTestHelper.indexMetric, 1689.0128f) - .put(Column.TIME_COLUMN_NAME, 1294876800000L) + .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) .build() ) ) diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 3bf304eb4f5a..516774a101aa 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -87,7 +87,7 @@ import io.druid.query.ordering.StringComparators; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.segment.TestHelper; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; import org.junit.Assert; @@ -3838,7 +3838,7 @@ public void testTopNTimeExtraction() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, "dayOfWeek", new TimeFormatExtractionFn("EEEE", null, null, null, false) ) @@ -5040,7 +5040,7 @@ public void testFullOnTopNLongTimeColumn() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(new DefaultDimensionSpec(Column.TIME_COLUMN_NAME, "time_alias", ValueType.LONG)) + .dimension(new DefaultDimensionSpec(ColumnHolder.TIME_COLUMN_NAME, "time_alias", ValueType.LONG)) .metric("maxIndex") .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -5115,7 +5115,7 @@ public void testFullOnTopNLongTimeColumnWithExFn() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(new ExtractionDimensionSpec(Column.TIME_COLUMN_NAME, "time_alias", jsExtractionFn)) + .dimension(new ExtractionDimensionSpec(ColumnHolder.TIME_COLUMN_NAME, "time_alias", jsExtractionFn)) .metric("maxIndex") .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -5590,7 +5590,7 @@ public void testTopNWithNonBitmapFilter() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(new BoundDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, "0", String.valueOf(Long.MAX_VALUE), true, diff --git a/processing/src/test/java/io/druid/segment/EmptyIndexTest.java b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java index 26f3aeee4087..761b18a4614c 100644 --- a/processing/src/test/java/io/druid/segment/EmptyIndexTest.java +++ b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java @@ -25,7 +25,7 @@ import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.java.util.common.Intervals; import io.druid.query.aggregation.AggregatorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexAdapter; import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; @@ -98,7 +98,7 @@ public void testEmptyIndex() throws Exception Assert.assertEquals( "getReadOnlyTimestamps", 0, - emptyQueryableIndex.getColumn(Column.TIME_COLUMN_NAME).getLength() + emptyQueryableIndex.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength() ); } finally { diff --git a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java index a7d5ec9f957c..f3d67e494113 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java @@ -30,7 +30,7 @@ import io.druid.java.util.common.guava.Comparators; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.data.IncrementalIndexTest; import io.druid.segment.incremental.IncrementalIndex; @@ -117,13 +117,13 @@ public void testStringColumnNullHandling() throws Exception final File tempDir = temporaryFolder.newFolder(); try (QueryableIndex index = indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))) { - final Column column = index.getColumn("d"); + final ColumnHolder columnHolder = index.getColumn("d"); if (subsetList.stream().allMatch(nullFlavors::contains)) { // all null -> should be missing - Assert.assertNull(subsetList.toString(), column); + Assert.assertNull(subsetList.toString(), columnHolder); } else { - Assert.assertNotNull(subsetList.toString(), column); + Assert.assertNotNull(subsetList.toString(), columnHolder); // The column has multiple values if there are any lists with > 1 element in the input set. final boolean hasMultipleValues = subsetList.stream() @@ -141,7 +141,8 @@ public void testStringColumnNullHandling() throws Exception } } - try (final DictionaryEncodedColumn dictionaryColumn = column.getDictionaryEncoding()) { + try (final DictionaryEncodedColumn dictionaryColumn = + (DictionaryEncodedColumn) columnHolder.getColumn()) { // Verify unique values against the dictionary. Assert.assertEquals( subsetList.toString(), @@ -174,7 +175,7 @@ public void testStringColumnNullHandling() throws Exception ); // Verify that the bitmap index for null is correct. - final BitmapIndex bitmapIndex = column.getBitmapIndex(); + final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); // Read through the column to find all the rows that should match null. final List expectedNullRows = new ArrayList<>(); diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index 7fdbe5b9bd9d..d907d433ff2d 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -43,7 +43,7 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.StringDictionaryEncodedColumn; @@ -168,7 +168,7 @@ public void testPersist() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); - Assert.assertEquals(2, index.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -209,7 +209,7 @@ public void testPersistWithDifferentDims() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); - Assert.assertEquals(2, index.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); assertDimCompression(index, indexSpec.getDimensionCompression()); @@ -245,7 +245,7 @@ public void testPersistWithSegmentMetadata() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); - Assert.assertEquals(2, index.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -299,7 +299,7 @@ public void testPersistMerge() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist1, tempDir1, indexSpec, null)) ); - Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(3, index1.getColumnNames().size()); @@ -307,7 +307,7 @@ public void testPersistMerge() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist2, tempDir2, indexSpec, null)) ); - Assert.assertEquals(2, index2.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index2.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index2.getAvailableDimensions())); Assert.assertEquals(3, index2.getColumnNames().size()); @@ -327,7 +327,7 @@ public void testPersistMerge() throws Exception ) ); - Assert.assertEquals(3, merged.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(3, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); assertDimCompression(index2, indexSpec.getDimensionCompression()); @@ -392,13 +392,13 @@ public void testPersistEmptyColumn() throws Exception ) ); - Assert.assertEquals(1, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(1, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(index1.getAvailableDimensions())); - Assert.assertEquals(1, index2.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(1, index2.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(index2.getAvailableDimensions())); - Assert.assertEquals(2, merged.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(merged.getAvailableDimensions())); assertDimCompression(index1, indexSpec.getDimensionCompression()); @@ -431,7 +431,7 @@ public void testMergeRetainsValues() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(3, index1.getColumnNames().size()); @@ -449,7 +449,7 @@ public void testMergeRetainsValues() throws Exception ) ); - Assert.assertEquals(2, merged.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); @@ -482,7 +482,7 @@ public void testAppendRetainsValues() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(3, index1.getColumnNames().size()); @@ -505,7 +505,7 @@ public void testAppendRetainsValues() throws Exception ) ); - Assert.assertEquals(2, merged.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); @@ -545,7 +545,7 @@ public void testMergeSpecChange() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(3, index1.getColumnNames().size()); @@ -576,7 +576,7 @@ public void testMergeSpecChange() throws Exception ) ); - Assert.assertEquals(2, merged.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); @@ -619,7 +619,7 @@ public void testConvertSame() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(4, index1.getColumnNames().size()); @@ -628,7 +628,7 @@ public void testConvertSame() throws Exception indexIO.loadIndex(indexMerger.convert(tempDir1, convertDir, indexSpec)) ); - Assert.assertEquals(2, converted.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, converted.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(converted.getAvailableDimensions())); Assert.assertEquals(4, converted.getColumnNames().size()); @@ -677,7 +677,7 @@ public void testConvertDifferent() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(4, index1.getColumnNames().size()); @@ -699,7 +699,7 @@ public void testConvertDifferent() throws Exception indexIO.loadIndex(indexMerger.convert(tempDir1, convertDir, newSpec)) ); - Assert.assertEquals(2, converted.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, converted.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(converted.getAvailableDimensions())); Assert.assertEquals(4, converted.getColumnNames().size()); @@ -722,7 +722,7 @@ private void assertDimCompression(QueryableIndex index, CompressionStrategy expe return; } - DictionaryEncodedColumn encodedColumn = index.getColumn("dim2").getDictionaryEncoding(); + DictionaryEncodedColumn encodedColumn = (DictionaryEncodedColumn) index.getColumn("dim2").getColumn(); Object obj; if (encodedColumn.hasMultipleValues()) { Field field = StringDictionaryEncodedColumn.class.getDeclaredField("multiValueColumn"); @@ -2015,7 +2015,7 @@ public void testMultiValueHandling() throws Exception adapter = new QueryableIndexIndexableAdapter(index); rowList = RowIteratorHelper.toList(adapter.getRows()); - Assert.assertEquals(2, index.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -2047,7 +2047,7 @@ public void testMultiValueHandling() throws Exception schema = makeDimensionSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.SORTED_SET); index = persistAndLoad(schema, rows); - Assert.assertEquals(1, index.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(1, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -2078,7 +2078,7 @@ public void testMultiValueHandling() throws Exception schema = makeDimensionSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.ARRAY); index = persistAndLoad(schema, rows); - Assert.assertEquals(2, index.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -2135,7 +2135,7 @@ public void testDimensionWithEmptyName() throws Exception ) ); - Assert.assertEquals(3, index.getColumn(Column.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(3, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals( Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()) diff --git a/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java index 53138e47dcea..6f6c8f57488e 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java @@ -34,12 +34,13 @@ import io.druid.query.filter.ExtractionDimFilter; import io.druid.query.filter.Filter; import io.druid.query.filter.SelectorDimFilter; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BitmapIndex; import io.druid.segment.data.ArrayIndexed; import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.Indexed; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.serde.BitmapIndexColumnPartSupplier; import org.junit.Assert; @@ -47,7 +48,10 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import javax.annotation.Nullable; +import java.io.IOException; import java.util.Arrays; +import java.util.Iterator; import java.util.Map; /** @@ -91,10 +95,59 @@ public ExtractionDimFilterTest(BitmapFactory bitmapFactory, BitmapSerdeFactory b private final BitmapIndexSelector BITMAP_INDEX_SELECTOR = new BitmapIndexSelector() { @Override - public Indexed getDimensionValues(String dimension) + public CloseableIndexed getDimensionValues(String dimension) { final String[] vals = DIM_VALS.get(dimension); - return vals == null ? null : new ArrayIndexed(vals, String.class); + if (vals == null) { + return null; + } else { + ArrayIndexed indexed = new ArrayIndexed<>(vals, String.class); + return new CloseableIndexed() + { + @Override + public Class getClazz() + { + return indexed.getClazz(); + } + + @Override + public int size() + { + return indexed.size(); + } + + @Nullable + @Override + public String get(int index) + { + return indexed.get(index); + } + + @Override + public int indexOf(@Nullable String value) + { + return indexed.indexOf(value); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("indexed", indexed); + } + + @Override + public void close() throws IOException + { + // close nothing + } + + @Override + public Iterator iterator() + { + return indexed.iterator(); + } + }; + } } @Override diff --git a/processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java b/processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java index bd6670718fce..f876bd2c0a9e 100644 --- a/processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java +++ b/processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java @@ -49,7 +49,7 @@ import io.druid.query.search.ContainsSearchQuerySpec; import io.druid.segment.IndexBuilder; import io.druid.segment.StorageAdapter; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import org.junit.AfterClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -108,25 +108,25 @@ public static void tearDown() throws Exception public void testTimeFilterAsLong() { assertFilterMatches( - new SelectorDimFilter(Column.TIME_COLUMN_NAME, "0", null), + new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "0", null), ImmutableList.of("0") ); assertFilterMatches( - new SelectorDimFilter(Column.TIME_COLUMN_NAME, "9000", null), + new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "9000", null), ImmutableList.of() ); assertFilterMatches( - new BoundDimFilter(Column.TIME_COLUMN_NAME, "0", "4", false, false, null, null, StringComparators.NUMERIC), + new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "0", "4", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("0", "1", "2", "3", "4") ); assertFilterMatches( - new BoundDimFilter(Column.TIME_COLUMN_NAME, "0", "4", true, true, null, null, StringComparators.NUMERIC), + new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "0", "4", true, true, null, null, StringComparators.NUMERIC), ImmutableList.of("1", "2", "3") ); assertFilterMatches( - new InDimFilter(Column.TIME_COLUMN_NAME, Arrays.asList("2", "4", "8"), null), + new InDimFilter(ColumnHolder.TIME_COLUMN_NAME, Arrays.asList("2", "4", "8"), null), ImmutableList.of("2", "4") ); @@ -136,23 +136,23 @@ public void testTimeFilterAsLong() infilterValues.add(String.valueOf(i * 2)); } assertFilterMatches( - new InDimFilter(Column.TIME_COLUMN_NAME, infilterValues, null), + new InDimFilter(ColumnHolder.TIME_COLUMN_NAME, infilterValues, null), ImmutableList.of("0", "2", "4") ); String jsFn = "function(x) { return(x === 3 || x === 5) }"; assertFilterMatches( - new JavaScriptDimFilter(Column.TIME_COLUMN_NAME, jsFn, null, JavaScriptConfig.getEnabledInstance()), + new JavaScriptDimFilter(ColumnHolder.TIME_COLUMN_NAME, jsFn, null, JavaScriptConfig.getEnabledInstance()), ImmutableList.of("3", "5") ); assertFilterMatches( - new RegexDimFilter(Column.TIME_COLUMN_NAME, "4", null), + new RegexDimFilter(ColumnHolder.TIME_COLUMN_NAME, "4", null), ImmutableList.of("4") ); assertFilterMatches( - new SearchQueryDimFilter(Column.TIME_COLUMN_NAME, new ContainsSearchQuerySpec("2", true), null), + new SearchQueryDimFilter(ColumnHolder.TIME_COLUMN_NAME, new ContainsSearchQuerySpec("2", true), null), ImmutableList.of("2") ); } @@ -171,25 +171,25 @@ public void testTimeFilterWithExtractionFn() LookupExtractionFn exfn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true); assertFilterMatches( - new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Monday", exfn), + new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Monday", exfn), ImmutableList.of("0") ); assertFilterMatches( - new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Notaday", exfn), + new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Notaday", exfn), ImmutableList.of() ); assertFilterMatches( - new BoundDimFilter(Column.TIME_COLUMN_NAME, "Fridax", "Fridaz", false, false, null, exfn, StringComparators.ALPHANUMERIC), + new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Fridax", "Fridaz", false, false, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.of("4") ); assertFilterMatches( - new BoundDimFilter(Column.TIME_COLUMN_NAME, "Friday", "Friday", true, true, null, exfn, StringComparators.ALPHANUMERIC), + new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Friday", "Friday", true, true, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.of() ); assertFilterMatches( - new InDimFilter(Column.TIME_COLUMN_NAME, Arrays.asList("Caturday", "Saturday", "Tuesday"), exfn), + new InDimFilter(ColumnHolder.TIME_COLUMN_NAME, Arrays.asList("Caturday", "Saturday", "Tuesday"), exfn), ImmutableList.of("1", "5") ); @@ -200,23 +200,23 @@ public void testTimeFilterWithExtractionFn() "Hello", "World", "1", "2", "3", "4", "5", "6", "7" ); assertFilterMatches( - new InDimFilter(Column.TIME_COLUMN_NAME, bigList, exfn), + new InDimFilter(ColumnHolder.TIME_COLUMN_NAME, bigList, exfn), ImmutableList.of("1", "5") ); String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday') }"; assertFilterMatches( - new JavaScriptDimFilter(Column.TIME_COLUMN_NAME, jsFn, exfn, JavaScriptConfig.getEnabledInstance()), + new JavaScriptDimFilter(ColumnHolder.TIME_COLUMN_NAME, jsFn, exfn, JavaScriptConfig.getEnabledInstance()), ImmutableList.of("2", "3") ); assertFilterMatches( - new RegexDimFilter(Column.TIME_COLUMN_NAME, ".*day", exfn), + new RegexDimFilter(ColumnHolder.TIME_COLUMN_NAME, ".*day", exfn), ImmutableList.of("0", "1", "2", "3", "4", "5") ); assertFilterMatches( - new SearchQueryDimFilter(Column.TIME_COLUMN_NAME, new ContainsSearchQuerySpec("s", true), exfn), + new SearchQueryDimFilter(ColumnHolder.TIME_COLUMN_NAME, new ContainsSearchQuerySpec("s", true), exfn), ImmutableList.of("1", "2", "3") ); } @@ -232,7 +232,7 @@ public void testTimeFilterWithTimeFormatExtractionFn() false ); assertFilterMatches( - new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Wednesday", exfn), + new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Wednesday", exfn), ImmutableList.of("0", "1", "2", "3", "4", "5") ); } @@ -242,7 +242,7 @@ public void testIntervalFilter() { assertFilterMatches( new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList(Intervals.of("1970-01-01T00:00:00.001Z/1970-01-01T00:00:00.005Z")), null ), @@ -251,7 +251,7 @@ public void testIntervalFilter() assertFilterMatches( new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.000Z/1970-01-01T00:00:00.003Z"), Intervals.of("1970-01-01T00:00:00.004Z/1970-01-01T00:00:00.006Z") @@ -263,7 +263,7 @@ public void testIntervalFilter() assertFilterMatches( new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList( Intervals.of("1970-01-01T00:00:00.000Z/1970-01-01T00:00:00.001Z"), Intervals.of("1970-01-01T00:00:00.003Z/1970-01-01T00:00:00.006Z"), @@ -279,7 +279,7 @@ public void testIntervalFilter() ExtractionFn exFn = new JavaScriptExtractionFn(timeBoosterJsFn, true, JavaScriptConfig.getEnabledInstance()); assertFilterMatches( new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, Arrays.asList(Intervals.of("1970-01-01T02:00:00.001Z/1970-01-01T02:00:00.005Z")), exFn ), diff --git a/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java b/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java index 11ed71272db7..236880733efa 100644 --- a/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java +++ b/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java @@ -28,7 +28,7 @@ import io.druid.java.util.common.io.smoosh.SmooshedWriter; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ObjectColumnSelector; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ComplexColumn; import io.druid.segment.column.ValueType; @@ -115,8 +115,8 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) .setFileMapper(mapper); serde.deserializeColumn(mapper.mapFile("test"), builder); - Column column = builder.build(); - ComplexColumn complexColumn = column.getComplexColumn(); + ColumnHolder columnHolder = builder.build(); + ComplexColumn complexColumn = (ComplexColumn) columnHolder.getColumn(); HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector(); for (int i = 0; i < aCase; i++) { diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java b/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java index 493196109739..341a9b9ffc66 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java @@ -39,7 +39,7 @@ import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.Filters; import io.druid.segment.transform.TransformSpec; @@ -92,7 +92,7 @@ public Sequence apply(WindowedStorageAdapter adapter) public Sequence apply(final Cursor cursor) { final BaseLongColumnValueSelector timestampColumnSelector = - cursor.getColumnSelectorFactory().makeColumnValueSelector(Column.TIME_COLUMN_NAME); + cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME); final Map dimSelectors = Maps.newHashMap(); for (String dim : dims) { diff --git a/services/src/main/java/io/druid/cli/DumpSegment.java b/services/src/main/java/io/druid/cli/DumpSegment.java index d1a1c3ed2365..ce48faed9332 100644 --- a/services/src/main/java/io/druid/cli/DumpSegment.java +++ b/services/src/main/java/io/druid/cli/DumpSegment.java @@ -74,7 +74,7 @@ import io.druid.segment.QueryableIndexStorageAdapter; import io.druid.segment.VirtualColumns; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnConfig; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.ConciseBitmapSerdeFactory; @@ -285,7 +285,7 @@ public Object apply(Cursor cursor) final String columnName = columnNames.get(i); final Object value = selectors.get(i).getObject(); - if (timeISO8601 && columnNames.get(i).equals(Column.TIME_COLUMN_NAME)) { + if (timeISO8601 && columnNames.get(i).equals(ColumnHolder.TIME_COLUMN_NAME)) { row.put(columnName, new DateTime(value, DateTimeZone.UTC).toString()); } else { row.put(columnName, value); @@ -350,8 +350,8 @@ public Object apply(final OutputStream out) jg.writeStartObject(); for (final String columnName : columnNames) { - final Column column = index.getColumn(columnName); - final BitmapIndex bitmapIndex = column.getBitmapIndex(); + final ColumnHolder columnHolder = index.getColumn(columnName); + final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); if (bitmapIndex == null) { jg.writeNullField(columnName); @@ -399,7 +399,7 @@ private List getColumnsToInclude(final QueryableIndex index) // Empty columnNames => include all columns. if (columnNames.isEmpty()) { - columnNames.add(Column.TIME_COLUMN_NAME); + columnNames.add(ColumnHolder.TIME_COLUMN_NAME); Iterables.addAll(columnNames, index.getColumnNames()); } else { // Remove any provided columns that do not exist in this segment. diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java index 8917260d7417..4883b673a334 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java @@ -41,7 +41,7 @@ import io.druid.query.filter.OrDimFilter; import io.druid.query.ordering.StringComparator; import io.druid.query.ordering.StringComparators; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.sql.calcite.filtration.BoundRefKey; import io.druid.sql.calcite.filtration.Bounds; @@ -385,7 +385,7 @@ private static DimFilter toSimpleLeafFilter( if (queryGranularity != null) { // lhs is FLOOR(__time TO granularity); rhs must be a timestamp final long rhsMillis = Calcites.calciteDateTimeLiteralToJoda(rhs, plannerContext.getTimeZone()).getMillis(); - return buildTimeFloorFilter(Column.TIME_COLUMN_NAME, queryGranularity, flippedKind, rhsMillis); + return buildTimeFloorFilter(ColumnHolder.TIME_COLUMN_NAME, queryGranularity, flippedKind, rhsMillis); } // In the general case, lhs must be translatable to a SimpleExtraction to be simple-filterable. @@ -396,7 +396,7 @@ private static DimFilter toSimpleLeafFilter( final String column = lhsExpression.getSimpleExtraction().getColumn(); final ExtractionFn extractionFn = lhsExpression.getSimpleExtraction().getExtractionFn(); - if (column.equals(Column.TIME_COLUMN_NAME) && extractionFn instanceof TimeFormatExtractionFn) { + if (column.equals(ColumnHolder.TIME_COLUMN_NAME) && extractionFn instanceof TimeFormatExtractionFn) { // Check if we can strip the extractionFn and convert the filter to a direct filter on __time. // This allows potential conversion to query-level "intervals" later on, which is ideal for Druid queries. @@ -557,7 +557,7 @@ public static Granularity toQueryGranularity(final DruidExpression expression, f final Expr arg = expr.getArg(); final Granularity granularity = expr.getGranularity(); - if (Column.TIME_COLUMN_NAME.equals(Parser.getIdentifierIfIdentifier(arg))) { + if (ColumnHolder.TIME_COLUMN_NAME.equals(Parser.getIdentifierIfIdentifier(arg))) { return granularity; } else { return null; diff --git a/sql/src/main/java/io/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java b/sql/src/main/java/io/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java index 1ec9f0691d10..06ce4f8efc2d 100644 --- a/sql/src/main/java/io/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java +++ b/sql/src/main/java/io/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java @@ -31,7 +31,7 @@ import io.druid.query.filter.NotDimFilter; import io.druid.query.filter.OrDimFilter; import io.druid.query.ordering.StringComparators; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import java.util.List; @@ -39,7 +39,7 @@ public class MoveTimeFiltersToIntervals implements Function apply(final Result result) final Object[] retVal = new Object[fieldList.size()]; for (RelDataTypeField field : fieldList) { final String outputName = outputRowSignature.getRowOrder().get(field.getIndex()); - if (outputName.equals(Column.TIME_COLUMN_NAME)) { + if (outputName.equals(ColumnHolder.TIME_COLUMN_NAME)) { retVal[field.getIndex()] = coerce( holder.getTimestamp().getMillis(), field.getType().getSqlTypeName() diff --git a/sql/src/main/java/io/druid/sql/calcite/table/RowSignature.java b/sql/src/main/java/io/druid/sql/calcite/table/RowSignature.java index 1e4b779170e4..c3721774a80b 100644 --- a/sql/src/main/java/io/druid/sql/calcite/table/RowSignature.java +++ b/sql/src/main/java/io/druid/sql/calcite/table/RowSignature.java @@ -29,7 +29,7 @@ import io.druid.java.util.common.Pair; import io.druid.query.ordering.StringComparator; import io.druid.query.ordering.StringComparators; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.sql.calcite.expression.SimpleExtraction; import io.druid.sql.calcite.planner.Calcites; @@ -150,7 +150,7 @@ public RelDataType getRelDataType(final RelDataTypeFactory typeFactory) final ValueType columnType = getColumnType(columnName); final RelDataType type; - if (Column.TIME_COLUMN_NAME.equals(columnName)) { + if (ColumnHolder.TIME_COLUMN_NAME.equals(columnName)) { type = Calcites.createSqlType(typeFactory, SqlTypeName.TIMESTAMP); } else { switch (columnType) { diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index a9372226a472..00ddb39470ca 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -83,7 +83,7 @@ import io.druid.query.topn.NumericTopNMetricSpec; import io.druid.query.topn.TopNQueryBuilder; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; import io.druid.server.security.AuthenticationResult; @@ -7022,7 +7022,7 @@ private static BoundDimFilter TIME_BOUND(final Object intervalObj) { final Interval interval = new Interval(intervalObj, ISOChronology.getInstanceUTC()); return new BoundDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, String.valueOf(interval.getStartMillis()), String.valueOf(interval.getEndMillis()), false, diff --git a/sql/src/test/java/io/druid/sql/calcite/filtration/FiltrationTest.java b/sql/src/test/java/io/druid/sql/calcite/filtration/FiltrationTest.java index c7734c0297f1..8fe3cb1d7614 100644 --- a/sql/src/test/java/io/druid/sql/calcite/filtration/FiltrationTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/filtration/FiltrationTest.java @@ -23,7 +23,7 @@ import io.druid.java.util.common.Intervals; import io.druid.query.filter.IntervalDimFilter; import io.druid.query.filter.NotDimFilter; -import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnHolder; import io.druid.sql.calcite.util.CalciteTestBase; import org.junit.Assert; import org.junit.Test; @@ -36,7 +36,7 @@ public void testNotIntervals() final Filtration filtration = Filtration.create( new NotDimFilter( new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, ImmutableList.of(Intervals.of("2000/2001"), Intervals.of("2002/2003")), null ) @@ -52,7 +52,7 @@ public void testNotIntervals() Assert.assertEquals( new NotDimFilter( new IntervalDimFilter( - Column.TIME_COLUMN_NAME, + ColumnHolder.TIME_COLUMN_NAME, ImmutableList.of(Intervals.of("2000/2001"), Intervals.of("2002/2003")), null ) From e768315b259f9cfe9e79c9385de27fd38c0b455e Mon Sep 17 00:00:00 2001 From: leventov Date: Sat, 7 Jul 2018 14:39:32 +0200 Subject: [PATCH 03/29] Fixes --- .../query/dimension/DefaultDimensionSpec.java | 30 ++++++------------- .../query/groupby/GroupByQueryEngine.java | 12 ++++---- .../groupby/GroupByQueryQueryToolChest.java | 17 ++--------- .../segment/QueryableIndexStorageAdapter.java | 7 +++-- .../java/io/druid/segment/StorageAdapter.java | 8 ++--- .../druid/segment/StringDimensionIndexer.java | 2 +- .../IncrementalIndexStorageAdapter.java | 8 +++-- .../query/groupby/GroupByQueryRunnerTest.java | 29 +++++++----------- 8 files changed, 40 insertions(+), 73 deletions(-) diff --git a/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java index eb5da594ef55..5e159e15960c 100644 --- a/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java @@ -21,17 +21,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Function; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.DimensionSelector; import io.druid.segment.column.ValueType; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; /** */ @@ -49,18 +49,9 @@ public static List toSpec(String... dimensionNames) public static List toSpec(Iterable dimensionNames) { - return Lists.newArrayList( - Iterables.transform( - dimensionNames, new Function() - { - @Override - public DimensionSpec apply(String input) - { - return new DefaultDimensionSpec(input, input); - } - } - ) - ); + return StreamSupport.stream(dimensionNames.spliterator(), false) + .map(input -> new DefaultDimensionSpec(input, input)) + .collect(Collectors.toList()); } private static final byte CACHE_TYPE_ID = 0x0; @@ -71,8 +62,8 @@ public DimensionSpec apply(String input) @JsonCreator public DefaultDimensionSpec( @JsonProperty("dimension") String dimension, - @JsonProperty("outputName") String outputName, - @JsonProperty("outputType") ValueType outputType + @JsonProperty("outputName") @Nullable String outputName, + @JsonProperty("outputType") @Nullable ValueType outputType ) { this.dimension = dimension; @@ -82,10 +73,7 @@ public DefaultDimensionSpec( this.outputName = outputName == null ? dimension : outputName; } - public DefaultDimensionSpec( - String dimension, - String outputName - ) + public DefaultDimensionSpec(String dimension, String outputName) { this(dimension, outputName, ValueType.STRING); } diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java index 9e87e76e2b83..460027a66343 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java @@ -333,14 +333,12 @@ public RowIterator(GroupByQuery query, final Cursor cursor, ByteBuffer metricsBu } final DimensionSelector selector = cursor.getColumnSelectorFactory().makeDimensionSelector(dimSpec); - if (selector != null) { - if (selector.getValueCardinality() == DimensionSelector.CARDINALITY_UNKNOWN) { - throw new UnsupportedOperationException( - "GroupBy v1 does not support dimension selectors with unknown cardinality."); - } - dimensions.add(selector); - dimNames.add(dimSpec.getOutputName()); + if (selector.getValueCardinality() == DimensionSelector.CARDINALITY_UNKNOWN) { + throw new UnsupportedOperationException( + "GroupBy v1 does not support dimension selectors with unknown cardinality."); } + dimensions.add(selector); + dimNames.add(dimSpec.getOutputName()); } aggregatorSpecs = query.getAggregatorSpecs(); diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java index 1252caa40ddd..0dd68159354c 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java @@ -122,11 +122,7 @@ public Sequence run(QueryPlus queryPlus, Map responseC final GroupByQuery groupByQuery = (GroupByQuery) queryPlus.getQuery(); if (strategySelector.strategize(groupByQuery).doMergeResults(groupByQuery)) { - return initAndMergeGroupByResults( - groupByQuery, - runner, - responseContext - ); + return initAndMergeGroupByResults(groupByQuery, runner, responseContext); } return runner.run(queryPlus, responseContext); } @@ -142,16 +138,7 @@ private Sequence initAndMergeGroupByResults( final GroupByStrategy groupByStrategy = strategySelector.strategize(query); final GroupByQueryResource resource = groupByStrategy.prepareResource(query, false); - return Sequences.withBaggage( - mergeGroupByResults( - groupByStrategy, - query, - resource, - runner, - context - ), - resource - ); + return Sequences.withBaggage(mergeGroupByResults(groupByStrategy, query, resource, runner, context), resource); } private Sequence mergeGroupByResults( diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index 491e23b02304..e4f32e7ad96f 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -37,11 +37,11 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BaseColumn; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnHolder; +import io.druid.segment.column.ComplexColumn; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.NumericColumn; -import io.druid.segment.column.ComplexColumn; import io.druid.segment.data.Indexed; import io.druid.segment.data.Offset; import io.druid.segment.data.ReadableOffset; @@ -101,6 +101,9 @@ public Iterable getAvailableMetrics() public int getDimensionCardinality(String dimension) { ColumnHolder columnHolder = index.getColumn(dimension); + if (columnHolder == null) { + return 0; + } try (BaseColumn col = columnHolder.getColumn()) { if (!(col instanceof DictionaryEncodedColumn)) { return Integer.MAX_VALUE; diff --git a/processing/src/main/java/io/druid/segment/StorageAdapter.java b/processing/src/main/java/io/druid/segment/StorageAdapter.java index f49d31cc39b4..12a1c4125f0c 100644 --- a/processing/src/main/java/io/druid/segment/StorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/StorageAdapter.java @@ -39,12 +39,8 @@ public interface StorageAdapter extends CursorFactory Iterable getAvailableMetrics(); /** - * Returns the number of distinct values for the given dimension column - * For dimensions of unknown cardinality, e.g. __time this currently returns - * Integer.MAX_VALUE - * - * @param column - * @return + * Returns the number of distinct values for the given column if known, or {@link Integer#MAX_VALUE} is unknown or + * the column is numeric. If the column doesn't exist, returns 0. */ int getDimensionCardinality(String column); DateTime getMinTime(); diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 31eca6d61c76..4e62644690a4 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -674,7 +674,7 @@ public Object convertUnsortedEncodedKeyComponentToActualList(int[] key) return null; } if (key.length == 1) { - return NullHandling.nullToEmptyIfNeeded(getActualValue(key[0], false)); + return getActualValue(key[0], false); } else { String[] rowArray = new String[key.length]; for (int i = 0; i < key.length; i++) { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index a572eddf4f8c..66bde8295d16 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -33,11 +33,12 @@ import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.Cursor; import io.druid.segment.DimensionIndexer; +import io.druid.segment.DimensionSelector; import io.druid.segment.Metadata; import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumns; -import io.druid.segment.column.ColumnHolder; import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnHolder; import io.druid.segment.data.Indexed; import io.druid.segment.data.ListIndexed; import io.druid.segment.filter.BooleanValueMatcher; @@ -94,8 +95,9 @@ public int getDimensionCardinality(String dimension) return 0; } - DimensionIndexer indexer = index.getDimension(dimension).getIndexer(); - return indexer.getCardinality(); + DimensionIndexer indexer = desc.getIndexer(); + int cardinality = indexer.getCardinality(); + return cardinality != DimensionSelector.CARDINALITY_UNKNOWN ? cardinality : Integer.MAX_VALUE; } @Override diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 8e45d0f497c1..608e9ab8a193 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -7221,24 +7221,17 @@ public void testGroupByWithExtractionDimFilterNullDims() MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); - GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) - .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec( - "null_column", - "alias" - ))) - .setAggregatorSpecs( - Arrays.asList( - QueryRunnerTestHelper.rowsCount, - new LongSumAggregatorFactory("idx", "index") - )) - .setGranularity(QueryRunnerTestHelper.dayGran) - .setDimFilter(new ExtractionDimFilter( - "null_column", - "EMPTY", - lookupExtractionFn, - null - )).build(); + GroupByQuery query = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("null_column", "alias"))) + .setAggregatorSpecs( + Arrays.asList(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index")) + ) + .setGranularity(QueryRunnerTestHelper.dayGran) + .setDimFilter(new ExtractionDimFilter("null_column", "EMPTY", lookupExtractionFn, null)) + .build(); List expectedResults = Arrays .asList( GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", null, "rows", 13L, "idx", 6619L), From 9dce33784eebe3e11a3c47990d487a87b0e82871 Mon Sep 17 00:00:00 2001 From: leventov Date: Sun, 8 Jul 2018 16:39:23 +0200 Subject: [PATCH 04/29] Null fix --- .../src/main/java/io/druid/segment/StringDimensionIndexer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 4e62644690a4..4d5ea94db54d 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -670,7 +670,7 @@ public ColumnValueSelector makeColumnValueSelector( @Override public Object convertUnsortedEncodedKeyComponentToActualList(int[] key) { - if (key.length == 0) { + if (key == null || key.length == 0) { return null; } if (key.length == 1) { From b5ef733470dbfaa798f2960bd5bc33cd80a7eb7b Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 9 Jul 2018 16:21:28 +0200 Subject: [PATCH 05/29] Fix inspection: --- .../java/io/druid/segment/StringDimensionMergerV9.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index f7cb2b141f57..6c4126306456 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -41,6 +41,7 @@ import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.BitmapValues; import io.druid.segment.data.ByteBufferWriter; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.ColumnarIntsSerializer; import io.druid.segment.data.ColumnarMultiIntsSerializer; import io.druid.segment.data.CompressedVSizeColumnarIntsSerializer; @@ -62,6 +63,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.IntBuffer; import java.util.ArrayList; import java.util.List; @@ -667,8 +669,12 @@ protected IndexSeeker[] toIndexSeekers( if (dimConversion != null) { seekers[i] = new IndexSeekerWithConversion((IntBuffer) dimConversion.asReadOnlyBuffer().rewind()); } else { - Indexed dimValueLookup = adapters.get(i).getDimValueLookup(dimension); - seekers[i] = new IndexSeekerWithoutConversion(dimValueLookup == null ? 0 : dimValueLookup.size()); + try (CloseableIndexed dimValueLookup = adapters.get(i).getDimValueLookup(dimension)) { + seekers[i] = new IndexSeekerWithoutConversion(dimValueLookup == null ? 0 : dimValueLookup.size()); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } } return seekers; From 9c3f4f6b5d11022d3de93bbe93eb70c90abe73e7 Mon Sep 17 00:00:00 2001 From: leventov Date: Sat, 14 Jul 2018 12:45:56 -0500 Subject: [PATCH 06/29] Fix in JavaScriptAggregatorFactory and License header fixes --- .../main/java/io/druid/benchmark/package-info.java | 12 ++++++------ .../datasketches/quantiles/package-info.java | 12 ++++++------ .../aggregation/datasketches/theta/package-info.java | 12 ++++++------ .../aggregation/datasketches/tuple/package-info.java | 12 ++++++------ .../query/aggregation/histogram/package-info.java | 12 ++++++------ .../query/aggregation/variance/package-info.java | 12 ++++++------ .../aggregation/JavaScriptAggregatorFactory.java | 5 +++-- .../query/aggregation/hyperloglog/package-info.java | 12 ++++++------ .../io/druid/query/aggregation/package-info.java | 12 ++++++------ .../druid/query/aggregation/post/package-info.java | 12 ++++++------ .../query/groupby/epinephelinae/package-info.java | 12 ++++++------ .../java/io/druid/query/groupby/package-info.java | 12 ++++++------ .../io/druid/segment/AbstractDimensionSelector.java | 12 ++++++------ .../java/io/druid/segment/data/package-info.java | 12 ++++++------ .../segment/selector/settable/package-info.java | 12 ++++++------ .../java/io/druid/segment/virtual/package-info.java | 12 ++++++------ 16 files changed, 93 insertions(+), 92 deletions(-) diff --git a/benchmarks/src/main/java/io/druid/benchmark/package-info.java b/benchmarks/src/main/java/io/druid/benchmark/package-info.java index 2efaefa21376..25ed52cebc98 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/package-info.java +++ b/benchmarks/src/main/java/io/druid/benchmark/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/package-info.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/package-info.java index afb0d18750a0..064381dce3b7 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/package-info.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/package-info.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/package-info.java index 6ab2f4ffe78a..d479808dc8ba 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/package-info.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/package-info.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/package-info.java index c7cba4854aca..d8dfd77a9968 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/package-info.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/package-info.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/package-info.java index 96299e0dcbe1..04f42a21298a 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/package-info.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/package-info.java b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/package-info.java index 543848b165cc..81ccff258530 100644 --- a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/package-info.java +++ b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java index bc8025e4b3d4..16e32bd00415 100644 --- a/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java @@ -356,8 +356,9 @@ public double aggregate(final double current, final BaseObjectColumnValueSelecto private Object[] arrayToObjectArray(Object array) { - final Object[] objectArray = new Object[Array.getLength(array)]; - for (int j = 0; j < Array.getLength(array); j++) { + int len = Array.getLength(array); + final Object[] objectArray = new Object[len]; + for (int j = 0; j < len; j++) { objectArray[j] = Array.get(array, j); } return objectArray; diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/package-info.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/package-info.java index 44d62d8b2bb7..79e20cc8bb67 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/package-info.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/processing/src/main/java/io/druid/query/aggregation/package-info.java b/processing/src/main/java/io/druid/query/aggregation/package-info.java index 9c684b73bbce..b1b9c2635c58 100644 --- a/processing/src/main/java/io/druid/query/aggregation/package-info.java +++ b/processing/src/main/java/io/druid/query/aggregation/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/processing/src/main/java/io/druid/query/aggregation/post/package-info.java b/processing/src/main/java/io/druid/query/aggregation/post/package-info.java index 3a599b6a3f2b..728d4133f762 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/package-info.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/package-info.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/package-info.java index ebdc7455142b..296a2d4a2454 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/package-info.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/processing/src/main/java/io/druid/query/groupby/package-info.java b/processing/src/main/java/io/druid/query/groupby/package-info.java index 6451cc6747cb..75731a28ba90 100644 --- a/processing/src/main/java/io/druid/query/groupby/package-info.java +++ b/processing/src/main/java/io/druid/query/groupby/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/processing/src/main/java/io/druid/segment/AbstractDimensionSelector.java b/processing/src/main/java/io/druid/segment/AbstractDimensionSelector.java index dd06398faf0c..6f09565c6eb1 100644 --- a/processing/src/main/java/io/druid/segment/AbstractDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/AbstractDimensionSelector.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/processing/src/main/java/io/druid/segment/data/package-info.java b/processing/src/main/java/io/druid/segment/data/package-info.java index 0e54ffa0fba4..75143832b754 100644 --- a/processing/src/main/java/io/druid/segment/data/package-info.java +++ b/processing/src/main/java/io/druid/segment/data/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/processing/src/main/java/io/druid/segment/selector/settable/package-info.java b/processing/src/main/java/io/druid/segment/selector/settable/package-info.java index cd5a886eddf1..ddfa0379878b 100644 --- a/processing/src/main/java/io/druid/segment/selector/settable/package-info.java +++ b/processing/src/main/java/io/druid/segment/selector/settable/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ diff --git a/processing/src/main/java/io/druid/segment/virtual/package-info.java b/processing/src/main/java/io/druid/segment/virtual/package-info.java index 8c0f70f313df..c864beb4fde0 100644 --- a/processing/src/main/java/io/druid/segment/virtual/package-info.java +++ b/processing/src/main/java/io/druid/segment/virtual/package-info.java @@ -1,18 +1,18 @@ /* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the + * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ From 1bf64714cf04c379089031d77b5e1e3094fcf342 Mon Sep 17 00:00:00 2001 From: leventov Date: Sat, 14 Jul 2018 12:58:37 -0500 Subject: [PATCH 07/29] Fix checkstyle --- .../src/main/java/io/druid/segment/column/ComplexColumn.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/segment/column/ComplexColumn.java b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java index 0e5f90bbcc42..a1d42235c374 100644 --- a/processing/src/main/java/io/druid/segment/column/ComplexColumn.java +++ b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java @@ -56,7 +56,8 @@ public Object getRowValue(int rowNum) return index.get(rowNum); } - public int getLength() { + public int getLength() + { return index.size(); } From b01a43fd44e8ce1729d7b8ffd2c95d9e61f91531 Mon Sep 17 00:00:00 2001 From: leventov Date: Sun, 15 Jul 2018 09:34:42 -0500 Subject: [PATCH 08/29] Close dimension value lookups in StringDimensionMergerV9 --- .../java/io/druid/segment/DimensionHandler.java | 5 +++-- .../io/druid/segment/DoubleDimensionHandler.java | 4 +++- .../io/druid/segment/FloatDimensionHandler.java | 4 +++- .../main/java/io/druid/segment/IndexMerger.java | 2 +- .../java/io/druid/segment/IndexMergerV9.java | 16 ++++++++-------- .../io/druid/segment/LongDimensionHandler.java | 4 +++- .../io/druid/segment/StringDimensionHandler.java | 6 ++++-- .../druid/segment/StringDimensionMergerV9.java | 14 ++++++++++---- 8 files changed, 35 insertions(+), 20 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/DimensionHandler.java b/processing/src/main/java/io/druid/segment/DimensionHandler.java index b84f6baf74d7..5001c1e27043 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandler.java @@ -20,6 +20,7 @@ package io.druid.segment; import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; +import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.selector.settable.SettableColumnValueSelector; import io.druid.segment.writeout.SegmentWriteOutMedium; @@ -92,14 +93,14 @@ default MultiValueHandling getMultivalueHandling() * @param segmentWriteOutMedium this SegmentWriteOutMedium object could be used internally in the created merger, if needed * @param capabilities The ColumnCapabilities of the dimension represented by this DimensionHandler * @param progress ProgressIndicator used by the merging process - * @return A new DimensionMergerV9 object. */ DimensionMergerV9 makeMerger( IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, - ProgressIndicator progress + ProgressIndicator progress, + Closer closer ); /** diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java b/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java index 5cb645ffe270..68a3dc52e303 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.selector.settable.SettableColumnValueSelector; import io.druid.segment.selector.settable.SettableDoubleColumnValueSelector; @@ -62,7 +63,8 @@ public DimensionMergerV9 makeMerger( IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, - ProgressIndicator progress + ProgressIndicator progress, + Closer closer ) { return new DoubleDimensionMergerV9( diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java index 0e72275376d1..5b7e6dcba756 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.selector.settable.SettableColumnValueSelector; import io.druid.segment.selector.settable.SettableFloatColumnValueSelector; @@ -62,7 +63,8 @@ public DimensionMergerV9 makeMerger( IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, - ProgressIndicator progress + ProgressIndicator progress, + Closer closer ) { return new FloatDimensionMergerV9( diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index 79bacc3c3a7a..4a832cb5d6bc 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -307,7 +307,7 @@ public int seek(int dictId) static TransformableRowIterator toMergedIndexRowIterator( TransformableRowIterator sourceRowIterator, int indexNumber, - final List mergers + final List mergers ) { RowPointer sourceRowPointer = sourceRowIterator.getPointer(); diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index 78e2ad5d0027..c78162193e8f 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -116,7 +116,7 @@ private File makeIndexFiles( List metadataList = Lists.transform(adapters, IndexableAdapter::getMetadata); - Metadata segmentMetadata = null; + Metadata segmentMetadata; if (metricAggs != null) { AggregatorFactory[] combiningMetricAggs = new AggregatorFactory[metricAggs.length]; for (int i = 0; i < metricAggs.length; i++) { @@ -161,10 +161,10 @@ private File makeIndexFiles( mergeCapabilities(adapters, mergedDimensions, metricsValueTypes, metricTypeNames, dimCapabilities); final Map handlers = makeDimensionHandlers(mergedDimensions, dimCapabilities); - final List mergers = new ArrayList<>(); + final List mergers = new ArrayList<>(); for (int i = 0; i < mergedDimensions.size(); i++) { DimensionHandler handler = handlers.get(mergedDimensions.get(i)); - mergers.add(handler.makeMerger(indexSpec, segmentWriteOutMedium, dimCapabilities.get(i), progress)); + mergers.add(handler.makeMerger(indexSpec, segmentWriteOutMedium, dimCapabilities.get(i), progress, closer)); } /************* Setup Dim Conversions **************/ @@ -212,7 +212,7 @@ private File makeIndexFiles( ); for (int i = 0; i < mergedDimensions.size(); i++) { - DimensionMergerV9 merger = (DimensionMergerV9) mergers.get(i); + DimensionMergerV9 merger = mergers.get(i); merger.writeIndexes(rowNumConversions); if (merger.canSkip()) { continue; @@ -262,7 +262,7 @@ private void makeIndexBinary( final List mergedMetrics, final ProgressIndicator progress, final IndexSpec indexSpec, - final List mergers + final List mergers ) throws IOException { final String section = "make index.drd"; @@ -468,7 +468,7 @@ private List mergeIndexesAndWriteColumns( final TimeAndDimsIterator timeAndDimsIterator, final ColumnSerializer timeWriter, final ArrayList metricWriters, - final List mergers, + final List mergers, final boolean fillRowNumConversions ) throws IOException { @@ -686,7 +686,7 @@ private void writeDimValuesAndSetupDimConversion( final List indexes, final ProgressIndicator progress, final List mergedDimensions, - final List mergers + final List mergers ) throws IOException { final String section = "setup dimension conversions"; @@ -1009,7 +1009,7 @@ private TimeAndDimsIterator makeMergedTimeAndDimsIterator( final List mergedMetrics, final Function, TimeAndDimsIterator> rowMergerFn, final Map handlers, - final List mergers + final List mergers ) { List perIndexRowIterators = Lists.newArrayListWithCapacity(indexes.size()); diff --git a/processing/src/main/java/io/druid/segment/LongDimensionHandler.java b/processing/src/main/java/io/druid/segment/LongDimensionHandler.java index a102573c80a9..9f55df4aaaf3 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionHandler.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.selector.settable.SettableColumnValueSelector; import io.druid.segment.selector.settable.SettableLongColumnValueSelector; @@ -62,7 +63,8 @@ public DimensionMergerV9 makeMerger( IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, - ProgressIndicator progress + ProgressIndicator progress, + Closer closer ) { return new LongDimensionMergerV9( diff --git a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java index c1273189a652..f7f42913f5fb 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java @@ -21,6 +21,7 @@ import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.ZeroIndexedInts; @@ -161,7 +162,8 @@ public DimensionMergerV9 makeMerger( IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, - ProgressIndicator progress + ProgressIndicator progress, + Closer closer ) { // Sanity-check capabilities. @@ -173,6 +175,6 @@ public DimensionMergerV9 makeMerger( ); } - return new StringDimensionMergerV9(dimensionName, indexSpec, segmentWriteOutMedium, capabilities, progress); + return new StringDimensionMergerV9(dimensionName, indexSpec, segmentWriteOutMedium, capabilities, progress, closer); } } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index 62f9f7deb6e0..7c460e353107 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -31,6 +31,7 @@ import io.druid.common.config.NullHandling; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.io.Closer; import io.druid.java.util.common.logger.Logger; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -91,24 +92,29 @@ public class StringDimensionMergerV9 implements DimensionMergerV9 private int rowCount = 0; private ColumnCapabilities capabilities; private List adapters; - private ProgressIndicator progress; private final IndexSpec indexSpec; private IndexMerger.DictionaryMergeIterator dictionaryMergeIterator; + private final ProgressIndicator progress; + private final Closer closer; + public StringDimensionMergerV9( String dimensionName, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, - ProgressIndicator progress + ProgressIndicator progress, + Closer closer ) { this.dimensionName = dimensionName; this.indexSpec = indexSpec; this.capabilities = capabilities; this.segmentWriteOutMedium = segmentWriteOutMedium; - this.progress = progress; nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); + + this.progress = progress; + this.closer = closer; } @Override @@ -130,7 +136,7 @@ public void writeMergedValueDictionary(List adapters) throws I Indexed dimValueLookup = null; Indexed[] dimValueLookups = new Indexed[adapters.size() + 1]; for (int i = 0; i < adapters.size(); i++) { - Indexed dimValues = adapters.get(i).getDimValueLookup(dimensionName); + Indexed dimValues = closer.register(adapters.get(i).getDimValueLookup(dimensionName)); if (dimValues != null && !allNull(dimValues)) { dimHasValues = true; hasNull |= dimValues.indexOf(null) >= 0; From 407890eb8b9f34647f0cf4b63adcd6367d5828d5 Mon Sep 17 00:00:00 2001 From: leventov Date: Sun, 15 Jul 2018 13:04:45 -0500 Subject: [PATCH 09/29] Suppress MustBeClosedChecker --- .../src/main/java/io/druid/segment/StringDimensionMergerV9.java | 1 + 1 file changed, 1 insertion(+) diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index 7c460e353107..06c9077fd3b9 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -136,6 +136,7 @@ public void writeMergedValueDictionary(List adapters) throws I Indexed dimValueLookup = null; Indexed[] dimValueLookups = new Indexed[adapters.size() + 1]; for (int i = 0; i < adapters.size(); i++) { + @SuppressWarnings("MustBeClosedChecker") // we register dimValues in the closer Indexed dimValues = closer.register(adapters.get(i).getDimValueLookup(dimensionName)); if (dimValues != null && !allNull(dimValues)) { dimHasValues = true; From bd10c1803a50752ff3e8749e0ac87856074c1b70 Mon Sep 17 00:00:00 2001 From: leventov Date: Sun, 15 Jul 2018 15:03:41 -0500 Subject: [PATCH 10/29] Close CloseableIndexed in tests --- .../segment/QueryableIndexIndexableAdapterTest.java | 9 ++++++--- .../segment/incremental/IncrementalIndexAdapterTest.java | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java b/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java index b080e3ec7787..c04ad8a0cb87 100644 --- a/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import io.druid.segment.data.BitmapValues; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; @@ -96,9 +97,11 @@ public void testGetBitmapIndex() throws Exception String dimension = "dim1"; @SuppressWarnings("UnusedAssignment") //null is added to all dimensions with value BitmapValues bitmapValues = adapter.getBitmapValues(dimension, 0); - for (int i = 0; i < adapter.getDimValueLookup(dimension).size(); i++) { - bitmapValues = adapter.getBitmapValues(dimension, i); - Assert.assertEquals(1, bitmapValues.size()); + try (CloseableIndexed dimValueLookup = adapter.getDimValueLookup(dimension)) { + for (int i = 0; i < dimValueLookup.size(); i++) { + bitmapValues = adapter.getBitmapValues(dimension, i); + Assert.assertEquals(1, bitmapValues.size()); + } } } } diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexAdapterTest.java index 55f05460b3e4..76044b3048f7 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexAdapterTest.java @@ -23,6 +23,7 @@ import io.druid.segment.IndexableAdapter; import io.druid.segment.RowIterator; import io.druid.segment.data.BitmapValues; +import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; @@ -54,9 +55,11 @@ public void testGetBitmapIndex() throws Exception INDEX_SPEC.getBitmapSerdeFactory().getBitmapFactory() ); String dimension = "dim1"; - for (int i = 0; i < adapter.getDimValueLookup(dimension).size(); i++) { - BitmapValues bitmapValues = adapter.getBitmapValues(dimension, i); - Assert.assertEquals(1, bitmapValues.size()); + try (CloseableIndexed dimValueLookup = adapter.getDimValueLookup(dimension)) { + for (int i = 0; i < dimValueLookup.size(); i++) { + BitmapValues bitmapValues = adapter.getBitmapValues(dimension, i); + Assert.assertEquals(1, bitmapValues.size()); + } } } From d5bff2a75ff423cc17ba177654298ffe21cd0fc6 Mon Sep 17 00:00:00 2001 From: leventov Date: Sat, 21 Jul 2018 14:29:23 -0500 Subject: [PATCH 11/29] Address review --- .../indexing/kafka/KafkaIndexTaskTest.java | 2 +- .../indexing/common/task/CompactionTask.java | 2 +- .../druid/indexing/overlord/TaskStorage.java | 2 - .../druid/query/metadata/SegmentAnalyzer.java | 2 +- .../io/druid/query/search/AutoStrategy.java | 2 +- .../query/search/UseIndexesStrategy.java | 4 +- .../java/io/druid/segment/ColumnSelector.java | 2 +- .../ColumnSelectorBitmapIndexSelector.java | 12 ++--- .../io/druid/segment/ColumnSerializer.java | 6 +++ .../QueryableIndexColumnSelectorFactory.java | 4 +- .../QueryableIndexIndexableAdapter.java | 14 +++--- .../segment/QueryableIndexStorageAdapter.java | 16 +++---- .../druid/segment/SimpleQueryableIndex.java | 4 +- .../segment/data/GenericIndexedWriter.java | 3 +- .../java/io/druid/segment/filter/Filters.java | 2 +- .../segment/serde/ComplexMetricSerde.java | 4 +- .../java/io/druid/segment/EmptyIndexTest.java | 2 +- .../segment/IndexMergerNullHandlingTest.java | 2 +- .../io/druid/segment/IndexMergerTestBase.java | 48 +++++++++---------- .../druid/segment/realtime/plumber/Sink.java | 2 +- .../main/java/io/druid/cli/DumpSegment.java | 4 +- 21 files changed, 72 insertions(+), 67 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java index ad66deb7cdf8..77229f6d6c30 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -2220,7 +2220,7 @@ private List readSegmentColumn(final String column, final SegmentDescrip ); IndexIO indexIO = new TestUtils().getTestIndexIO(); QueryableIndex index = indexIO.loadIndex(outputLocation); - DictionaryEncodedColumn theColumn = (DictionaryEncodedColumn) index.getColumn(column).getColumn(); + DictionaryEncodedColumn theColumn = (DictionaryEncodedColumn) index.getColumnHolder(column).getColumn(); List values = Lists.newArrayList(); for (int i = 0; i < theColumn.length(); i++) { int id = theColumn.getSingleValueRow(i); diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java index e736cf9ce6c8..dbfa6579948d 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java @@ -397,7 +397,7 @@ private static DimensionsSpec createDimensionsSpec(List> getActiveTaskInfo(@Nullable String dataSource); diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java index 83f38b59c2dd..6728198a1fdb 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java @@ -99,7 +99,7 @@ public Map analyze(Segment segment) Map columns = Maps.newTreeMap(); for (String columnName : columnNames) { - final ColumnHolder columnHolder = index == null ? null : index.getColumn(columnName); + final ColumnHolder columnHolder = index == null ? null : index.getColumnHolder(columnName); final ColumnCapabilities capabilities = columnHolder != null ? columnHolder.getCapabilities() : storageAdapter.getColumnCapabilities(columnName); diff --git a/processing/src/main/java/io/druid/query/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/AutoStrategy.java index cb51b9071e7e..69d90a1e56c3 100644 --- a/processing/src/main/java/io/druid/query/search/AutoStrategy.java +++ b/processing/src/main/java/io/druid/query/search/AutoStrategy.java @@ -114,7 +114,7 @@ private static long computeTotalCard(final QueryableIndex index, final Iterable< { long totalCard = 0; for (DimensionSpec dimension : dimensionSpecs) { - final ColumnHolder columnHolder = index.getColumn(dimension.getDimension()); + final ColumnHolder columnHolder = index.getColumnHolder(dimension.getDimension()); if (columnHolder != null) { final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); if (bitmapIndex != null) { diff --git a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java index 1086e1b6bea7..acff06500623 100644 --- a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java @@ -165,7 +165,7 @@ static ImmutableBitmap makeTimeFilteredBitmap( final ImmutableBitmap timeFilteredBitmap; if (!interval.contains(segment.getDataInterval())) { final MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); - final ColumnHolder timeColumnHolder = index.getColumn(ColumnHolder.TIME_COLUMN_NAME); + final ColumnHolder timeColumnHolder = index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME); try (final NumericColumn timeValues = (NumericColumn) timeColumnHolder.getColumn()) { int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true)); @@ -246,7 +246,7 @@ public Object2IntRBTreeMap execute(int limit) final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); for (DimensionSpec dimension : dimsToSearch) { - final ColumnHolder columnHolder = index.getColumn(dimension.getDimension()); + final ColumnHolder columnHolder = index.getColumnHolder(dimension.getDimension()); if (columnHolder == null) { continue; } diff --git a/processing/src/main/java/io/druid/segment/ColumnSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelector.java index ffdea3cd6117..e7537cc4afd0 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelector.java @@ -31,5 +31,5 @@ public interface ColumnSelector List getColumnNames(); @Nullable - ColumnHolder getColumn(String columnName); + ColumnHolder getColumnHolder(String columnName); } diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index eb3de05cf5dd..2ddc43d2594b 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -67,7 +67,7 @@ public CloseableIndexed getDimensionValues(String dimension) return null; } - final ColumnHolder columnHolder = index.getColumn(dimension); + final ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder == null) { return null; } @@ -129,14 +129,14 @@ public boolean hasMultipleValues(final String dimension) return virtualColumns.getVirtualColumn(dimension).capabilities(dimension).hasMultipleValues(); } - final ColumnHolder columnHolder = index.getColumn(dimension); + final ColumnHolder columnHolder = index.getColumnHolder(dimension); return columnHolder != null && columnHolder.getCapabilities().hasMultipleValues(); } @Override public int getNumRows() { - try (final NumericColumn column = (NumericColumn) index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { + try (final NumericColumn column = (NumericColumn) index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { return column.length(); } } @@ -155,7 +155,7 @@ public BitmapIndex getBitmapIndex(String dimension) return null; } - final ColumnHolder columnHolder = index.getColumn(dimension); + final ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder == null || !columnSupportsFiltering(columnHolder)) { // for missing columns and columns with types that do not support filtering, // treat the column as if it were a String column full of nulls. @@ -225,7 +225,7 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) return null; } - final ColumnHolder columnHolder = index.getColumn(dimension); + final ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder == null || !columnSupportsFiltering(columnHolder)) { if (NullHandling.isNullOrEquivalent(value)) { return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows()); @@ -249,7 +249,7 @@ public ImmutableRTree getSpatialIndex(String dimension) return ImmutableRTree.empty(); } - final ColumnHolder columnHolder = index.getColumn(dimension); + final ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder == null || !columnHolder.getCapabilities().hasSpatialIndexes()) { return ImmutableRTree.empty(); } diff --git a/processing/src/main/java/io/druid/segment/ColumnSerializer.java b/processing/src/main/java/io/druid/segment/ColumnSerializer.java index d8743af69f2a..8fad3f59b9a2 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/ColumnSerializer.java @@ -24,6 +24,12 @@ import java.io.IOException; +/** + * ColumnSerializer can be implemented in custom aggregator extensions that would like to take full control of column + * serialization. That implementation would be returned by overriding {@link + * io.druid.segment.serde.ComplexMetricSerde#getSerializer} + * @param + */ @ExtensionPoint public interface ColumnSerializer extends Serializer { diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java index 3904a703eb34..d5ba5bb8ddae 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java @@ -78,7 +78,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi final String dimension = dimensionSpec.getDimension(); final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); - final ColumnHolder columnHolder = index.getColumn(dimension); + final ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder == null) { return DimensionSelectorUtils.constantSelector(null, extractionFn); } @@ -117,7 +117,7 @@ public ColumnValueSelector makeColumnValueSelector(String columnName) } BaseColumn column = columnCache.computeIfAbsent(columnName, name -> { - ColumnHolder holder = index.getColumn(name); + ColumnHolder holder = index.getColumnHolder(name); if (holder != null) { return closer.register(holder.getColumn()); } else { diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java index 617698540173..f85894c7e0a4 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java @@ -98,7 +98,7 @@ public List getMetricNames() @Override public > CloseableIndexed getDimValueLookup(String dimension) { - final ColumnHolder columnHolder = input.getColumn(dimension); + final ColumnHolder columnHolder = input.getColumnHolder(dimension); if (columnHolder == null) { return null; @@ -236,7 +236,7 @@ class RowIteratorImpl implements TransformableRowIterator .toArray(SettableColumnValueSelector[]::new); rowMetricSelectors = metricNames .stream() - .map(metric -> input.getColumn(metric).makeNewSettableColumnValueSelector()) + .map(metric -> input.getColumnHolder(metric).makeNewSettableColumnValueSelector()) .toArray(SettableColumnValueSelector[]::new); rowPointer = new RowPointer( @@ -255,7 +255,7 @@ class RowIteratorImpl implements TransformableRowIterator .toArray(SettableColumnValueSelector[]::new); markedMetricSelectors = metricNames .stream() - .map(metric -> input.getColumn(metric).makeNewSettableColumnValueSelector()) + .map(metric -> input.getColumnHolder(metric).makeNewSettableColumnValueSelector()) .toArray(SettableColumnValueSelector[]::new); markedRowPointer = new TimeAndDimsPointer( markedTimestampSelector, @@ -359,7 +359,7 @@ void resetToMemoizedOffset() @Override public String getMetricType(String metric) { - final ColumnHolder columnHolder = input.getColumn(metric); + final ColumnHolder columnHolder = input.getColumnHolder(metric); final ValueType type = columnHolder.getCapabilities().getType(); switch (type) { @@ -382,13 +382,13 @@ public String getMetricType(String metric) @Override public ColumnCapabilities getCapabilities(String column) { - return input.getColumn(column).getCapabilities(); + return input.getColumnHolder(column).getCapabilities(); } @Override public BitmapValues getBitmapValues(String dimension, int dictId) { - final ColumnHolder columnHolder = input.getColumn(dimension); + final ColumnHolder columnHolder = input.getColumnHolder(dimension); if (columnHolder == null) { return BitmapValues.EMPTY; } @@ -408,7 +408,7 @@ public BitmapValues getBitmapValues(String dimension, int dictId) @VisibleForTesting BitmapValues getBitmapIndex(String dimension, String value) { - final ColumnHolder columnHolder = input.getColumn(dimension); + final ColumnHolder columnHolder = input.getColumnHolder(dimension); if (columnHolder == null) { return BitmapValues.EMPTY; diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index 947d4678343a..c08e16a9f48e 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -100,7 +100,7 @@ public Iterable getAvailableMetrics() @Override public int getDimensionCardinality(String dimension) { - ColumnHolder columnHolder = index.getColumn(dimension); + ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder == null) { return 0; } @@ -124,7 +124,7 @@ public int getNumRows() @Override public DateTime getMinTime() { - try (final NumericColumn column = (NumericColumn) index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { + try (final NumericColumn column = (NumericColumn) index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { return DateTimes.utc(column.getLongSingleValueRow(0)); } } @@ -132,7 +132,7 @@ public DateTime getMinTime() @Override public DateTime getMaxTime() { - try (final NumericColumn column = (NumericColumn) index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { + try (final NumericColumn column = (NumericColumn) index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn()) { return DateTimes.utc(column.getLongSingleValueRow(column.length() - 1)); } } @@ -141,7 +141,7 @@ public DateTime getMaxTime() @Nullable public Comparable getMinValue(String dimension) { - ColumnHolder columnHolder = index.getColumn(dimension); + ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(0) : null; @@ -153,7 +153,7 @@ public Comparable getMinValue(String dimension) @Nullable public Comparable getMaxValue(String dimension) { - ColumnHolder columnHolder = index.getColumn(dimension); + ColumnHolder columnHolder = index.getColumnHolder(dimension); if (columnHolder != null && columnHolder.getCapabilities().hasBitmapIndexes()) { BitmapIndex bitmap = columnHolder.getBitmapIndex(); return bitmap.getCardinality() > 0 ? bitmap.getValue(bitmap.getCardinality() - 1) : null; @@ -177,7 +177,7 @@ public ColumnCapabilities getColumnCapabilities(String column) @Override public String getColumnTypeName(String columnName) { - final ColumnHolder columnHolder = index.getColumn(columnName); + final ColumnHolder columnHolder = index.getColumnHolder(columnName); try (final BaseColumn col = columnHolder.getColumn()) { if (col instanceof ComplexColumn) { return ((ComplexColumn) col).getTypeName(); @@ -329,7 +329,7 @@ public Sequence makeCursors( @Nullable static ColumnCapabilities getColumnCapabilities(ColumnSelector index, String columnName) { - ColumnHolder columnHolder = index.getColumn(columnName); + ColumnHolder columnHolder = index.getColumnHolder(columnName); if (columnHolder == null) { return null; } @@ -382,7 +382,7 @@ public Sequence build() // Column caches shared amongst all cursors in this sequence. final Map columnCache = new HashMap<>(); - final NumericColumn timestamps = (NumericColumn) index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getColumn(); + final NumericColumn timestamps = (NumericColumn) index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn(); final Closer closer = Closer.create(); closer.register(timestamps); diff --git a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java index bfb8ce70e826..6901124bed46 100644 --- a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java @@ -134,7 +134,7 @@ public BitmapFactory getBitmapFactoryForDimensions() @Nullable @Override - public ColumnHolder getColumn(String columnName) + public ColumnHolder getColumnHolder(String columnName) { return columns.get(columnName); } @@ -172,7 +172,7 @@ public Map getDimensionHandlers() private void initDimensionHandlers() { for (String dim : availableDimensions) { - ColumnCapabilities capabilities = getColumn(dim).getCapabilities(); + ColumnCapabilities capabilities = getColumnHolder(dim).getCapabilities(); DimensionHandler handler = DimensionHandlerUtils.getHandlerFromCapabilities(dim, capabilities, null); dimensionHandlers.put(dim, handler); } diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java index 85f6c2eab14f..69201b846fe3 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java @@ -138,6 +138,7 @@ public int compare(ByteBuffer o1, ByteBuffer o2) private final int fileSizeLimit; private final byte[] fileNameByteArray; private boolean objectsSorted = true; + @Nullable private T prevObject = null; private WriteOutBytes headerOut = null; private WriteOutBytes valuesOut = null; @@ -207,7 +208,7 @@ public void setObjectsNotSorted() objectsSorted = false; } - public void write(T objectToWrite) throws IOException + public void write(@Nullable T objectToWrite) throws IOException { if (objectsSorted && prevObject != null && strategy.compare(prevObject, objectToWrite) >= 0) { objectsSorted = false; diff --git a/processing/src/main/java/io/druid/segment/filter/Filters.java b/processing/src/main/java/io/druid/segment/filter/Filters.java index 301df04a5334..dac579bea5bc 100644 --- a/processing/src/main/java/io/druid/segment/filter/Filters.java +++ b/processing/src/main/java/io/druid/segment/filter/Filters.java @@ -453,7 +453,7 @@ static boolean supportsSelectivityEstimation( ) { if (filter.supportsBitmapIndex(indexSelector)) { - final ColumnHolder columnHolder = columnSelector.getColumn(dimension); + final ColumnHolder columnHolder = columnSelector.getColumnHolder(dimension); if (columnHolder != null) { return !columnHolder.getCapabilities().hasMultipleValues(); } diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index bc8a0675e644..9819b1387fc0 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -107,8 +107,8 @@ public Object fromBytes(byte[] data, int start, int numBytes) /** * This method provides the ability for a ComplexMetricSerde to control its own serialization. - * For large column (i.e columns greater than Integer.MAX) use - * (@link LargeColumnSupportedComplexColumnSerializer) + * For large column (i.e columns greater than {@link Integer#MAX_VALUE}) use + * {@link LargeColumnSupportedComplexColumnSerializer} * * @return an instance of ColumnSerializer used for serialization. */ diff --git a/processing/src/test/java/io/druid/segment/EmptyIndexTest.java b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java index 13c76b91d02b..4c55c8de81fb 100644 --- a/processing/src/test/java/io/druid/segment/EmptyIndexTest.java +++ b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java @@ -98,7 +98,7 @@ public void testEmptyIndex() throws Exception Assert.assertEquals( "getReadOnlyTimestamps", 0, - emptyQueryableIndex.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength() + emptyQueryableIndex.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength() ); } finally { diff --git a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java index ba57d8e290f0..87d995c739f4 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java @@ -117,7 +117,7 @@ public void testStringColumnNullHandling() throws Exception final File tempDir = temporaryFolder.newFolder(); try (QueryableIndex index = indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))) { - final ColumnHolder columnHolder = index.getColumn("d"); + final ColumnHolder columnHolder = index.getColumnHolder("d"); if (subsetList.stream().allMatch(nullFlavors::contains)) { // all null -> should be missing diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index bcdcfe1e00e6..734dd1a3883d 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -168,7 +168,7 @@ public void testPersist() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); - Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -209,7 +209,7 @@ public void testPersistWithDifferentDims() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); - Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); assertDimCompression(index, indexSpec.getDimensionCompression()); @@ -245,7 +245,7 @@ public void testPersistWithSegmentMetadata() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null)) ); - Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -299,7 +299,7 @@ public void testPersistMerge() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist1, tempDir1, indexSpec, null)) ); - Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(3, index1.getColumnNames().size()); @@ -307,7 +307,7 @@ public void testPersistMerge() throws Exception indexIO.loadIndex(indexMerger.persist(toPersist2, tempDir2, indexSpec, null)) ); - Assert.assertEquals(2, index2.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index2.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index2.getAvailableDimensions())); Assert.assertEquals(3, index2.getColumnNames().size()); @@ -327,7 +327,7 @@ public void testPersistMerge() throws Exception ) ); - Assert.assertEquals(3, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(3, merged.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); assertDimCompression(index2, indexSpec.getDimensionCompression()); @@ -392,13 +392,13 @@ public void testPersistEmptyColumn() throws Exception ) ); - Assert.assertEquals(1, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(1, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(index1.getAvailableDimensions())); - Assert.assertEquals(1, index2.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(1, index2.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(index2.getAvailableDimensions())); - Assert.assertEquals(2, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, merged.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(ImmutableList.of("dim2"), ImmutableList.copyOf(merged.getAvailableDimensions())); assertDimCompression(index1, indexSpec.getDimensionCompression()); @@ -431,7 +431,7 @@ public void testMergeRetainsValues() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(3, index1.getColumnNames().size()); @@ -449,7 +449,7 @@ public void testMergeRetainsValues() throws Exception ) ); - Assert.assertEquals(2, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, merged.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); @@ -482,7 +482,7 @@ public void testAppendRetainsValues() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(3, index1.getColumnNames().size()); @@ -505,7 +505,7 @@ public void testAppendRetainsValues() throws Exception ) ); - Assert.assertEquals(2, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, merged.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); @@ -545,7 +545,7 @@ public void testMergeSpecChange() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(3, index1.getColumnNames().size()); @@ -576,7 +576,7 @@ public void testMergeSpecChange() throws Exception ) ); - Assert.assertEquals(2, merged.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, merged.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); @@ -619,7 +619,7 @@ public void testConvertSame() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(4, index1.getColumnNames().size()); @@ -628,7 +628,7 @@ public void testConvertSame() throws Exception indexIO.loadIndex(indexMerger.convert(tempDir1, convertDir, indexSpec)) ); - Assert.assertEquals(2, converted.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, converted.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(converted.getAvailableDimensions())); Assert.assertEquals(4, converted.getColumnNames().size()); @@ -677,7 +677,7 @@ public void testConvertDifferent() throws Exception indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); - Assert.assertEquals(2, index1.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index1.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); Assert.assertEquals(4, index1.getColumnNames().size()); @@ -699,7 +699,7 @@ public void testConvertDifferent() throws Exception indexIO.loadIndex(indexMerger.convert(tempDir1, convertDir, newSpec)) ); - Assert.assertEquals(2, converted.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, converted.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(converted.getAvailableDimensions())); Assert.assertEquals(4, converted.getColumnNames().size()); @@ -722,7 +722,7 @@ private void assertDimCompression(QueryableIndex index, CompressionStrategy expe return; } - DictionaryEncodedColumn encodedColumn = (DictionaryEncodedColumn) index.getColumn("dim2").getColumn(); + DictionaryEncodedColumn encodedColumn = (DictionaryEncodedColumn) index.getColumnHolder("dim2").getColumn(); Object obj; if (encodedColumn.hasMultipleValues()) { Field field = StringDictionaryEncodedColumn.class.getDeclaredField("multiValueColumn"); @@ -2015,7 +2015,7 @@ public void testMultiValueHandling() throws Exception adapter = new QueryableIndexIndexableAdapter(index); rowList = RowIteratorHelper.toList(adapter.getRows()); - Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -2047,7 +2047,7 @@ public void testMultiValueHandling() throws Exception schema = makeDimensionSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.SORTED_SET); index = persistAndLoad(schema, rows); - Assert.assertEquals(1, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(1, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -2078,7 +2078,7 @@ public void testMultiValueHandling() throws Exception schema = makeDimensionSchemas(Arrays.asList("dim1", "dim2"), MultiValueHandling.ARRAY); index = persistAndLoad(schema, rows); - Assert.assertEquals(2, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(2, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions())); Assert.assertEquals(3, index.getColumnNames().size()); @@ -2135,7 +2135,7 @@ public void testDimensionWithEmptyName() throws Exception ) ); - Assert.assertEquals(3, index.getColumn(ColumnHolder.TIME_COLUMN_NAME).getLength()); + Assert.assertEquals(3, index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getLength()); Assert.assertEquals( Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()) diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java b/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java index 6e55cb7e07d0..c119b3880d83 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java @@ -356,7 +356,7 @@ private FireHydrant makeNewCurrIndex(long minTimestamp, DataSchema schema) QueryableIndex oldIndex = segment.asQueryableIndex(); for (String dim : oldIndex.getAvailableDimensions()) { dimOrder.add(dim); - oldCapabilities.put(dim, (ColumnCapabilitiesImpl) oldIndex.getColumn(dim).getCapabilities()); + oldCapabilities.put(dim, (ColumnCapabilitiesImpl) oldIndex.getColumnHolder(dim).getCapabilities()); } } finally { diff --git a/services/src/main/java/io/druid/cli/DumpSegment.java b/services/src/main/java/io/druid/cli/DumpSegment.java index 136ad2f22025..42d93e24bb18 100644 --- a/services/src/main/java/io/druid/cli/DumpSegment.java +++ b/services/src/main/java/io/druid/cli/DumpSegment.java @@ -350,7 +350,7 @@ public Object apply(final OutputStream out) jg.writeStartObject(); for (final String columnName : columnNames) { - final ColumnHolder columnHolder = index.getColumn(columnName); + final ColumnHolder columnHolder = index.getColumnHolder(columnName); final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex(); if (bitmapIndex == null) { @@ -404,7 +404,7 @@ private List getColumnsToInclude(final QueryableIndex index) } else { // Remove any provided columns that do not exist in this segment. for (String columnName : ImmutableList.copyOf(columnNames)) { - if (index.getColumn(columnName) == null) { + if (index.getColumnHolder(columnName) == null) { columnNames.remove(columnName); } } From 13383f317752f0e4b3e724a04d7116c1155898eb Mon Sep 17 00:00:00 2001 From: leventov Date: Sat, 21 Jul 2018 17:12:26 -0500 Subject: [PATCH 12/29] Removed unused getClazz() methods; Fixed a performance bug in ObjectStrategy.writeTo() and some potential nullability issues --- .../benchmark/GenericIndexedBenchmark.java | 4 +- .../DoublesSketchObjectStrategy.java | 12 ++- .../ApproximateHistogramFoldingSerde.java | 2 +- .../aggregation/variance/VarianceSerde.java | 2 +- .../common/task/CompactionTaskTest.java | 2 +- .../hyperloglog/HyperUniquesSerde.java | 2 +- .../query/timeboundary/TimeBoundaryQuery.java | 3 +- .../ColumnSelectorBitmapIndexSelector.java | 5 -- .../QueryableIndexIndexableAdapter.java | 5 -- .../druid/segment/StringDimensionIndexer.java | 5 -- .../segment/StringDimensionMergerV9.java | 5 +- .../druid/segment/column/ComplexColumn.java | 7 +- .../column/DictionaryEncodedColumn.java | 1 - .../column/StringDictionaryEncodedColumn.java | 6 -- .../io/druid/segment/data/ArrayIndexed.java | 78 ------------------- .../io/druid/segment/data/CachingIndexed.java | 6 -- ...pressedVSizeColumnarMultiIntsSupplier.java | 6 -- ...DecompressingByteBufferObjectStrategy.java | 2 +- .../io/druid/segment/data/GenericIndexed.java | 21 +++-- .../data/ImmutableRTreeObjectStrategy.java | 2 +- .../java/io/druid/segment/data/Indexed.java | 1 - .../io/druid/segment/data/ListIndexed.java | 17 ++-- .../io/druid/segment/data/ObjectStrategy.java | 2 +- .../segment/data/VSizeColumnarMultiInts.java | 6 -- .../IncrementalIndexStorageAdapter.java | 2 +- .../segment/serde/ComplexMetricSerde.java | 7 +- .../segment/DictionaryMergeIteratorTest.java | 12 +-- .../filter/ExtractionDimFilterTest.java | 13 +--- .../serde/HyperUniquesSerdeForTest.java | 3 +- .../main/java/io/druid/cli/DumpSegment.java | 5 +- 30 files changed, 60 insertions(+), 184 deletions(-) delete mode 100644 processing/src/main/java/io/druid/segment/data/ArrayIndexed.java diff --git a/benchmarks/src/main/java/io/druid/benchmark/GenericIndexedBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/GenericIndexedBenchmark.java index b2ab25b989a6..3450ccf0dbd0 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/GenericIndexedBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/GenericIndexedBenchmark.java @@ -23,10 +23,10 @@ import com.google.common.primitives.Ints; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; -import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.GenericIndexedWriter; import io.druid.segment.data.ObjectStrategy; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -65,7 +65,7 @@ public class GenericIndexedBenchmark static final ObjectStrategy byteArrayStrategy = new ObjectStrategy() { @Override - public Class getClazz() + public Class getClazz() { return byte[].class; } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java index 9e2af155a98f..817c8ba7a8fe 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java @@ -19,18 +19,16 @@ package io.druid.query.aggregation.datasketches.quantiles; -import java.nio.ByteBuffer; - import com.yahoo.memory.Memory; import com.yahoo.sketches.quantiles.DoublesSketch; - import io.druid.segment.data.ObjectStrategy; +import it.unimi.dsi.fastutil.bytes.ByteArrays; + +import java.nio.ByteBuffer; public class DoublesSketchObjectStrategy implements ObjectStrategy { - private static final byte[] EMPTY_BYTES = new byte[] {}; - @Override public int compare(final DoublesSketch s1, final DoublesSketch s2) { @@ -47,7 +45,7 @@ public DoublesSketch fromByteBuffer(final ByteBuffer buffer, final int numBytes) } @Override - public Class getClazz() + public Class getClazz() { return DoublesSketch.class; } @@ -56,7 +54,7 @@ public Class getClazz() public byte[] toBytes(final DoublesSketch sketch) { if (sketch == null || sketch.isEmpty()) { - return EMPTY_BYTES; + return ByteArrays.EMPTY_ARRAY; } return sketch.toByteArray(true); } diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java index 1c11652162c4..623e09d14370 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java @@ -103,7 +103,7 @@ public ObjectStrategy getObjectStrategy() return new ObjectStrategy() { @Override - public Class getClazz() + public Class getClazz() { return ApproximateHistogram.class; } diff --git a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java index d5a4415d8994..20ab37537038 100644 --- a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java +++ b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java @@ -93,7 +93,7 @@ public ObjectStrategy getObjectStrategy() return new ObjectStrategy() { @Override - public Class getClazz() + public Class getClazz() { return VarianceAggregatorCollector.class; } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java index 630c445a17bf..9feb8658829f 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java @@ -649,7 +649,7 @@ private static class TestIndexIO extends IndexIO entry.getValue(), new SimpleQueryableIndex( segment.getInterval(), - new ListIndexed<>(segment.getDimensions(), String.class), + new ListIndexed<>(segment.getDimensions()), null, columnMap, null, diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java index b8498ceeedb2..86093978d99f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java @@ -104,7 +104,7 @@ public ObjectStrategy getObjectStrategy() return new ObjectStrategy() { @Override - public Class getClazz() + public Class getClazz() { return HyperLogLogCollector.class; } diff --git a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java index 41eda3421338..1eed48b22ea5 100644 --- a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java @@ -34,6 +34,7 @@ import io.druid.query.filter.DimFilter; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.spec.QuerySegmentSpec; +import it.unimi.dsi.fastutil.bytes.ByteArrays; import org.joda.time.DateTime; import java.nio.ByteBuffer; @@ -114,7 +115,7 @@ public Query> withDataSource(DataSource dataSour public byte[] getCacheKey() { - final byte[] filterBytes = dimFilter == null ? new byte[] {} : dimFilter.getCacheKey(); + final byte[] filterBytes = dimFilter == null ? ByteArrays.EMPTY_ARRAY : dimFilter.getCacheKey(); final byte[] boundBytes = StringUtils.toUtf8(bound); final byte delimiter = (byte) 0xff; return ByteBuffer.allocate(2 + boundBytes.length + filterBytes.length) diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index 2ddc43d2594b..3f0b95ee3c7d 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -78,11 +78,6 @@ public CloseableIndexed getDimensionValues(String dimension) final DictionaryEncodedColumn column = (DictionaryEncodedColumn) col; return new CloseableIndexed() { - @Override - public Class getClazz() - { - return String.class; - } @Override public int size() diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java index f85894c7e0a4..57c3b669b1b5 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java @@ -115,11 +115,6 @@ public > CloseableIndexed getDimValueLookup(S return new CloseableIndexed() { - @Override - public Class getClazz() - { - return dict.getClazz(); - } @Override public int size() diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index dc4e94552c49..b60770f84c80 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -328,11 +328,6 @@ public CloseableIndexed getSortedIndexedValues() { return new CloseableIndexed() { - @Override - public Class getClazz() - { - return String.class; - } @Override public int size() diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index 06c9077fd3b9..237560f16406 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -38,7 +38,6 @@ import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.ArrayIndexed; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.BitmapValues; import io.druid.segment.data.ByteBufferWriter; @@ -52,6 +51,7 @@ import io.druid.segment.data.ImmutableRTreeObjectStrategy; import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.ListIndexed; import io.druid.segment.data.SingleValueColumnarIntsSerializer; import io.druid.segment.data.V3CompressedVSizeColumnarMultiIntsSerializer; import io.druid.segment.data.VSizeColumnarIntsSerializer; @@ -67,13 +67,14 @@ import java.io.UncheckedIOException; import java.nio.IntBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class StringDimensionMergerV9 implements DimensionMergerV9 { private static final Logger log = new Logger(StringDimensionMergerV9.class); - private static final Indexed NULL_STR_DIM_VAL = new ArrayIndexed<>(new String[]{null}, String.class); + private static final Indexed NULL_STR_DIM_VAL = new ListIndexed<>(Collections.singletonList(null)); private static final Splitter SPLITTER = Splitter.on(","); private ColumnarIntsSerializer encodedValueSerializer; diff --git a/processing/src/main/java/io/druid/segment/column/ComplexColumn.java b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java index a1d42235c374..2c1ed0c61edf 100644 --- a/processing/src/main/java/io/druid/segment/column/ComplexColumn.java +++ b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java @@ -40,11 +40,6 @@ public ComplexColumn(String typeName, GenericIndexed index) this.typeName = typeName; } - public Class getClazz() - { - return index.getClazz(); - } - public String getTypeName() { return typeName; @@ -76,7 +71,7 @@ public Object getObject() @Override public Class classOfObject() { - return getClazz(); + return index.getClazz(); } @Override diff --git a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java index ed052f158b40..89db29e11390 100644 --- a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java @@ -31,7 +31,6 @@ */ public interface DictionaryEncodedColumn> extends BaseColumn { - Class getClazz(); int length(); boolean hasMultipleValues(); int getSingleValueRow(int rowNum); diff --git a/processing/src/main/java/io/druid/segment/column/StringDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/StringDictionaryEncodedColumn.java index 622c44866ad1..bb3a95270766 100644 --- a/processing/src/main/java/io/druid/segment/column/StringDictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/StringDictionaryEncodedColumn.java @@ -61,12 +61,6 @@ public StringDictionaryEncodedColumn( this.cachedLookups = cachedLookups; } - @Override - public Class getClazz() - { - return String.class; - } - @Override public int length() { diff --git a/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java b/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java deleted file mode 100644 index 718f9bcccdc3..000000000000 --- a/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - -import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; - -import java.util.Arrays; -import java.util.Iterator; - -/** - */ -public class ArrayIndexed implements Indexed -{ - private final T[] baseArray; - private final Class clazz; - - public ArrayIndexed( - T[] baseArray, - Class clazz - ) - { - this.baseArray = baseArray; - this.clazz = clazz; - } - - @Override - public Class getClazz() - { - return clazz; - } - - @Override - public int size() - { - return baseArray.length; - } - - @Override - public T get(int index) - { - return baseArray[index]; - } - - @Override - public int indexOf(T value) - { - return Arrays.asList(baseArray).indexOf(value); - } - - @Override - public Iterator iterator() - { - return Arrays.asList(baseArray).iterator(); - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - // nothing to inspect - } -} diff --git a/processing/src/main/java/io/druid/segment/data/CachingIndexed.java b/processing/src/main/java/io/druid/segment/data/CachingIndexed.java index 14175d7edc71..73aba1365660 100644 --- a/processing/src/main/java/io/druid/segment/data/CachingIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/CachingIndexed.java @@ -60,12 +60,6 @@ public CachingIndexed(GenericIndexed delegate, final int lookupCacheSize) } } - @Override - public Class getClazz() - { - return delegate.getClazz(); - } - @Override public int size() { diff --git a/processing/src/main/java/io/druid/segment/data/CompressedVSizeColumnarMultiIntsSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedVSizeColumnarMultiIntsSupplier.java index c6e35f759c56..9aecc0fcaf3a 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedVSizeColumnarMultiIntsSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedVSizeColumnarMultiIntsSupplier.java @@ -163,12 +163,6 @@ public void close() throws IOException values.close(); } - @Override - public Class getClazz() - { - return IndexedInts.class; - } - @Override public int size() { diff --git a/processing/src/main/java/io/druid/segment/data/DecompressingByteBufferObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/DecompressingByteBufferObjectStrategy.java index 15f4c2692526..8f5a29b2d061 100644 --- a/processing/src/main/java/io/druid/segment/data/DecompressingByteBufferObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/DecompressingByteBufferObjectStrategy.java @@ -38,7 +38,7 @@ public class DecompressingByteBufferObjectStrategy implements ObjectStrategy> getClazz() + public Class> getClazz() { return (Class) ResourceHolder.class; } diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index c43479941586..975472cd4e5c 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -272,7 +272,6 @@ private void checkIndex(int index) } } - @Override public Class getClazz() { return strategy.getClazz(); @@ -300,12 +299,12 @@ public T get(int index) * @return index of value, or negative number equal to (-(insertion point) - 1). */ @Override - public int indexOf(T value) + public int indexOf(@Nullable T value) { return indexOf(this, value); } - private int indexOf(Indexed indexed, T value) + private int indexOf(Indexed indexed, @Nullable T value) { if (!allowReverseLookup) { throw new UnsupportedOperationException("Reverse lookup not allowed."); @@ -368,6 +367,7 @@ public GenericIndexed.BufferIndexed singleThreaded() return versionOne ? singleThreadedVersionOne() : singleThreadedVersionTwo(); } + @Nullable private T copyBufferAndGet(ByteBuffer valueBuffer, int startOffset, int endOffset) { ByteBuffer copyValueBuffer = valueBuffer.asReadOnlyBuffer(); @@ -417,18 +417,13 @@ abstract class BufferIndexed implements Indexed { int lastReadSize; - @Override - public Class getClazz() - { - return strategy.getClazz(); - } - @Override public int size() { return size; } + @Nullable T bufferedIndexedGet(ByteBuffer copyValueBuffer, int startOffset, int endOffset) { int size = endOffset - startOffset; @@ -460,7 +455,7 @@ int getLastValueSize() } @Override - public int indexOf(T value) + public int indexOf(@Nullable T value) { return GenericIndexed.this.indexOf(this, value); } @@ -524,9 +519,11 @@ private static GenericIndexed fromIterableVersionOne( allowReverseLookup = false; } - valuesOut.writeInt(next == null ? NULL_VALUE_SIZE_MARKER : 0); if (next != null) { + valuesOut.writeInt(0); strategy.writeTo(next, valuesOut); + } else { + valuesOut.writeInt(NULL_VALUE_SIZE_MARKER); } headerOut.writeInt(Ints.checkedCast(valuesOut.size())); @@ -559,6 +556,7 @@ private long getSerializedSizeVersionOne() return metaSerdeHelper.size(this) + (long) theBuffer.remaining(); } + @Nullable private T getVersionOne(int index) { checkIndex(index); @@ -660,6 +658,7 @@ private static GenericIndexed createGenericIndexedVersionTwo( } } + @Nullable private T getVersionTwo(int index) { checkIndex(index); diff --git a/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java index 95c986318032..289e2c9a88a8 100644 --- a/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java @@ -57,7 +57,7 @@ public ImmutableRTreeObjectStrategy(BitmapFactory bitmapFactory) } @Override - public Class getClazz() + public Class getClazz() { return ImmutableRTree.class; } diff --git a/processing/src/main/java/io/druid/segment/data/Indexed.java b/processing/src/main/java/io/druid/segment/data/Indexed.java index b957638897e2..07b9fb004fe3 100644 --- a/processing/src/main/java/io/druid/segment/data/Indexed.java +++ b/processing/src/main/java/io/druid/segment/data/Indexed.java @@ -28,7 +28,6 @@ @PublicApi public interface Indexed extends Iterable, HotLoopCallee { - Class getClazz(); int size(); diff --git a/processing/src/main/java/io/druid/segment/data/ListIndexed.java b/processing/src/main/java/io/druid/segment/data/ListIndexed.java index 3da19f8dfd7a..4b339f4d5680 100644 --- a/processing/src/main/java/io/druid/segment/data/ListIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/ListIndexed.java @@ -21,6 +21,8 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import javax.annotation.Nullable; +import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -29,21 +31,16 @@ public class ListIndexed implements Indexed { private final List baseList; - private final Class clazz; - public ListIndexed( - List baseList, - Class clazz - ) + public ListIndexed(List baseList) { this.baseList = baseList; - this.clazz = clazz; } - @Override - public Class getClazz() + @SafeVarargs + public ListIndexed(T... values) { - return clazz; + this(Arrays.asList(values)); } @Override @@ -59,7 +56,7 @@ public T get(int index) } @Override - public int indexOf(T value) + public int indexOf(@Nullable T value) { return baseList.indexOf(value); } diff --git a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java index d53504a5e4ac..adadfe962344 100644 --- a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java @@ -68,7 +68,7 @@ default void writeTo(T val, WriteOutBytes out) throws IOException { byte[] bytes = toBytes(val); if (bytes != null) { - out.write(toBytes(val)); + out.write(bytes); } } } diff --git a/processing/src/main/java/io/druid/segment/data/VSizeColumnarMultiInts.java b/processing/src/main/java/io/druid/segment/data/VSizeColumnarMultiInts.java index 87c52637fc1c..ce427c983df6 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeColumnarMultiInts.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeColumnarMultiInts.java @@ -108,12 +108,6 @@ public static VSizeColumnarMultiInts fromIterable(Iterable ob bufferBytes = 4 - numBytes; } - @Override - public Class getClazz() - { - return VSizeColumnarInts.class; - } - @Override public int size() { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index f7b9a27677a7..e01943cec0db 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -74,7 +74,7 @@ public Interval getInterval() @Override public Indexed getAvailableDimensions() { - return new ListIndexed<>(index.getDimensionNames(), String.class); + return new ListIndexed<>(index.getDimensionNames()); } @Override diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index 9819b1387fc0..74540d4daaf2 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -84,7 +84,12 @@ public Function inputSizeFn() */ public byte[] toBytes(@Nullable Object val) { - return val != null ? getObjectStrategy().toBytes(val) : ByteArrays.EMPTY_ARRAY; + if (val != null) { + byte[] bytes = getObjectStrategy().toBytes(val); + return bytes != null ? bytes : ByteArrays.EMPTY_ARRAY; + } else { + return ByteArrays.EMPTY_ARRAY; + } } /** diff --git a/processing/src/test/java/io/druid/segment/DictionaryMergeIteratorTest.java b/processing/src/test/java/io/druid/segment/DictionaryMergeIteratorTest.java index 630b1056774b..7df7c3ef65ed 100644 --- a/processing/src/test/java/io/druid/segment/DictionaryMergeIteratorTest.java +++ b/processing/src/test/java/io/druid/segment/DictionaryMergeIteratorTest.java @@ -20,8 +20,8 @@ package io.druid.segment; import com.google.common.collect.Iterators; -import io.druid.segment.data.ArrayIndexed; import io.druid.segment.data.Indexed; +import io.druid.segment.data.ListIndexed; import org.junit.Assert; import org.junit.Test; @@ -39,11 +39,11 @@ public void basicTest() String[] s3 = {"a", "d", "f"}; // 0 3 5 String[] s4 = {"a", "b", "c"}; String[] s5 = {"a", "b", "c", "d", "e", "f"}; - Indexed i1 = new ArrayIndexed(s1, String.class); - Indexed i2 = new ArrayIndexed(s2, String.class); - Indexed i3 = new ArrayIndexed(s3, String.class); - Indexed i4 = new ArrayIndexed(s4, String.class); - Indexed i5 = new ArrayIndexed(s5, String.class); + Indexed i1 = new ListIndexed(s1); + Indexed i2 = new ListIndexed(s2); + Indexed i3 = new ListIndexed(s3); + Indexed i4 = new ListIndexed(s4); + Indexed i5 = new ListIndexed(s5); IndexMerger.DictionaryMergeIterator iterator = new IndexMerger.DictionaryMergeIterator(new Indexed[]{i1, i2, i3, i4, i5}, false); diff --git a/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java index 135888ba35ee..03e136eefe49 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java @@ -36,11 +36,12 @@ import io.druid.query.filter.SelectorDimFilter; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BitmapIndex; -import io.druid.segment.data.ArrayIndexed; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.CloseableIndexed; import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.ListIndexed; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.serde.BitmapIndexColumnPartSupplier; import org.junit.Assert; @@ -49,7 +50,6 @@ import org.junit.runners.Parameterized; import javax.annotation.Nullable; -import java.io.IOException; import java.util.Arrays; import java.util.Iterator; import java.util.Map; @@ -101,14 +101,9 @@ public CloseableIndexed getDimensionValues(String dimension) if (vals == null) { return null; } else { - ArrayIndexed indexed = new ArrayIndexed<>(vals, String.class); + Indexed indexed = new ListIndexed<>(vals); return new CloseableIndexed() { - @Override - public Class getClazz() - { - return indexed.getClazz(); - } @Override public int size() @@ -136,7 +131,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) } @Override - public void close() throws IOException + public void close() { // close nothing } diff --git a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java index 69974dcd97b0..5909b8c9b451 100644 --- a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java +++ b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java @@ -29,6 +29,7 @@ import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; +import it.unimi.dsi.fastutil.bytes.ByteArrays; import java.nio.ByteBuffer; import java.util.List; @@ -135,7 +136,7 @@ public HyperLogLogCollector fromByteBuffer(ByteBuffer buffer, int numBytes) public byte[] toBytes(HyperLogLogCollector collector) { if (collector == null) { - return new byte[]{}; + return ByteArrays.EMPTY_ARRAY; } ByteBuffer val = collector.toByteBuffer(); byte[] retVal = new byte[val.remaining()]; diff --git a/services/src/main/java/io/druid/cli/DumpSegment.java b/services/src/main/java/io/druid/cli/DumpSegment.java index 42d93e24bb18..c2627a22acc7 100644 --- a/services/src/main/java/io/druid/cli/DumpSegment.java +++ b/services/src/main/java/io/druid/cli/DumpSegment.java @@ -371,7 +371,10 @@ public Object apply(final OutputStream out) } jg.writeEndArray(); } else { - jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap)); + byte[] bytes = bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap); + if (bytes != null) { + jg.writeBinary(bytes); + } } } } From 3eab0a882313ec20954103c53054f2d7ccb4f38d Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 3 Aug 2018 11:18:02 -0500 Subject: [PATCH 13/29] Fix SingleStringInputDimensionSelector --- .../segment/virtual/SingleStringInputDimensionSelector.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputDimensionSelector.java b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputDimensionSelector.java index 3524bb1842e4..33408f9c43d9 100644 --- a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputDimensionSelector.java @@ -139,7 +139,7 @@ public String lookupName(final int id) @Override public boolean nameLookupPossibleInAdvance() { - return selector.nameLookupPossibleInAdvance(); + return true; } @Nullable @@ -153,7 +153,7 @@ public IdLookup idLookup() @Override public Object getObject() { - return lookupName(getRow().get(0)); + return defaultGetObject(); } @Override From 4e72cb2bb9fbe23891e447369e50c34e09053bdf Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 3 Aug 2018 12:22:13 -0500 Subject: [PATCH 14/29] Rename ColumnSerializer to GenericColumnSerializer --- .../DoublesSketchComplexMetricSerde.java | 4 +-- .../theta/SketchMergeComplexMetricSerde.java | 4 +-- ...fDoublesSketchMergeComplexMetricSerde.java | 4 +-- .../ApproximateHistogramFoldingSerde.java | 4 +-- .../aggregation/variance/VarianceSerde.java | 4 +-- .../SerializablePairLongStringSerde.java | 4 +-- .../hyperloglog/HyperUniquesSerde.java | 4 +-- .../druid/segment/DoubleColumnSerializer.java | 2 +- .../segment/DoubleColumnSerializerV2.java | 2 +- .../segment/DoubleDimensionMergerV9.java | 2 +- .../druid/segment/FloatColumnSerializer.java | 2 +- .../segment/FloatColumnSerializerV2.java | 2 +- .../druid/segment/FloatDimensionMergerV9.java | 2 +- ...izer.java => GenericColumnSerializer.java} | 7 ++-- .../java/io/druid/segment/IndexMergerV9.java | 36 +++++++++---------- .../druid/segment/LongColumnSerializer.java | 2 +- .../druid/segment/LongColumnSerializerV2.java | 2 +- .../druid/segment/LongDimensionMergerV9.java | 2 +- .../segment/NumericDimensionMergerV9.java | 4 +-- .../segment/serde/ComplexColumnPartSerde.java | 6 ++-- .../serde/ComplexColumnSerializer.java | 4 +-- .../segment/serde/ComplexMetricSerde.java | 6 ++-- ...olumnSupportedComplexColumnSerializer.java | 4 +-- .../serde/HyperUniquesSerdeForTest.java | 4 +-- 24 files changed, 59 insertions(+), 58 deletions(-) rename processing/src/main/java/io/druid/segment/{ColumnSerializer.java => GenericColumnSerializer.java} (83%) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java index f01ca360315e..7de9a8611b4e 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java @@ -24,7 +24,7 @@ import com.yahoo.sketches.quantiles.UpdateDoublesSketch; import io.druid.data.input.InputRow; import io.druid.java.util.common.IAE; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -109,7 +109,7 @@ public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder build // support large columns @Override - public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java index 7ebe7f029a21..a5782c674b10 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java @@ -20,7 +20,7 @@ package io.druid.query.aggregation.datasketches.theta; import io.druid.data.input.InputRow; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -78,7 +78,7 @@ public ObjectStrategy getObjectStrategy() } @Override - public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java index 40bcc8a4ea32..78483bf5f8e2 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchMergeComplexMetricSerde.java @@ -21,7 +21,7 @@ import com.yahoo.sketches.tuple.ArrayOfDoublesSketch; import io.druid.data.input.InputRow; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -79,7 +79,7 @@ public ObjectStrategy getObjectStrategy() } @Override - public ColumnSerializer getSerializer(final SegmentWriteOutMedium segmentWriteOutMedium, final String column) + public GenericColumnSerializer getSerializer(final SegmentWriteOutMedium segmentWriteOutMedium, final String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java index 623e09d14370..7cadb7049d5a 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java @@ -21,7 +21,7 @@ import io.druid.data.input.InputRow; import io.druid.data.input.Rows; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -92,7 +92,7 @@ public void deserializeColumn(ByteBuffer byteBuffer, ColumnBuilder columnBuilder } @Override - public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java index 20ab37537038..6448b4b982b1 100644 --- a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java +++ b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java @@ -21,7 +21,7 @@ import com.google.common.collect.Ordering; import io.druid.data.input.InputRow; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; @@ -120,7 +120,7 @@ public int compare(VarianceAggregatorCollector o1, VarianceAggregatorCollector o } @Override - public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/processing/src/main/java/io/druid/query/aggregation/SerializablePairLongStringSerde.java b/processing/src/main/java/io/druid/query/aggregation/SerializablePairLongStringSerde.java index e71c8405213b..e588c5675312 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SerializablePairLongStringSerde.java +++ b/processing/src/main/java/io/druid/query/aggregation/SerializablePairLongStringSerde.java @@ -22,7 +22,7 @@ import io.druid.data.input.InputRow; import io.druid.java.util.common.StringUtils; import io.druid.query.aggregation.first.StringFirstAggregatorFactory; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -139,7 +139,7 @@ public byte[] toBytes(SerializablePairLongString val) } @Override - public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java index 86093978d99f..4c8f3c79015a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java @@ -22,7 +22,7 @@ import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogCollector; import io.druid.hll.HyperLogLogHash; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.ObjectStrategy; @@ -140,7 +140,7 @@ public int compare(HyperLogLogCollector o1, HyperLogLogCollector o2) } @Override - public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java index dbdb4cf45aa6..ea3dda7f8a01 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java @@ -30,7 +30,7 @@ import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; -public class DoubleColumnSerializer implements ColumnSerializer +public class DoubleColumnSerializer implements GenericColumnSerializer { public static DoubleColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSerializerV2.java b/processing/src/main/java/io/druid/segment/DoubleColumnSerializerV2.java index 97bb037dd0ba..30c61f0511d0 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSerializerV2.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSerializerV2.java @@ -42,7 +42,7 @@ * and second the actual row values. * This class is unsafe for concurrent use from multiple threads. */ -public class DoubleColumnSerializerV2 implements ColumnSerializer +public class DoubleColumnSerializerV2 implements GenericColumnSerializer { public static DoubleColumnSerializerV2 create( SegmentWriteOutMedium segmentWriteOutMedium, diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java index b8127480e6cb..ffffac334381 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java @@ -33,7 +33,7 @@ public class DoubleDimensionMergerV9 extends NumericDimensionMergerV9 } @Override - ColumnSerializer setupEncodedValueWriter() + GenericColumnSerializer setupEncodedValueWriter() { return IndexMergerV9.createDoubleColumnSerializer(segmentWriteOutMedium, dimensionName, indexSpec); } diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java index 3390d9b29470..ae833aefdd92 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java @@ -30,7 +30,7 @@ import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; -public class FloatColumnSerializer implements ColumnSerializer +public class FloatColumnSerializer implements GenericColumnSerializer { public static FloatColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSerializerV2.java b/processing/src/main/java/io/druid/segment/FloatColumnSerializerV2.java index e76c1bf9699a..ccb42320e306 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSerializerV2.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSerializerV2.java @@ -42,7 +42,7 @@ * and second the actual row values. * This class is unsafe for concurrent use from multiple threads. */ -public class FloatColumnSerializerV2 implements ColumnSerializer +public class FloatColumnSerializerV2 implements GenericColumnSerializer { public static FloatColumnSerializerV2 create( SegmentWriteOutMedium segmentWriteOutMedium, diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java index d905522c772f..2f2a4977b278 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java @@ -33,7 +33,7 @@ public class FloatDimensionMergerV9 extends NumericDimensionMergerV9 } @Override - ColumnSerializer setupEncodedValueWriter() + GenericColumnSerializer setupEncodedValueWriter() { return IndexMergerV9.createFloatColumnSerializer(segmentWriteOutMedium, dimensionName, indexSpec); } diff --git a/processing/src/main/java/io/druid/segment/ColumnSerializer.java b/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java similarity index 83% rename from processing/src/main/java/io/druid/segment/ColumnSerializer.java rename to processing/src/main/java/io/druid/segment/GenericColumnSerializer.java index 8fad3f59b9a2..1eacb59424cc 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java @@ -25,13 +25,14 @@ import java.io.IOException; /** - * ColumnSerializer can be implemented in custom aggregator extensions that would like to take full control of column + * GenericColumnSerializer can be implemented in custom aggregator extensions that would like to take full control of column * serialization. That implementation would be returned by overriding {@link * io.druid.segment.serde.ComplexMetricSerde#getSerializer} - * @param + * + * TODO rename to "ColumnSerializer" prior to Druid 1.0 */ @ExtensionPoint -public interface ColumnSerializer extends Serializer +public interface GenericColumnSerializer extends Serializer { void open() throws IOException; diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index c7eab0fef737..ea21377d57a1 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -185,8 +185,8 @@ private File makeIndexFiles( mergers ); closer.register(timeAndDimsIterator); - final ColumnSerializer timeWriter = setupTimeWriter(segmentWriteOutMedium, indexSpec); - final ArrayList metricWriters = + final GenericColumnSerializer timeWriter = setupTimeWriter(segmentWriteOutMedium, indexSpec); + final ArrayList metricWriters = setupMetricsWriters(segmentWriteOutMedium, mergedMetrics, metricsValueTypes, metricTypeNames, indexSpec); List rowNumConversions = mergeIndexesAndWriteColumns( adapters, @@ -321,7 +321,7 @@ private void makeMetricsColumns( final List mergedMetrics, final Map metricsValueTypes, final Map metricTypeNames, - final List metWriters, + final List metWriters, final IndexSpec indexSpec ) throws IOException { @@ -332,7 +332,7 @@ private void makeMetricsColumns( for (int i = 0; i < mergedMetrics.size(); ++i) { String metric = mergedMetrics.get(i); long metricStartTime = System.currentTimeMillis(); - ColumnSerializer writer = metWriters.get(i); + GenericColumnSerializer writer = metWriters.get(i); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); ValueType type = metricsValueTypes.get(metric); @@ -370,7 +370,7 @@ private void makeMetricsColumns( progress.stopSection(section); } - static ColumnPartSerde createLongColumnPartSerde(ColumnSerializer serializer, IndexSpec indexSpec) + static ColumnPartSerde createLongColumnPartSerde(GenericColumnSerializer serializer, IndexSpec indexSpec) { // If using default values for null use LongNumericColumnPartSerde to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { @@ -387,7 +387,7 @@ static ColumnPartSerde createLongColumnPartSerde(ColumnSerializer serializer, In } } - static ColumnPartSerde createDoubleColumnPartSerde(ColumnSerializer serializer, IndexSpec indexSpec) + static ColumnPartSerde createDoubleColumnPartSerde(GenericColumnSerializer serializer, IndexSpec indexSpec) { // If using default values for null use DoubleNumericColumnPartSerde to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { @@ -404,7 +404,7 @@ static ColumnPartSerde createDoubleColumnPartSerde(ColumnSerializer serializer, } } - static ColumnPartSerde createFloatColumnPartSerde(ColumnSerializer serializer, IndexSpec indexSpec) + static ColumnPartSerde createFloatColumnPartSerde(GenericColumnSerializer serializer, IndexSpec indexSpec) { // If using default values for null use FloatNumericColumnPartSerde to allow rollback to previous versions. if (NullHandling.replaceWithDefault()) { @@ -424,7 +424,7 @@ static ColumnPartSerde createFloatColumnPartSerde(ColumnSerializer serializer, I private void makeTimeColumn( final FileSmoosher v9Smoosher, final ProgressIndicator progress, - final ColumnSerializer timeWriter, + final GenericColumnSerializer timeWriter, final IndexSpec indexSpec ) throws IOException { @@ -467,8 +467,8 @@ private List mergeIndexesAndWriteColumns( final List adapters, final ProgressIndicator progress, final TimeAndDimsIterator timeAndDimsIterator, - final ColumnSerializer timeWriter, - final ArrayList metricWriters, + final GenericColumnSerializer timeWriter, + final ArrayList metricWriters, final List mergers, final boolean fillRowNumConversions ) throws IOException @@ -558,10 +558,10 @@ private List mergeIndexesAndWriteColumns( return rowNumConversions; } - private ColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMedium, IndexSpec indexSpec) + private GenericColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMedium, IndexSpec indexSpec) throws IOException { - ColumnSerializer timeWriter = createLongColumnSerializer( + GenericColumnSerializer timeWriter = createLongColumnSerializer( segmentWriteOutMedium, "little_end_time", indexSpec @@ -571,7 +571,7 @@ private ColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMe return timeWriter; } - private ArrayList setupMetricsWriters( + private ArrayList setupMetricsWriters( final SegmentWriteOutMedium segmentWriteOutMedium, final List mergedMetrics, final Map metricsValueTypes, @@ -579,11 +579,11 @@ private ArrayList setupMetricsWriters( final IndexSpec indexSpec ) throws IOException { - ArrayList metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size()); + ArrayList metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size()); for (String metric : mergedMetrics) { ValueType type = metricsValueTypes.get(metric); - ColumnSerializer writer; + GenericColumnSerializer writer; switch (type) { case LONG: writer = createLongColumnSerializer(segmentWriteOutMedium, metric, indexSpec); @@ -612,7 +612,7 @@ private ArrayList setupMetricsWriters( return metWriters; } - static ColumnSerializer createLongColumnSerializer( + static GenericColumnSerializer createLongColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String columnName, IndexSpec indexSpec @@ -637,7 +637,7 @@ static ColumnSerializer createLongColumnSerializer( } } - static ColumnSerializer createDoubleColumnSerializer( + static GenericColumnSerializer createDoubleColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String columnName, IndexSpec indexSpec @@ -660,7 +660,7 @@ static ColumnSerializer createDoubleColumnSerializer( } } - static ColumnSerializer createFloatColumnSerializer( + static GenericColumnSerializer createFloatColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String columnName, IndexSpec indexSpec diff --git a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java index cd1beec6a7fb..86652f1543d2 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java @@ -33,7 +33,7 @@ /** * Unsafe for concurrent use from multiple threads. */ -public class LongColumnSerializer implements ColumnSerializer +public class LongColumnSerializer implements GenericColumnSerializer { public static LongColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, diff --git a/processing/src/main/java/io/druid/segment/LongColumnSerializerV2.java b/processing/src/main/java/io/druid/segment/LongColumnSerializerV2.java index 3f2a658e23ad..24324f849e2a 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSerializerV2.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSerializerV2.java @@ -42,7 +42,7 @@ * and second the actual row values. * This class is unsafe for concurrent use from multiple threads. */ -public class LongColumnSerializerV2 implements ColumnSerializer +public class LongColumnSerializerV2 implements GenericColumnSerializer { public static LongColumnSerializerV2 create( SegmentWriteOutMedium segmentWriteOutMedium, diff --git a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java index aca78c0c133a..e526c5c14bd0 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java @@ -33,7 +33,7 @@ public class LongDimensionMergerV9 extends NumericDimensionMergerV9 } @Override - ColumnSerializer setupEncodedValueWriter() + GenericColumnSerializer setupEncodedValueWriter() { return IndexMergerV9.createLongColumnSerializer(segmentWriteOutMedium, dimensionName, indexSpec); } diff --git a/processing/src/main/java/io/druid/segment/NumericDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/NumericDimensionMergerV9.java index 7a09d809fc86..3144285a850f 100644 --- a/processing/src/main/java/io/druid/segment/NumericDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/NumericDimensionMergerV9.java @@ -35,7 +35,7 @@ public abstract class NumericDimensionMergerV9 implements DimensionMergerV9 protected final IndexSpec indexSpec; protected final SegmentWriteOutMedium segmentWriteOutMedium; - protected final ColumnSerializer serializer; + protected final GenericColumnSerializer serializer; NumericDimensionMergerV9( String dimensionName, @@ -56,7 +56,7 @@ public abstract class NumericDimensionMergerV9 implements DimensionMergerV9 } } - abstract ColumnSerializer setupEncodedValueWriter(); + abstract GenericColumnSerializer setupEncodedValueWriter(); @Override public final void writeMergedValueDictionary(List adapters) diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java index a4199782d0d8..b7f30d0542ce 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; @@ -84,7 +84,7 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo public static class SerializerBuilder { private String typeName = null; - private ColumnSerializer delegate = null; + private GenericColumnSerializer delegate = null; public SerializerBuilder withTypeName(final String typeName) { @@ -92,7 +92,7 @@ public SerializerBuilder withTypeName(final String typeName) return this; } - public SerializerBuilder withDelegate(final ColumnSerializer delegate) + public SerializerBuilder withDelegate(final GenericColumnSerializer delegate) { this.delegate = delegate; return this; diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java index e2e9817f611b..b277a98bec6a 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java @@ -23,7 +23,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.ColumnValueSelector; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.data.GenericIndexedWriter; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.writeout.SegmentWriteOutMedium; @@ -31,7 +31,7 @@ import java.io.IOException; import java.nio.channels.WritableByteChannel; -public class ComplexColumnSerializer implements ColumnSerializer +public class ComplexColumnSerializer implements GenericColumnSerializer { @PublicApi public static ComplexColumnSerializer create(SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy) diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index af1825e14bdf..6f1ec6c50a98 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -21,7 +21,7 @@ import com.google.common.base.Function; import io.druid.guice.annotations.ExtensionPoint; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.writeout.SegmentWriteOutMedium; @@ -115,9 +115,9 @@ public Object fromBytes(byte[] data, int start, int numBytes) * For large column (i.e columns greater than {@link Integer#MAX_VALUE}) use * {@link LargeColumnSupportedComplexColumnSerializer} * - * @return an instance of ColumnSerializer used for serialization. + * @return an instance of GenericColumnSerializer used for serialization. */ - public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return ComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } diff --git a/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java b/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java index 535d51569840..8e6286bc3dd0 100644 --- a/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java @@ -23,7 +23,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.ColumnValueSelector; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.data.GenericIndexedWriter; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.writeout.SegmentWriteOutMedium; @@ -31,7 +31,7 @@ import java.io.IOException; import java.nio.channels.WritableByteChannel; -public class LargeColumnSupportedComplexColumnSerializer implements ColumnSerializer +public class LargeColumnSupportedComplexColumnSerializer implements GenericColumnSerializer { @PublicApi public static LargeColumnSupportedComplexColumnSerializer create( diff --git a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java index 5909b8c9b451..c12d6e4d7e4c 100644 --- a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java +++ b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java @@ -24,7 +24,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogCollector; -import io.druid.segment.ColumnSerializer; +import io.druid.segment.GenericColumnSerializer; import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; @@ -153,7 +153,7 @@ public int compare(HyperLogLogCollector o1, HyperLogLogCollector o2) } @Override - public ColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String metric) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String metric) { return LargeColumnSupportedComplexColumnSerializer.createWithColumnSize( segmentWriteOutMedium, From dac2c4f62d648c978d23620d901a5af65c937142 Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 9 Aug 2018 19:17:39 -0500 Subject: [PATCH 15/29] Address comments --- .idea/inspectionProfiles/Druid.xml | 3 ++ .idea/misc.xml | 15 ++++++-- .../query/aggregation/AggregatorUtil.java | 36 ++++++++++++------- .../first/DoubleFirstAggregatorFactory.java | 15 ++++---- .../first/FloatFirstAggregatorFactory.java | 15 ++++---- .../first/LongFirstAggregatorFactory.java | 12 +++++-- .../last/DoubleLastAggregatorFactory.java | 8 ++++- .../last/FloatLastAggregatorFactory.java | 8 ++++- .../last/LongLastAggregatorFactory.java | 14 ++++---- 9 files changed, 88 insertions(+), 38 deletions(-) diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml index d0029e103fac..253b3e6708bc 100644 --- a/.idea/inspectionProfiles/Druid.xml +++ b/.idea/inspectionProfiles/Druid.xml @@ -13,6 +13,9 @@ + + diff --git a/.idea/misc.xml b/.idea/misc.xml index 846d8ad4f8c7..f194dc231522 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -40,26 +40,35 @@ diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index f6e6d69eedd3..2412c0908461 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -157,6 +157,9 @@ public static Pair, List> condensedAggre return new Pair<>(condensedAggs, condensedPostAggs); } + /** + * Only one of fieldName and fieldExpression should be non-null + */ static BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, @@ -165,10 +168,12 @@ static BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefault( final float nullValue ) { - if (fieldName != null && fieldExpression == null) { - return metricFactory.makeColumnValueSelector(fieldName); + if ((fieldName == null) == (fieldExpression == null)) { + throw new IllegalArgumentException("Only one of fieldName and fieldExpression should be non-null"); } - if (fieldName == null && fieldExpression != null) { + if (fieldName != null) { + return metricFactory.makeColumnValueSelector(fieldName); + } else { final Expr expr = Parser.parse(fieldExpression, macroTable); final ColumnValueSelector baseSelector = ExpressionSelectors.makeExprEvalSelector(metricFactory, expr); class ExpressionFloatColumnSelector implements FloatColumnSelector @@ -197,9 +202,11 @@ public boolean isNull() } return new ExpressionFloatColumnSelector(); } - throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } + /** + * Only one of fieldName and fieldExpression should be non-null + */ static BaseLongColumnValueSelector makeColumnValueSelectorWithLongDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, @@ -208,10 +215,12 @@ static BaseLongColumnValueSelector makeColumnValueSelectorWithLongDefault( final long nullValue ) { - if (fieldName != null && fieldExpression == null) { - return metricFactory.makeColumnValueSelector(fieldName); + if ((fieldName == null) == (fieldExpression == null)) { + throw new IllegalArgumentException("Only one of fieldName and fieldExpression should be non-null"); } - if (fieldName == null && fieldExpression != null) { + if (fieldName != null) { + return metricFactory.makeColumnValueSelector(fieldName); + } else { final Expr expr = Parser.parse(fieldExpression, macroTable); final ColumnValueSelector baseSelector = ExpressionSelectors.makeExprEvalSelector(metricFactory, expr); class ExpressionLongColumnSelector implements LongColumnSelector @@ -238,9 +247,11 @@ public boolean isNull() } return new ExpressionLongColumnSelector(); } - throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } + /** + * Only one of fieldName and fieldExpression should be non-null + */ static BaseDoubleColumnValueSelector makeColumnValueSelectorWithDoubleDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, @@ -249,10 +260,12 @@ static BaseDoubleColumnValueSelector makeColumnValueSelectorWithDoubleDefault( final double nullValue ) { - if (fieldName != null && fieldExpression == null) { - return metricFactory.makeColumnValueSelector(fieldName); + if ((fieldName == null) == (fieldExpression == null)) { + throw new IllegalArgumentException("Only one of fieldName and fieldExpression should be non-null"); } - if (fieldName == null && fieldExpression != null) { + if (fieldName != null) { + return metricFactory.makeColumnValueSelector(fieldName); + } else { final Expr expr = Parser.parse(fieldExpression, macroTable); final ColumnValueSelector baseSelector = ExpressionSelectors.makeExprEvalSelector(metricFactory, expr); class ExpressionDoubleColumnSelector implements DoubleColumnSelector @@ -279,6 +292,5 @@ public boolean isNull() } return new ExpressionDoubleColumnSelector(); } - throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index bc34ac73bf5a..7a723b868f06 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -47,11 +47,8 @@ public class DoubleFirstAggregatorFactory extends NullableAggregatorFactory { - public static final Comparator VALUE_COMPARATOR = - Comparator.comparingDouble(o -> ((SerializablePair) o).rhs); - - public static final Comparator TIME_COMPARATOR = - Comparator.comparingLong(o -> ((SerializablePair) o).lhs); + public static final Comparator> VALUE_COMPARATOR = + Comparator.comparingDouble(o -> o.rhs); private final String fieldName; private final String name; @@ -108,7 +105,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) if (lhs == null) { return rhs; } - return TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; + Long leftTime = ((SerializablePair) lhs).lhs; + Long rightTime = ((SerializablePair) rhs).lhs; + if (leftTime <= rightTime) { + return lhs; + } else { + return rhs; + } } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 258432d4e53b..ed4f0de84281 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -47,11 +47,8 @@ public class FloatFirstAggregatorFactory extends NullableAggregatorFactory { - public static final Comparator VALUE_COMPARATOR = - Comparator.comparingDouble(o -> ((SerializablePair) o).rhs); - - public static final Comparator TIME_COMPARATOR = - Comparator.comparingLong(o -> ((SerializablePair) o).lhs); + public static final Comparator> VALUE_COMPARATOR = + Comparator.comparingDouble(o -> o.rhs); private final String fieldName; private final String name; @@ -106,7 +103,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) if (lhs == null) { return rhs; } - return TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; + Long leftTime = ((SerializablePair) lhs).lhs; + Long rightTime = ((SerializablePair) rhs).lhs; + if (leftTime <= rightTime) { + return lhs; + } else { + return rhs; + } } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index be3bf1b3b2dd..9af2d0283479 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -46,8 +46,8 @@ public class LongFirstAggregatorFactory extends NullableAggregatorFactory { - public static final Comparator VALUE_COMPARATOR = - Comparator.comparingLong(o -> ((SerializablePair) o).rhs); + public static final Comparator> VALUE_COMPARATOR = + Comparator.comparingLong(o -> o.rhs); private final String fieldName; private final String name; @@ -102,7 +102,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) if (rhs == null) { return lhs; } - return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; + Long leftTime = ((SerializablePair) lhs).lhs; + Long rightTime = ((SerializablePair) rhs).lhs; + if (leftTime <= rightTime) { + return lhs; + } else { + return rhs; + } } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 0aa03e8c0dc2..018fd0d6af33 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -104,7 +104,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) if (lhs == null) { return rhs; } - return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; + Long leftTime = ((SerializablePair) lhs).lhs; + Long rightTime = ((SerializablePair) rhs).lhs; + if (leftTime >= rightTime) { + return lhs; + } else { + return rhs; + } } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index f682e5651c6d..2b782856fdc9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -102,7 +102,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) if (lhs == null) { return rhs; } - return FloatFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; + Long leftTime = ((SerializablePair) lhs).lhs; + Long rightTime = ((SerializablePair) rhs).lhs; + if (leftTime >= rightTime) { + return lhs; + } else { + return rhs; + } } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index cc131a0b9a01..e15b213ae3d5 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -31,7 +31,6 @@ import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.NullableAggregatorFactory; -import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelectorFactory; @@ -79,10 +78,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS @Override protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - return new LongLastBufferAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - selector - ); + return new LongLastBufferAggregator(metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), selector); } @Override @@ -101,7 +97,13 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) if (lhs == null) { return rhs; } - return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; + Long leftTime = ((SerializablePair) lhs).lhs; + Long rightTime = ((SerializablePair) rhs).lhs; + if (leftTime >= rightTime) { + return lhs; + } else { + return rhs; + } } @Override From 0359f4d2bff060cd39eabdda7b8906cb8ba6cf57 Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 30 Aug 2018 17:52:02 -0500 Subject: [PATCH 16/29] Comments --- .../druid/segment/QueryableIndexColumnSelectorFactory.java | 2 ++ .../main/java/org/apache/druid/segment/StorageAdapter.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java index aa51b6244bdb..2beca4a0d276 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java @@ -98,6 +98,8 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi if (col instanceof DictionaryEncodedColumn) { return closer.register(col); } else { + // Return null from the lambda in computeIfAbsent() results in no recorded value in the columnCache and + // the column variable is set to null. return null; } }); diff --git a/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java index bcba475aace3..3319d09c0075 100644 --- a/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/StorageAdapter.java @@ -39,8 +39,8 @@ public interface StorageAdapter extends CursorFactory Iterable getAvailableMetrics(); /** - * Returns the number of distinct values for the given column if known, or {@link Integer#MAX_VALUE} is unknown or - * the column is numeric. If the column doesn't exist, returns 0. + * Returns the number of distinct values for the given column if known, or {@link Integer#MAX_VALUE} if unknown, + * e. g. the column is numeric. If the column doesn't exist, returns 0. */ int getDimensionCardinality(String column); DateTime getMinTime(); From 7127ceada082fb2332313869870c7e384775d6bd Mon Sep 17 00:00:00 2001 From: leventov Date: Thu, 30 Aug 2018 23:00:01 -0500 Subject: [PATCH 17/29] Remove DoubleGenericColumnPartSerde --- .../serde/DoubleGenericColumnPartSerde.java | 117 ------------------ 1 file changed, 117 deletions(-) delete mode 100644 processing/src/main/java/org/apache/druid/segment/serde/DoubleGenericColumnPartSerde.java diff --git a/processing/src/main/java/org/apache/druid/segment/serde/DoubleGenericColumnPartSerde.java b/processing/src/main/java/org/apache/druid/segment/serde/DoubleGenericColumnPartSerde.java deleted file mode 100644 index d06c0666f4f3..000000000000 --- a/processing/src/main/java/org/apache/druid/segment/serde/DoubleGenericColumnPartSerde.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.segment.serde; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Supplier; -import org.apache.druid.segment.IndexIO; -import org.apache.druid.segment.column.ColumnBuilder; -import org.apache.druid.segment.column.ColumnConfig; -import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.data.ColumnarDoubles; -import org.apache.druid.segment.data.CompressedColumnarDoublesSuppliers; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -public class DoubleGenericColumnPartSerde implements ColumnPartSerde -{ - @JsonCreator - public static DoubleGenericColumnPartSerde getDoubleGenericColumnPartSerde( - @JsonProperty("byteOrder") ByteOrder byteOrder - ) - { - return new DoubleGenericColumnPartSerde(byteOrder, null); - } - - private final ByteOrder byteOrder; - private final Serializer serializer; - - private DoubleGenericColumnPartSerde(ByteOrder byteOrder, Serializer serializer) - { - this.byteOrder = byteOrder; - this.serializer = serializer; - } - - @JsonProperty - public ByteOrder getByteOrder() - { - return byteOrder; - } - - public static SerializerBuilder serializerBuilder() - { - return new SerializerBuilder(); - } - - public static class SerializerBuilder - { - private ByteOrder byteOrder = null; - private Serializer delegate = null; - - public SerializerBuilder withByteOrder(final ByteOrder byteOrder) - { - this.byteOrder = byteOrder; - return this; - } - - public SerializerBuilder withDelegate(final Serializer delegate) - { - this.delegate = delegate; - return this; - } - - public DoubleGenericColumnPartSerde build() - { - return new DoubleGenericColumnPartSerde(byteOrder, delegate); - } - } - - @Override - public Serializer getSerializer() - { - return serializer; - } - - @Override - public Deserializer getDeserializer() - { - return new Deserializer() - { - @Override - public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) - { - final Supplier column = CompressedColumnarDoublesSuppliers.fromByteBuffer( - buffer, - byteOrder - ); - DoubleNumericColumnSupplier columnSupplier = new DoubleNumericColumnSupplier( - column, - IndexIO.LEGACY_FACTORY.getBitmapFactory().makeEmptyImmutableBitmap() - ); - builder.setType(ValueType.DOUBLE) - .setHasMultipleValues(false) - .setNumericColumnSupplier(columnSupplier); - - } - }; - } -} From 1c4c5fe7565ca20ab4f7e99c5a4c179266cfda64 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 31 Aug 2018 18:24:11 -0500 Subject: [PATCH 18/29] Deadlock watching in CuratorDruidCoordinatorTest --- .../CuratorDruidCoordinatorTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index 870a8aaa397a..d1d925fcbb78 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -65,6 +65,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -102,6 +103,8 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; + private CountDownLatch testFinished; + public CuratorDruidCoordinatorTest() { jsonMapper = TestHelper.makeJsonMapper(); @@ -228,11 +231,29 @@ public void unannounce(DruidNode node) EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector() ); + testFinished = new CountDownLatch(1); + Thread deadLockWatcher = new Thread(() -> { + try { + if (!testFinished.await(50, TimeUnit.SECONDS)) { + Thread.getAllStackTraces().forEach((thread, stackTrace) -> { + Throwable t = new Throwable(thread.getName()); + t.setStackTrace(stackTrace); + t.printStackTrace(); + }); + } + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + deadLockWatcher.setDaemon(true); + deadLockWatcher.start(); } @After public void tearDown() throws Exception { + testFinished.countDown(); baseView.stop(); sourceLoadQueuePeon.stop(); sourceLoadQueueChildrenCache.close(); From 776552220ae3491854082789d02f1e56f20449b6 Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 3 Sep 2018 14:29:50 -0500 Subject: [PATCH 19/29] Enable looking for stuck thread in CuratorDruidCoordinatorTest --- pom.xml | 2 +- .../CuratorDruidCoordinatorTest.java | 29 +++++-------------- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/pom.xml b/pom.xml index 52e750db210c..ef59cd7ab83f 100644 --- a/pom.xml +++ b/pom.xml @@ -745,7 +745,7 @@ junit junit - 4.11 + 4.12 test diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index d1d925fcbb78..451be84799b9 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -57,7 +57,10 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -103,8 +106,6 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; - private CountDownLatch testFinished; - public CuratorDruidCoordinatorTest() { jsonMapper = TestHelper.makeJsonMapper(); @@ -231,29 +232,11 @@ public void unannounce(DruidNode node) EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector() ); - testFinished = new CountDownLatch(1); - Thread deadLockWatcher = new Thread(() -> { - try { - if (!testFinished.await(50, TimeUnit.SECONDS)) { - Thread.getAllStackTraces().forEach((thread, stackTrace) -> { - Throwable t = new Throwable(thread.getName()); - t.setStackTrace(stackTrace); - t.printStackTrace(); - }); - } - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - }); - deadLockWatcher.setDaemon(true); - deadLockWatcher.start(); } @After public void tearDown() throws Exception { - testFinished.countDown(); baseView.stop(); sourceLoadQueuePeon.stop(); sourceLoadQueueChildrenCache.close(); @@ -261,7 +244,11 @@ public void tearDown() throws Exception tearDownServerAndCurator(); } - @Test(timeout = 60_000L) + @Rule + public final TestRule timeout = + Timeout.builder().withTimeout(60, TimeUnit.SECONDS).withLookingForStuckThread(true).build(); + + @Test public void testMoveSegment() throws Exception { segmentViewInitLatch = new CountDownLatch(1); From ea7a34118ca5142869556ab71b956e00f3af15af Mon Sep 17 00:00:00 2001 From: leventov Date: Mon, 3 Sep 2018 18:55:37 -0500 Subject: [PATCH 20/29] Enable looking for stuck thread in RemoteTaskRunnerTest --- .../overlord/RemoteTaskRunnerTest.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTest.java index 837e5f2d1865..009f47fd8f02 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTest.java @@ -48,7 +48,10 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; import java.util.ArrayList; import java.util.Collection; @@ -62,7 +65,6 @@ public class RemoteTaskRunnerTest private static final String workerHost = "worker"; private static final String announcementsPath = joiner.join(RemoteTaskRunnerTestUtils.announcementsPath, workerHost); private static final String statusPath = joiner.join(RemoteTaskRunnerTestUtils.statusPath, workerHost); - private static final int TIMEOUT_SECONDS = 20; private RemoteTaskRunner remoteTaskRunner; private RemoteTaskRunnerTestUtils rtrTestUtils = new RemoteTaskRunnerTestUtils(); @@ -72,6 +74,10 @@ public class RemoteTaskRunnerTest private Task task; private Worker worker; + @Rule + public final TestRule timeout = + Timeout.builder().withTimeout(60, TimeUnit.SECONDS).withLookingForStuckThread(true).build(); + @Before public void setUp() throws Exception { @@ -312,7 +318,7 @@ public void testStatusRemoved() throws Exception cf.delete().forPath(joiner.join(statusPath, task.getId())); - TaskStatus status = future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + TaskStatus status = future.get(); Assert.assertEquals(status.getStatusCode(), TaskState.FAILED); } @@ -403,7 +409,7 @@ public void testWorkerRemoved() throws Exception cf.delete().forPath(announcementsPath); - TaskStatus status = future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + TaskStatus status = future.get(); Assert.assertEquals(TaskState.FAILED, status.getStatusCode()); RemoteTaskRunnerConfig config = remoteTaskRunner.getRemoteTaskRunnerConfig(); @@ -659,7 +665,7 @@ public boolean isValid() ); mockWorkerCompleteSuccessfulTask(task); - TaskStatus status = future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + TaskStatus status = future.get(); Assert.assertEquals(status.getStatusCode(), TaskState.SUCCESS); Assert.assertEquals(TaskState.SUCCESS, status.getStatusCode()); } @@ -702,7 +708,7 @@ public void testBlacklistZKWorkers() throws Exception Assert.assertTrue(taskAnnounced(task1.getId())); mockWorkerRunningTask(task1); mockWorkerCompleteFailedTask(task1); - Assert.assertTrue(taskFuture1.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).isFailure()); + Assert.assertTrue(taskFuture1.get().isFailure()); Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size()); Assert.assertEquals( 1, @@ -720,7 +726,7 @@ public void testBlacklistZKWorkers() throws Exception Assert.assertTrue(taskAnnounced(task2.getId())); mockWorkerRunningTask(task2); mockWorkerCompleteFailedTask(task2); - Assert.assertTrue(taskFuture2.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).isFailure()); + Assert.assertTrue(taskFuture2.get().isFailure()); Assert.assertEquals(1, remoteTaskRunner.getBlackListedWorkers().size()); Assert.assertEquals( 2, @@ -755,7 +761,7 @@ public void testBlacklistZKWorkers() throws Exception Assert.assertTrue(taskAnnounced(task3.getId())); mockWorkerRunningTask(task3); mockWorkerCompleteSuccessfulTask(task3); - Assert.assertTrue(taskFuture3.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).isSuccess()); + Assert.assertTrue(taskFuture3.get().isSuccess()); Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size()); Assert.assertEquals( 0, @@ -807,7 +813,7 @@ taskId, new TaskResource(taskId, 1), "foo", TaskStatus.success(taskId), jsonMapp rtrTestUtils.mockWorkerRunningTask(expectedWorker, task); rtrTestUtils.mockWorkerCompleteFailedTask(expectedWorker, task); - Assert.assertTrue(taskFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).isFailure()); + Assert.assertTrue(taskFuture.get().isFailure()); Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size()); Assert.assertEquals( ((i + 1) / 2), @@ -860,7 +866,7 @@ taskId, new TaskResource(taskId, 1), "foo", TaskStatus.success(taskId), jsonMapp rtrTestUtils.mockWorkerRunningTask(expectedWorker, task); rtrTestUtils.mockWorkerCompleteFailedTask(expectedWorker, task); - Assert.assertTrue(taskFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).isFailure()); + Assert.assertTrue(taskFuture.get().isFailure()); Assert.assertEquals(i > 2 ? 1 : 0, remoteTaskRunner.getBlackListedWorkers().size()); Assert.assertEquals( i > 4 ? i - 2 : ((i + 1) / 2), @@ -894,7 +900,7 @@ public void testSuccessfulTaskOnBlacklistedWorker() throws Exception Assert.assertTrue(taskAnnounced(task1.getId())); mockWorkerRunningTask(task1); mockWorkerCompleteFailedTask(task1); - Assert.assertTrue(taskFuture1.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).isFailure()); + Assert.assertTrue(taskFuture1.get().isFailure()); Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size()); Future taskFuture2 = remoteTaskRunner.run(task2); @@ -905,11 +911,11 @@ public void testSuccessfulTaskOnBlacklistedWorker() throws Exception Assert.assertTrue(taskAnnounced(task3.getId())); mockWorkerRunningTask(task3); mockWorkerCompleteFailedTask(task3); - Assert.assertTrue(taskFuture3.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).isFailure()); + Assert.assertTrue(taskFuture3.get().isFailure()); Assert.assertEquals(1, remoteTaskRunner.getBlackListedWorkers().size()); mockWorkerCompleteSuccessfulTask(task2); - Assert.assertTrue(taskFuture2.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).isSuccess()); + Assert.assertTrue(taskFuture2.get().isSuccess()); Assert.assertEquals(0, remoteTaskRunner.getBlackListedWorkers().size()); } } From d31453b1ae0b7d1c82548f2897feb260aafaeb39 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 4 Sep 2018 13:51:28 -0500 Subject: [PATCH 21/29] Use JUnit's TemporaryFolder in PrefetchableTextFilesFirehoseFactoryTest --- ...fetchableTextFilesFirehoseFactoryTest.java | 34 +++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/api/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java b/api/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java index 0465d9a95a07..1d264a79e789 100644 --- a/api/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java +++ b/api/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java @@ -37,12 +37,13 @@ import org.apache.druid.java.util.common.RetryUtils; import org.apache.druid.java.util.common.StringUtils; import org.hamcrest.CoreMatchers; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; import java.io.BufferedWriter; import java.io.File; @@ -62,8 +63,6 @@ public class PrefetchableTextFilesFirehoseFactoryTest { - private static final List FIREHOSE_TMP_DIRS = new ArrayList<>(); - private static File TEST_DIR; private static long FILE_SIZE = -1; private static final StringInputRowParser parser = new StringInputRowParser( @@ -86,16 +85,17 @@ public class PrefetchableTextFilesFirehoseFactoryTest StandardCharsets.UTF_8.name() ); + @ClassRule + public static TemporaryFolder tempDir = new TemporaryFolder(); + private static File TEST_DIR; + @Rule public ExpectedException expectedException = ExpectedException.none(); @BeforeClass public static void setup() throws IOException { - TEST_DIR = File.createTempFile(PrefetchableTextFilesFirehoseFactoryTest.class.getSimpleName(), "testDir"); - FileUtils.forceDelete(TEST_DIR); - FileUtils.forceMkdir(TEST_DIR); - + TEST_DIR = tempDir.newFolder(); for (int i = 0; i < 100; i++) { try ( CountingOutputStream cos = new CountingOutputStream( @@ -118,15 +118,6 @@ public static void setup() throws IOException } } - @AfterClass - public static void teardown() throws IOException - { - FileUtils.forceDelete(TEST_DIR); - for (File dir : FIREHOSE_TMP_DIRS) { - FileUtils.forceDelete(dir); - } - } - private static void assertResult(List rows) { Assert.assertEquals(10000, rows.size()); @@ -160,16 +151,9 @@ private static void assertNumRemainingCacheFiles(File firehoseTmpDir, int expect Assert.assertEquals(expectedNumFiles, files.length); } - private static File createFirehoseTmpDir(String dirSuffix) throws IOException + private static File createFirehoseTmpDir(String dirPrefix) throws IOException { - final File firehoseTempDir = File.createTempFile( - PrefetchableTextFilesFirehoseFactoryTest.class.getSimpleName(), - dirSuffix - ); - FileUtils.forceDelete(firehoseTempDir); - FileUtils.forceMkdir(firehoseTempDir); - FIREHOSE_TMP_DIRS.add(firehoseTempDir); - return firehoseTempDir; + return Files.createTempDirectory(tempDir.getRoot().toPath(), dirPrefix).toFile(); } @Test From 2ebc85c9841ffdc15ec628f47ffa4e47c22f83d1 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 4 Sep 2018 15:25:24 -0500 Subject: [PATCH 22/29] Increase timeout in PrefetchableTextFilesFirehoseFactoryTest --- .../prefetch/PrefetchableTextFilesFirehoseFactoryTest.java | 5 +++-- .../indexing/kafka/supervisor/KafkaSupervisorTest.java | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java b/api/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java index 1d264a79e789..9c3508e8e0dd 100644 --- a/api/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java +++ b/api/src/test/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactoryTest.java @@ -396,6 +396,7 @@ static TestPrefetchableTextFilesFirehoseFactory with(File baseDir, long cacheCap 1024, cacheCapacity, fetchCapacity, + 60_000, // fetch timeout 3, 0, 0, @@ -504,7 +505,7 @@ private static long computeTimeout(int maxRetry) long prefetchTriggerThreshold, long maxCacheCapacityBytes, long maxFetchCapacityBytes, - long timeout, + long fetchTimeout, int maxRetry, int numOpenExceptions, int maxConnectionResets, @@ -515,7 +516,7 @@ private static long computeTimeout(int maxRetry) maxCacheCapacityBytes, maxFetchCapacityBytes, prefetchTriggerThreshold, - timeout, + fetchTimeout, maxRetry ); this.numOpenExceptions = numOpenExceptions; diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 7666dab95e02..e916f1cd34d1 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -2078,9 +2078,9 @@ public void testCheckpointForInactiveTaskGroup() expect(taskStorage.getTask("id1")).andReturn(Optional.of(id1)).anyTimes(); expect(taskStorage.getTask("id2")).andReturn(Optional.of(id2)).anyTimes(); expect(taskStorage.getTask("id3")).andReturn(Optional.of(id3)).anyTimes(); - expect( - indexerMetadataStorageCoordinator.getDataSourceMetadata(DATASOURCE)).andReturn(new KafkaDataSourceMetadata(null) - ).anyTimes(); + expect(indexerMetadataStorageCoordinator.getDataSourceMetadata(DATASOURCE)) + .andReturn(new KafkaDataSourceMetadata(null)) + .anyTimes(); expect(taskClient.getStatusAsync("id1")).andReturn(Futures.immediateFuture(KafkaIndexTask.Status.READING)); expect(taskClient.getStatusAsync("id2")).andReturn(Futures.immediateFuture(KafkaIndexTask.Status.READING)); expect(taskClient.getStatusAsync("id3")).andReturn(Futures.immediateFuture(KafkaIndexTask.Status.READING)); From 186b8558f733eeeda83c6ce937aded331db83b20 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 4 Sep 2018 15:38:24 -0500 Subject: [PATCH 23/29] KafkaSupervisor.awaitAllNoticesHandled --- .../kafka/supervisor/KafkaSupervisor.java | 27 ++++++++++++------- .../kafka/supervisor/KafkaSupervisorTest.java | 12 +++------ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java index 43d14c8ba8c9..31945a581718 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java @@ -379,15 +379,18 @@ public void run() { try { while (!Thread.currentThread().isInterrupted()) { - final Notice notice = notices.take(); + synchronized (notices) { + final Notice notice = notices.take(); - try { - notice.handle(); - } - catch (Throwable e) { - log.makeAlert(e, "KafkaSupervisor[%s] failed to handle notice", dataSource) - .addData("noticeClass", notice.getClass().getSimpleName()) - .emit(); + try { + notice.handle(); + notices.notifyAll(); + } + catch (Throwable e) { + log.makeAlert(e, "KafkaSupervisor[%s] failed to handle notice", dataSource) + .addData("noticeClass", notice.getClass().getSimpleName()) + .emit(); + } } } } @@ -2382,9 +2385,13 @@ void moveTaskGroupToPendingCompletion(int taskGroupId) } @VisibleForTesting - int getNoticesQueueSize() + void awaitAllNoticesHandled() throws InterruptedException { - return notices.size(); + synchronized (notices) { + while (!notices.isEmpty()) { + notices.wait(); + } + } } private static class StatsFromTaskResult diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index e916f1cd34d1..f4f1e9abfd65 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -2117,9 +2117,7 @@ public void testCheckpointForInactiveTaskGroup() new KafkaDataSourceMetadata(new KafkaPartitions(topic, fakeCheckpoints)) ); - while (supervisor.getNoticesQueueSize() > 0) { - Thread.sleep(100); - } + supervisor.awaitAllNoticesHandled(); verifyAll(); @@ -2187,9 +2185,7 @@ public void testCheckpointForUnknownTaskGroup() throws InterruptedException new KafkaDataSourceMetadata(new KafkaPartitions(topic, Collections.emptyMap())) ); - while (supervisor.getNoticesQueueSize() > 0) { - Thread.sleep(100); - } + supervisor.awaitAllNoticesHandled(); verifyAll(); @@ -2285,9 +2281,7 @@ public void testCheckpointWithNullTaskGroupId() new KafkaDataSourceMetadata(new KafkaPartitions(topic, newCheckpoints.get(0))) ); - while (supervisor.getNoticesQueueSize() > 0) { - Thread.sleep(100); - } + supervisor.awaitAllNoticesHandled(); verifyAll(); } From 316e33bcd2c9a19eeb7627d51f746da55d415dc0 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 4 Sep 2018 17:40:42 -0500 Subject: [PATCH 24/29] Add DeadlockDetectingTimeout TestRule --- .../DeadlockDetectingFailOnTimeout.java | 301 ++++++++++++++++++ .../testing/DeadlockDetectingTimeout.java | 47 +++ indexing-service/pom.xml | 7 + .../overlord/RemoteTaskRunnerTest.java | 5 +- pom.xml | 1 + .../CuratorDruidCoordinatorTest.java | 5 +- 6 files changed, 360 insertions(+), 6 deletions(-) create mode 100644 common/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java create mode 100644 common/src/test/java/org/apache/druid/testing/DeadlockDetectingTimeout.java diff --git a/common/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java b/common/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java new file mode 100644 index 000000000000..2d61bdf301c8 --- /dev/null +++ b/common/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing; + +import org.junit.runners.model.MultipleFailureException; +import org.junit.runners.model.Statement; +import org.junit.runners.model.TestTimedOutException; + +import javax.annotation.Nullable; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This class is based on {@link org.junit.internal.runners.statements.FailOnTimeout}, additionally deadlocked + * threads are detected. + */ +final class DeadlockDetectingFailOnTimeout extends Statement +{ + private final Statement originalStatement; + private final TimeUnit timeUnit; + private final long timeout; + + DeadlockDetectingFailOnTimeout(long timeout, TimeUnit timeoutUnit, Statement statement) + { + originalStatement = statement; + this.timeout = timeout; + timeUnit = timeoutUnit; + } + + @Override + public void evaluate() throws Throwable + { + CallableStatement callable = new CallableStatement(); + FutureTask task = new FutureTask<>(callable); + ThreadGroup threadGroup = new ThreadGroup("FailOnTimeoutGroup"); + Thread thread = new Thread(threadGroup, task, "Time-limited test"); + try { + thread.setDaemon(true); + thread.start(); + callable.awaitStarted(); + Throwable throwable = getResult(task, thread); + if (throwable != null) { + throw throwable; + } + } + finally { + try { + thread.join(1); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + threadGroup.destroy(); + } + catch (IllegalThreadStateException e) { + // If a thread from the group is still alive, the ThreadGroup cannot be destroyed. + // Swallow the exception to keep the same behavior prior to this change. + } + } + } + + /** + * Wait for the test task, returning the exception thrown by the test if the + * test failed, an exception indicating a timeout if the test timed out, or + * {@code null} if the test passed. + */ + private Throwable getResult(FutureTask task, Thread thread) + { + try { + if (timeout > 0) { + return task.get(timeout, timeUnit); + } else { + return task.get(); + } + } + catch (InterruptedException e) { + return e; // caller will re-throw; no need to call Thread.interrupt() + } + catch (ExecutionException e) { + // test failed; have caller re-throw the exception thrown by the test + return e.getCause(); + } + catch (TimeoutException e) { + return createTimeoutException(thread); + } + } + + private Exception createTimeoutException(Thread thread) + { + StackTraceElement[] stackTrace = thread.getStackTrace(); + Exception currThreadException = new TestTimedOutException(timeout, timeUnit); + if (stackTrace != null) { + currThreadException.setStackTrace(stackTrace); + thread.interrupt(); + } + Exception stuckThreadException = getStuckThreadException(thread); + Exception deadlockException = getDeadlockedThreadsException(); + if (stuckThreadException != null || deadlockException != null) { + List exceptions = Stream + .of(currThreadException, stuckThreadException, deadlockException) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + return new MultipleFailureException(exceptions); + } else { + return currThreadException; + } + } + + /** + * Retrieves the stack trace for a given thread. + * + * @param thread The thread whose stack is to be retrieved. + * + * @return The stack trace; returns a zero-length array if the thread has + * terminated or the stack cannot be retrieved for some other reason. + */ + private StackTraceElement[] getStackTrace(Thread thread) + { + try { + return thread.getStackTrace(); + } + catch (SecurityException e) { + return new StackTraceElement[0]; + } + } + + @Nullable + private Exception getStuckThreadException(Thread mainThread) { + final Thread stuckThread = getStuckThread(mainThread); + if (stuckThread == null) { + return null; + } + Exception stuckThreadException = new Exception("Appears to be stuck in thread " + stuckThread.getName()); + stuckThreadException.setStackTrace(getStackTrace(stuckThread)); + return stuckThreadException; + } + + /** + * Determines whether the test appears to be stuck in some thread other than + * the "main thread" (the one created to run the test). This feature is experimental. + * Behavior may change after the 4.12 release in response to feedback. + * + * @param mainThread The main thread created by {@code evaluate()} + * + * @return The thread which appears to be causing the problem, if different from + * {@code mainThread}, or {@code null} if the main thread appears to be the + * problem or if the thread cannot be determined. The return value is never equal + * to {@code mainThread}. + */ + private Thread getStuckThread(Thread mainThread) + { + List threadsInGroup = getThreadsInGroup(mainThread.getThreadGroup()); + if (threadsInGroup.isEmpty()) { + return null; + } + + // Now that we have all the threads in the test's thread group: Assume that + // any thread we're "stuck" in is RUNNABLE. Look for all RUNNABLE threads. + // If just one, we return that (unless it equals threadMain). If there's more + // than one, pick the one that's using the most CPU time, if this feature is + // supported. + Thread stuckThread = null; + long maxCpuTime = 0; + for (Thread thread : threadsInGroup) { + if (thread.getState() == Thread.State.RUNNABLE) { + long threadCpuTime = cpuTime(thread); + if (stuckThread == null || threadCpuTime > maxCpuTime) { + stuckThread = thread; + maxCpuTime = threadCpuTime; + } + } + } + return (stuckThread == mainThread) ? null : stuckThread; + } + + /** + * Returns all active threads belonging to a thread group. + * + * @param group The thread group. + * + * @return The active threads in the thread group. The result should be a + * complete list of the active threads at some point in time. Returns an empty list + * if this cannot be determined, e.g. because new threads are being created at an + * extremely fast rate. + */ + private List getThreadsInGroup(ThreadGroup group) + { + final int activeThreadCount = group.activeCount(); // this is just an estimate + int threadArraySize = Math.max(activeThreadCount * 2, 100); + for (int loopCount = 0; loopCount < 5; loopCount++) { + Thread[] threads = new Thread[threadArraySize]; + int enumCount = group.enumerate(threads); + if (enumCount < threadArraySize) { + return Arrays.asList(threads).subList(0, enumCount); + } + // if there are too many threads to fit into the array, enumerate's result + // is >= the array's length; therefore we can't trust that it returned all + // the threads. Try again. + threadArraySize += 100; + } + // threads are proliferating too fast for us. Bail before we get into + // trouble. + return Collections.emptyList(); + } + + /** + * Returns the CPU time used by a thread, if possible. + * + * @param thr The thread to query. + * + * @return The CPU time used by {@code thr}, or 0 if it cannot be determined. + */ + private long cpuTime(Thread thr) + { + ThreadMXBean mxBean = ManagementFactory.getThreadMXBean(); + if (mxBean.isThreadCpuTimeSupported()) { + try { + return mxBean.getThreadCpuTime(thr.getId()); + } + catch (UnsupportedOperationException ignore) { + // fall through + } + } + return 0; + } + + @Nullable + private Exception getDeadlockedThreadsException() + { + final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + final long[] deadlockedThreadIds = threadMXBean.findDeadlockedThreads(); + if (deadlockedThreadIds == null) { + return null; + } + Exception deadlockException = new Exception("Deadlocked threads:"); + for (long deadlockedThreadId : deadlockedThreadIds) { + ThreadInfo threadInfo = threadMXBean.getThreadInfo(deadlockedThreadId); + Exception threadException = new Exception(threadInfo.getThreadName() + " at " + threadInfo.getLockName()); + threadException.setStackTrace(threadInfo.getStackTrace()); + deadlockException.addSuppressed(threadException); + } + return deadlockException; + } + + private class CallableStatement implements Callable + { + private final CountDownLatch startLatch = new CountDownLatch(1); + + @Override + public Throwable call() throws Exception + { + try { + startLatch.countDown(); + originalStatement.evaluate(); + } + catch (Exception e) { + throw e; + } + catch (Throwable e) { + return e; + } + return null; + } + + public void awaitStarted() throws InterruptedException + { + startLatch.await(); + } + } +} diff --git a/common/src/test/java/org/apache/druid/testing/DeadlockDetectingTimeout.java b/common/src/test/java/org/apache/druid/testing/DeadlockDetectingTimeout.java new file mode 100644 index 000000000000..cae3b0082fc4 --- /dev/null +++ b/common/src/test/java/org/apache/druid/testing/DeadlockDetectingTimeout.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing; + +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import java.util.concurrent.TimeUnit; + +/** + * This Rule is based on {@link org.junit.rules.Timeout}, additionally deadlocked threads are detected. + */ +public final class DeadlockDetectingTimeout implements TestRule +{ + private final long timeout; + private final TimeUnit timeoutUnit; + + public DeadlockDetectingTimeout(long timeout, TimeUnit timeoutUnit) + { + this.timeout = timeout; + this.timeoutUnit = timeoutUnit; + } + + @Override + public Statement apply(Statement base, Description description) + { + return new DeadlockDetectingFailOnTimeout(timeout, timeoutUnit, base); + } +} diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index b77a09a9109e..9b0a31b4a5f2 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -69,6 +69,13 @@ hadoop-client test + + org.apache.druid + druid-common + ${project.parent.version} + test-jar + test + org.apache.druid druid-processing diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTest.java index 009f47fd8f02..74e19365f7c0 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTest.java @@ -43,6 +43,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.testing.DeadlockDetectingTimeout; import org.easymock.EasyMock; import org.joda.time.Period; import org.junit.After; @@ -51,7 +52,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; -import org.junit.rules.Timeout; import java.util.ArrayList; import java.util.Collection; @@ -75,8 +75,7 @@ public class RemoteTaskRunnerTest private Worker worker; @Rule - public final TestRule timeout = - Timeout.builder().withTimeout(60, TimeUnit.SECONDS).withLookingForStuckThread(true).build(); + public final TestRule timeout = new DeadlockDetectingTimeout(60, TimeUnit.SECONDS); @Before public void setUp() throws Exception diff --git a/pom.xml b/pom.xml index ef59cd7ab83f..161b9ab46690 100644 --- a/pom.xml +++ b/pom.xml @@ -1034,6 +1034,7 @@ -Ddruid.indexing.doubleStorage=double + false true diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index 451be84799b9..e51629d7a38c 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -50,6 +50,7 @@ import org.apache.druid.server.initialization.ZkPathsConfig; import org.apache.druid.server.lookup.cache.LookupCoordinatorManager; import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.testing.DeadlockDetectingTimeout; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NoneShardSpec; import org.easymock.EasyMock; @@ -60,7 +61,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; -import org.junit.rules.Timeout; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -245,8 +245,7 @@ public void tearDown() throws Exception } @Rule - public final TestRule timeout = - Timeout.builder().withTimeout(60, TimeUnit.SECONDS).withLookingForStuckThread(true).build(); + public final TestRule timeout = new DeadlockDetectingTimeout(60, TimeUnit.SECONDS); @Test public void testMoveSegment() throws Exception From 0375f783862b15b65b0927abc084fe4299c728fb Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 4 Sep 2018 17:40:55 -0500 Subject: [PATCH 25/29] Revert "KafkaSupervisor.awaitAllNoticesHandled" This reverts commit 186b8558f733eeeda83c6ce937aded331db83b20. --- .../kafka/supervisor/KafkaSupervisor.java | 27 +++++++------------ .../kafka/supervisor/KafkaSupervisorTest.java | 12 ++++++--- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java index 31945a581718..43d14c8ba8c9 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java @@ -379,18 +379,15 @@ public void run() { try { while (!Thread.currentThread().isInterrupted()) { - synchronized (notices) { - final Notice notice = notices.take(); + final Notice notice = notices.take(); - try { - notice.handle(); - notices.notifyAll(); - } - catch (Throwable e) { - log.makeAlert(e, "KafkaSupervisor[%s] failed to handle notice", dataSource) - .addData("noticeClass", notice.getClass().getSimpleName()) - .emit(); - } + try { + notice.handle(); + } + catch (Throwable e) { + log.makeAlert(e, "KafkaSupervisor[%s] failed to handle notice", dataSource) + .addData("noticeClass", notice.getClass().getSimpleName()) + .emit(); } } } @@ -2385,13 +2382,9 @@ void moveTaskGroupToPendingCompletion(int taskGroupId) } @VisibleForTesting - void awaitAllNoticesHandled() throws InterruptedException + int getNoticesQueueSize() { - synchronized (notices) { - while (!notices.isEmpty()) { - notices.wait(); - } - } + return notices.size(); } private static class StatsFromTaskResult diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index f4f1e9abfd65..e916f1cd34d1 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -2117,7 +2117,9 @@ public void testCheckpointForInactiveTaskGroup() new KafkaDataSourceMetadata(new KafkaPartitions(topic, fakeCheckpoints)) ); - supervisor.awaitAllNoticesHandled(); + while (supervisor.getNoticesQueueSize() > 0) { + Thread.sleep(100); + } verifyAll(); @@ -2185,7 +2187,9 @@ public void testCheckpointForUnknownTaskGroup() throws InterruptedException new KafkaDataSourceMetadata(new KafkaPartitions(topic, Collections.emptyMap())) ); - supervisor.awaitAllNoticesHandled(); + while (supervisor.getNoticesQueueSize() > 0) { + Thread.sleep(100); + } verifyAll(); @@ -2281,7 +2285,9 @@ public void testCheckpointWithNullTaskGroupId() new KafkaDataSourceMetadata(new KafkaPartitions(topic, newCheckpoints.get(0))) ); - supervisor.awaitAllNoticesHandled(); + while (supervisor.getNoticesQueueSize() > 0) { + Thread.sleep(100); + } verifyAll(); } From 2a79e3f20d40aba2eee1fcda22581aafa351c973 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 4 Sep 2018 17:55:07 -0500 Subject: [PATCH 26/29] Fix brace --- .../apache/druid/testing/DeadlockDetectingFailOnTimeout.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java b/common/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java index 2d61bdf301c8..ecb271101222 100644 --- a/common/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java +++ b/common/src/test/java/org/apache/druid/testing/DeadlockDetectingFailOnTimeout.java @@ -156,7 +156,8 @@ private StackTraceElement[] getStackTrace(Thread thread) } @Nullable - private Exception getStuckThreadException(Thread mainThread) { + private Exception getStuckThreadException(Thread mainThread) + { final Thread stuckThread = getStuckThread(mainThread); if (stuckThread == null) { return null; From e460e3e02b07a8706746d673c4c25b2ab78a3245 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 4 Sep 2018 19:05:06 -0500 Subject: [PATCH 27/29] Don't miss fetchFutures in Fetcher --- .../input/impl/FileIteratingFirehose.java | 21 +++------------ .../data/input/impl/prefetch/Fetcher.java | 26 ++++++++++++------- .../PrefetchableTextFilesFirehoseFactory.java | 26 ++++++++++--------- 3 files changed, 34 insertions(+), 39 deletions(-) diff --git a/api/src/main/java/org/apache/druid/data/input/impl/FileIteratingFirehose.java b/api/src/main/java/org/apache/druid/data/input/impl/FileIteratingFirehose.java index 50972b676bf5..6b75ed047294 100644 --- a/api/src/main/java/org/apache/druid/data/input/impl/FileIteratingFirehose.java +++ b/api/src/main/java/org/apache/druid/data/input/impl/FileIteratingFirehose.java @@ -101,24 +101,9 @@ public Runnable commit() @Override public void close() throws IOException { - try { - if (lineIterator != null) { - lineIterator.close(); - } - } - catch (Throwable t) { - try { - if (closer != null) { - closer.close(); - } - } - catch (Exception e) { - t.addSuppressed(e); - } - throw t; - } - if (closer != null) { - closer.close(); + try (Closeable ignore = closer; + Closeable ignore2 = lineIterator != null ? lineIterator::close : null) { + // close both via try-with-resources } } } diff --git a/api/src/main/java/org/apache/druid/data/input/impl/prefetch/Fetcher.java b/api/src/main/java/org/apache/druid/data/input/impl/prefetch/Fetcher.java index 85276ea96d70..7d3138e29121 100644 --- a/api/src/main/java/org/apache/druid/data/input/impl/prefetch/Fetcher.java +++ b/api/src/main/java/org/apache/druid/data/input/impl/prefetch/Fetcher.java @@ -28,6 +28,8 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; @@ -62,7 +64,7 @@ public abstract class Fetcher implements Iterator> // This is updated when a file is successfully fetched, a fetched file is deleted, or a fetched file is // cached. private final AtomicLong fetchedBytes = new AtomicLong(0); - private Future fetchFuture; + private final Deque> fetchFutures = new ArrayDeque<>(); private PrefetchConfig prefetchConfig; // nextFetchIndex indicates which object should be downloaded when fetch is triggered. @@ -103,12 +105,13 @@ public abstract class Fetcher implements Iterator> */ private void fetchIfNeeded(long remainingBytes) { - if ((fetchFuture == null || fetchFuture.isDone()) + if ((fetchFutures.isEmpty() || fetchFutures.peekLast().isDone()) && remainingBytes <= prefetchConfig.getPrefetchTriggerBytes()) { - fetchFuture = fetchExecutor.submit(() -> { + Future fetchFuture = fetchExecutor.submit(() -> { fetch(); return null; }); + fetchFutures.add(fetchFuture); } } @@ -180,12 +183,17 @@ public OpenedObject next() private void checkFetchException(boolean wait) { try { - if (wait) { - fetchFuture.get(prefetchConfig.getFetchTimeout(), TimeUnit.MILLISECONDS); - fetchFuture = null; - } else if (fetchFuture != null && fetchFuture.isDone()) { - fetchFuture.get(); - fetchFuture = null; + for (Future fetchFuture; (fetchFuture = fetchFutures.poll()) != null; ) { + if (wait) { + fetchFuture.get(prefetchConfig.getFetchTimeout(), TimeUnit.MILLISECONDS); + } else { + if (fetchFuture.isDone()) { + fetchFuture.get(); + } else { + fetchFutures.addFirst(fetchFuture); + break; + } + } } } catch (InterruptedException | ExecutionException e) { diff --git a/api/src/main/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactory.java b/api/src/main/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactory.java index 13d644788dbe..2a0d917ff052 100644 --- a/api/src/main/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactory.java +++ b/api/src/main/java/org/apache/druid/data/input/impl/prefetch/PrefetchableTextFilesFirehoseFactory.java @@ -221,21 +221,24 @@ public LineIterator next() } final OpenedObject openedObject = fetcher.next(); - final InputStream stream; try { - stream = wrapObjectStream( - openedObject.getObject(), - openedObject.getObjectStream() + return new ResourceCloseableLineIterator( + new InputStreamReader( + wrapObjectStream(openedObject.getObject(), openedObject.getObjectStream()), + StandardCharsets.UTF_8 + ), + openedObject.getResourceCloser() ); } catch (IOException e) { + try { + openedObject.getResourceCloser().close(); + } + catch (Throwable t) { + e.addSuppressed(t); + } throw new RuntimeException(e); } - - return new ResourceCloseableLineIterator( - new InputStreamReader(stream, StandardCharsets.UTF_8), - openedObject.getResourceCloser() - ); } }, firehoseParser, @@ -288,9 +291,8 @@ static class ResourceCloseableLineIterator extends LineIterator @Override public void close() { - super.close(); - try { - resourceCloser.close(); + try (Closeable ignore = this.resourceCloser) { + super.close(); } catch (IOException e) { throw new RuntimeException(e); From 7668da5ff89bf3ba0da27754f537c9ace803f9b4 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 4 Sep 2018 19:21:39 -0500 Subject: [PATCH 28/29] volatile in CuratorDruidCoordinatorTest --- .../coordinator/CuratorDruidCoordinatorTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index e51629d7a38c..ed1064bd6000 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -101,8 +101,13 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase private BatchServerInventoryView baseView; private CoordinatorServerView serverView; private CountDownLatch segmentViewInitLatch; - private CountDownLatch segmentAddedLatch; - private CountDownLatch segmentRemovedLatch; + /** + * The following two fields are changed during {@link #testMoveSegment()}, the change might not be visible from the + * thread, that runs the callback, registered in {@link #setupView()}. volatile modificator doesn't guarantee + * visibility either, but somewhat increases the chances. + */ + private volatile CountDownLatch segmentAddedLatch; + private volatile CountDownLatch segmentRemovedLatch; private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; @@ -436,7 +441,8 @@ private void setupView() throws Exception public void registerSegmentCallback(Executor exec, final SegmentCallback callback) { super.registerSegmentCallback( - exec, new SegmentCallback() + exec, + new SegmentCallback() { @Override public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) From 5fff23703b126da75d3c9751aa789c84dbc4c5ec Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 2 Oct 2018 13:19:43 -0300 Subject: [PATCH 29/29] Fix test --- .../org/apache/druid/sql/calcite/http/SqlResourceTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/http/SqlResourceTest.java index ef268f139b82..99b702400d7a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/http/SqlResourceTest.java @@ -499,8 +499,8 @@ public void testCsvResultFormat() throws Exception Assert.assertEquals( ImmutableList.of( - "2000-01-01T00:00:00.000Z,1,,a,1.0,1.0,org.apache.druid.hll.HLLCV1,", - "2000-01-02T00:00:00.000Z,1,10.1,,2.0,2.0,org.apache.druid.hll.HLLCV1,", + "2000-01-01T00:00:00.000Z,1,,a,1.0,1.0,org.apache.druid.hll.VersionOneHyperLogLogCollector,", + "2000-01-02T00:00:00.000Z,1,10.1,,2.0,2.0,org.apache.druid.hll.VersionOneHyperLogLogCollector,", "", "" ),