From aae8d98f3a2f6f33821c565d23a933b0111246c2 Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 01:48:57 +0530 Subject: [PATCH 01/50] Support Nulls in Storage Layer Storage changes --- .../ColumnSelectorBitmapIndexSelector.java | 5 +- .../io/druid/segment/ColumnValueSelector.java | 2 + .../segment/ConstantDimensionSelector.java | 3 +- .../main/java/io/druid/segment/IndexIO.java | 16 +- .../java/io/druid/segment/IndexMerger.java | 25 +-- .../java/io/druid/segment/IndexMergerV9.java | 24 +-- .../io/druid/segment/NullHandlingHelper.java | 106 ++++++++++ .../segment/NullValueHandlingConfig.java | 45 +++++ .../druid/segment/ObjectColumnSelector.java | 12 ++ .../druid/segment/StringDimensionHandler.java | 16 +- .../druid/segment/column/ColumnBuilder.java | 26 ++- .../segment/column/ColumnCapabilities.java | 2 + .../column/ColumnCapabilitiesImpl.java | 15 ++ .../segment/column/ColumnDescriptor.java | 24 ++- .../io/druid/segment/column/DoubleColumn.java | 14 +- .../io/druid/segment/column/FloatColumn.java | 12 +- .../column/IndexedDoublesGenericColumn.java | 11 +- .../column/IndexedFloatsGenericColumn.java | 14 +- .../column/IndexedLongsGenericColumn.java | 14 +- .../io/druid/segment/column/LongColumn.java | 13 +- .../io/druid/segment/column/SimpleColumn.java | 6 +- .../column/SimpleDictionaryEncodedColumn.java | 6 +- .../segment/data/ByteBufferSerializer.java | 10 +- .../druid/segment/data/ByteBufferWriter.java | 7 +- .../io/druid/segment/data/GenericIndexed.java | 47 +++-- .../segment/data/GenericIndexedWriter.java | 8 +- .../io/druid/segment/data/IndexedDoubles.java | 29 ++- .../io/druid/segment/data/IndexedFloats.java | 27 ++- .../io/druid/segment/data/IndexedLongs.java | 25 ++- .../data/RoaringBitmapSerdeFactory.java | 3 + .../druid/segment/serde/ColumnPartSerde.java | 5 +- .../serde/DoubleGenericColumnPartSerde.java | 7 +- .../serde/DoubleGenericColumnPartSerdeV2.java | 188 ++++++++++++++++++ .../serde/DoubleGenericColumnSupplier.java | 10 +- .../serde/FloatGenericColumnPartSerde.java | 9 +- .../serde/FloatGenericColumnPartSerdeV2.java | 187 +++++++++++++++++ .../serde/FloatGenericColumnSupplier.java | 11 +- .../serde/LongGenericColumnPartSerde.java | 8 +- .../serde/LongGenericColumnPartSerdeV2.java | 185 +++++++++++++++++ .../serde/LongGenericColumnSupplier.java | 7 +- 40 files changed, 1065 insertions(+), 119 deletions(-) create mode 100644 processing/src/main/java/io/druid/segment/NullHandlingHelper.java create mode 100644 processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java create mode 100644 processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java create mode 100644 processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java create mode 100644 processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index ce9dbbf2727b..936016b574ba 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -19,7 +19,6 @@ package io.druid.segment; -import com.google.common.base.Strings; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.spatial.ImmutableRTree; @@ -180,7 +179,7 @@ public int getIndex(String value) // Return -2 for non-null values to match what the BitmapIndex implementation in BitmapIndexColumnPartSupplier // would return for getIndex() when there is only a single index, for the null value. // i.e., return an 'insertion point' of 1 for non-null values (see BitmapIndex interface) - return Strings.isNullOrEmpty(value) ? 0 : -2; + return value == null ? 0 : -2; } @Override @@ -210,7 +209,7 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) final Column column = index.getColumn(dimension); if (column == null || !columnSupportsFiltering(column)) { - if (Strings.isNullOrEmpty(value)) { + if (value == null) { return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows()); } else { return bitmapFactory.makeEmptyImmutableBitmap(); diff --git a/processing/src/main/java/io/druid/segment/ColumnValueSelector.java b/processing/src/main/java/io/druid/segment/ColumnValueSelector.java index 2ca6e623b335..0d788a9143d8 100644 --- a/processing/src/main/java/io/druid/segment/ColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnValueSelector.java @@ -33,4 +33,6 @@ public interface ColumnValueSelector double getDouble(); long getLong(); + + boolean isNull(); } diff --git a/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java b/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java index f61e8b6e1ebf..1616605a5b13 100644 --- a/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java @@ -20,7 +20,6 @@ package io.druid.segment; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.IndexedInts; @@ -37,7 +36,7 @@ public class ConstantDimensionSelector implements SingleValueHistoricalDimension public ConstantDimensionSelector(final String value) { - if (Strings.isNullOrEmpty(value)) { + if (NullHandlingHelper.isNullOrDefault(value)) { // There's an optimized implementation for nulls that callers should use instead. throw new IllegalArgumentException("Use NullDimensionSelector or DimensionSelectorUtils.constantSelector"); } diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 494b556fd3c1..f62072da108e 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -86,6 +86,7 @@ public class IndexIO public static final byte V8_VERSION = 0x8; public static final byte V9_VERSION = 0x9; public static final int CURRENT_VERSION_ID = V9_VERSION; + public static BitmapSerdeFactory LEGACY_FACTORY = new BitmapSerde.LegacyBitmapSerdeFactory(); public static final ByteOrder BYTE_ORDER = ByteOrder.nativeOrder(); @@ -162,14 +163,12 @@ public void validateTwoSegments(final IndexableAdapter adapter1, final Indexable if (rb1.getRowNum() != rb2.getRowNum()) { throw new SegmentValidationException("Row number mismatch: [%d] vs [%d]", rb1.getRowNum(), rb2.getRowNum()); } - if (rb1.compareTo(rb2) != 0) { try { validateRowValues(dimHandlers, rb1, adapter1, rb2, adapter2); } catch (SegmentValidationException ex) { throw new SegmentValidationException(ex, "Validation failure on row %d: [%s] vs [%s]", row, rb1, rb2); } - } } if (it2.hasNext()) { throw new SegmentValidationException("Unexpected end of first adapter"); @@ -477,7 +476,12 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException metric, new ColumnBuilder() .setType(ValueType.FLOAT) - .setGenericColumn(new FloatGenericColumnSupplier(metricHolder.floatType, BYTE_ORDER)) + .setGenericColumn(new FloatGenericColumnSupplier( + metricHolder.floatType, + BYTE_ORDER, + LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap() + )) .build() ); } else if (metricHolder.getType() == MetricHolder.MetricType.COMPLEX) { @@ -507,7 +511,11 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException columns.put( Column.TIME_COLUMN_NAME, new ColumnBuilder() .setType(ValueType.LONG) - .setGenericColumn(new LongGenericColumnSupplier(index.timestamps)) + .setGenericColumn(new LongGenericColumnSupplier( + index.timestamps, + LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap() + )) .build() ); return new SimpleQueryableIndex( diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index 7e75122cfba3..bd8abba39cf9 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -20,10 +20,10 @@ package io.druid.segment; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; import com.google.common.collect.PeekingIterator; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; @@ -52,10 +52,10 @@ import java.nio.IntBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.PriorityQueue; import java.util.Set; import java.util.stream.Collectors; @@ -430,16 +430,10 @@ class DictionaryMergeIterator implements CloseableIterator DictionaryMergeIterator(Indexed[] dimValueLookups, boolean useDirect) { + final Ordering stringOrdering = Comparators.naturalNullsFirst(); pQueue = new PriorityQueue<>( dimValueLookups.length, - new Comparator>>() - { - @Override - public int compare(Pair> lhs, Pair> rhs) - { - return lhs.rhs.peek().compareTo(rhs.rhs.peek()); - } - } + (lhs, rhs) -> stringOrdering.compare(lhs.rhs.peek(), rhs.rhs.peek()) ); conversions = new IntBuffer[dimValueLookups.length]; for (int i = 0; i < conversions.length; i++) { @@ -460,14 +454,7 @@ public int compare(Pair> lhs, Pair iter = Iterators.peekingIterator( Iterators.transform( indexed.iterator(), - new Function() - { - @Override - public String apply(@Nullable String input) - { - return Strings.nullToEmpty(input); - } - } + input -> NullHandlingHelper.nullToDefault(input) ) ); if (iter.hasNext()) { @@ -491,7 +478,7 @@ public String next() } final String value = writeTranslate(smallest, counter); - while (!pQueue.isEmpty() && value.equals(pQueue.peek().rhs.peek())) { + while (!pQueue.isEmpty() && Objects.equals(value, pQueue.peek().rhs.peek())) { writeTranslate(pQueue.remove(), counter); } counter++; diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index af59c1522d7e..694c657db390 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -34,11 +34,11 @@ import com.google.common.primitives.Longs; import com.google.inject.Inject; import io.druid.collections.CombiningIterable; -import io.druid.java.util.common.DateTimes; -import io.druid.java.util.common.JodaUtils; import io.druid.io.ZeroCopyByteArrayOutputStream; +import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.JodaUtils; import io.druid.java.util.common.guava.Comparators; import io.druid.java.util.common.guava.FunctionalIterable; import io.druid.java.util.common.guava.MergeIterable; @@ -64,9 +64,9 @@ import io.druid.segment.serde.ComplexColumnPartSerde; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; -import io.druid.segment.serde.DoubleGenericColumnPartSerde; -import io.druid.segment.serde.FloatGenericColumnPartSerde; -import io.druid.segment.serde.LongGenericColumnPartSerde; +import io.druid.segment.serde.DoubleGenericColumnPartSerdeV2; +import io.druid.segment.serde.FloatGenericColumnPartSerdeV2; +import io.druid.segment.serde.LongGenericColumnPartSerdeV2; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntSortedSet; @@ -369,7 +369,7 @@ private void makeMetricsColumns( case LONG: builder.setValueType(ValueType.LONG); builder.addSerde( - LongGenericColumnPartSerde + LongGenericColumnPartSerdeV2 .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((LongColumnSerializer) writer) @@ -379,7 +379,7 @@ private void makeMetricsColumns( case FLOAT: builder.setValueType(ValueType.FLOAT); builder.addSerde( - FloatGenericColumnPartSerde + FloatGenericColumnPartSerdeV2 .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((FloatColumnSerializer) writer) @@ -389,7 +389,7 @@ private void makeMetricsColumns( case DOUBLE: builder.setValueType(ValueType.DOUBLE); builder.addSerde( - DoubleGenericColumnPartSerde + DoubleGenericColumnPartSerdeV2 .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((DoubleColumnSerializer) writer) @@ -433,10 +433,10 @@ private void makeTimeColumn( .builder() .setValueType(ValueType.LONG) .addSerde( - LongGenericColumnPartSerde.serializerBuilder() - .withByteOrder(IndexIO.BYTE_ORDER) - .withDelegate(timeWriter) - .build() + LongGenericColumnPartSerdeV2.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withDelegate(timeWriter) + .build() ) .build(); makeColumn(v9Smoosher, Column.TIME_COLUMN_NAME, serdeficator); diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java new file mode 100644 index 000000000000..dac18d223854 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -0,0 +1,106 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.google.common.base.Strings; +import com.google.inject.Inject; +import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.DoubleAggregateCombiner; +import io.druid.query.aggregation.LongAggregateCombiner; +import io.druid.query.aggregation.NullableAggregator; +import io.druid.query.aggregation.NullableBufferAggregator; +import io.druid.query.aggregation.NullableDoubleAggregateCombiner; +import io.druid.query.aggregation.NullableLongAggregateCombiner; +import io.druid.query.aggregation.NullableObjectAggregateCombiner; +import io.druid.query.aggregation.ObjectAggregateCombiner; + +public class NullHandlingHelper +{ + // use these values to ensure that convertObjectToLong(), convertObjectToDouble() and convertObjectToFloat() + // return the same boxed object when returning a constant zero. + public static final Double ZERO_DOUBLE = 0.0d; + public static final Float ZERO_FLOAT = 0.0f; + public static final Long ZERO_LONG = 0L; + + // Using static injection to avoid adding JacksonInject annotations all over the code. + @Inject + private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig(false); + + public static boolean useDefaultValuesForNull() + { + return INSTANCE.isUseDefaultValuesForNull(); + } + + public static String nullToDefault(String value) + { + return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; + } + + public static String defaultToNull(String value) + { + return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; + } + + public static boolean isNullOrDefault(String value) + { + return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; + } + + public static Long nullToDefault(Long value) + { + return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_LONG : value; + } + + public static Double nullToDefault(Double value) + { + return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_DOUBLE : value; + } + + public static Float nullToDefault(Float value) + { + return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_FLOAT : value; + } + + public static Aggregator getNullableAggregator(Aggregator aggregator, ColumnValueSelector selector) + { + return INSTANCE.isUseDefaultValuesForNull() ? aggregator : new NullableAggregator(aggregator, selector); + } + + public static BufferAggregator getNullableAggregator(BufferAggregator aggregator, ColumnValueSelector selector) + { + return INSTANCE.isUseDefaultValuesForNull() ? aggregator : new NullableBufferAggregator(aggregator, selector); + } + + public static DoubleAggregateCombiner getNullableCombiner(DoubleAggregateCombiner combiner) + { + return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableDoubleAggregateCombiner(combiner); + } + + public static LongAggregateCombiner getNullableCombiner(LongAggregateCombiner combiner) + { + return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableLongAggregateCombiner(combiner); + } + + public static ObjectAggregateCombiner getNullableCombiner(ObjectAggregateCombiner combiner) + { + return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableObjectAggregateCombiner(combiner); + } +} diff --git a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java b/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java new file mode 100644 index 000000000000..8104aefd1663 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java @@ -0,0 +1,45 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class NullValueHandlingConfig +{ + private static boolean DEFAULT_USE_DEFAULT_VALUES_FOR_NULL = true; + + @JsonProperty("useDefaultValueForNull") + private Boolean useDefaultValuesForNull = DEFAULT_USE_DEFAULT_VALUES_FOR_NULL; + + @JsonCreator + public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) + { + this.useDefaultValuesForNull = useDefaultValuesForNull == null + ? DEFAULT_USE_DEFAULT_VALUES_FOR_NULL + : useDefaultValuesForNull; + } + + + public boolean isUseDefaultValuesForNull() + { + return useDefaultValuesForNull; + } +} diff --git a/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java index 498aa6273752..92e946d9a5e8 100644 --- a/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java @@ -80,4 +80,16 @@ default long getLong() } return ((Number) value).longValue(); } + + /** + * @deprecated This method is marked as deprecated in ObjectColumnSelector to minimize the probability of accidential + * calling. "Polymorphism" of ObjectColumnSelector should be used only when operating on {@link ColumnValueSelector} + * objects. + */ + @Deprecated + default boolean isNull() + { + T value = get(); + return value == null; + } } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java index bfe510c6f781..516f3c18c610 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java @@ -72,6 +72,20 @@ public int compareSortedEncodedKeyComponents(int[] lhs, int[] rhs) return retVal; } + public boolean isNUllRow(int[] row, Indexed encodings) + { + if (row == null) { + return true; + } + for (int i : row) { + if (encodings.get(i) != null) { + // Non-Null value + return false; + } + } + return true; + } + @Override public void validateSortedEncodedKeyComponents( int[] lhs, @@ -81,7 +95,7 @@ public void validateSortedEncodedKeyComponents( ) throws SegmentValidationException { if (lhs == null || rhs == null) { - if (lhs != null || rhs != null) { + if (!isNUllRow(lhs, lhsEncodings) || !isNUllRow(rhs, rhsEncodings)) { throw new SegmentValidationException( "Expected nulls, found %s and %s", Arrays.toString(lhs), diff --git a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java index 5e2141cd3b93..bbf56499fc01 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; /** @@ -29,6 +30,7 @@ public class ColumnBuilder { private ValueType type = null; private boolean hasMultipleValues = false; + private boolean hasNullValues = false; private Supplier dictionaryEncodedColumn = null; private Supplier runLengthColumn = null; @@ -37,6 +39,7 @@ public class ColumnBuilder private Supplier bitmapIndex = null; private Supplier spatialIndex = null; private SmooshedFileMapper fileMapper = null; + private Supplier nullValueBitmap = null; public ColumnBuilder setFileMapper(SmooshedFileMapper fileMapper) { @@ -97,6 +100,23 @@ public ColumnBuilder setSpatialIndex(Supplier spatialIndex) return this; } + public ColumnBuilder setHasNullValues(boolean hasNullValues) + { + this.hasNullValues = hasNullValues; + return this; + } + + public boolean isHasNullValues() + { + return hasNullValues; + } + + public ColumnBuilder setNullValueBitmap(Supplier nullValueBitmap) + { + this.nullValueBitmap = nullValueBitmap; + return this; + } + public Column build() { Preconditions.checkState(type != null, "Type must be set."); @@ -108,13 +128,15 @@ public Column build() .setHasBitmapIndexes(bitmapIndex != null) .setHasSpatialIndexes(spatialIndex != null) .setRunLengthEncoded(runLengthColumn != null) - .setHasMultipleValues(hasMultipleValues), + .setHasMultipleValues(hasMultipleValues) + .setHasNullValues(hasNullValues), dictionaryEncodedColumn, runLengthColumn, genericColumn, complexColumn, bitmapIndex, - spatialIndex + spatialIndex, + nullValueBitmap ); } } diff --git a/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java b/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java index f6b416415326..ea88a8985e0e 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java @@ -31,5 +31,7 @@ public interface ColumnCapabilities public boolean hasSpatialIndexes(); public boolean hasMultipleValues(); + public boolean hasNullValues(); + public ColumnCapabilitiesImpl merge(ColumnCapabilities other); } diff --git a/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java b/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java index fff3d1c05dc4..2ba4bd7a5389 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java @@ -32,6 +32,7 @@ public class ColumnCapabilitiesImpl implements ColumnCapabilities private boolean hasInvertedIndexes = false; private boolean hasSpatialIndexes = false; private boolean hasMultipleValues = false; + private boolean hasNullValues = false; @Override @JsonProperty @@ -111,6 +112,19 @@ public ColumnCapabilitiesImpl setHasMultipleValues(boolean hasMultipleValues) return this; } + @Override + @JsonProperty("hasNullValues") + public boolean hasNullValues() + { + return hasNullValues; + } + + public ColumnCapabilitiesImpl setHasNullValues(boolean hasNullValues) + { + this.hasNullValues = hasNullValues; + return this; + } + @Override public ColumnCapabilitiesImpl merge(ColumnCapabilities other) { @@ -131,6 +145,7 @@ public ColumnCapabilitiesImpl merge(ColumnCapabilities other) this.hasInvertedIndexes |= other.hasBitmapIndexes(); this.hasSpatialIndexes |= other.hasSpatialIndexes(); this.hasMultipleValues |= other.hasMultipleValues(); + this.hasNullValues |= other.hasNullValues(); return this; } diff --git a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java index 7119012abb83..b51f41fb7612 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java @@ -44,17 +44,20 @@ public static Builder builder() private final ValueType valueType; private final boolean hasMultipleValues; + private final boolean hasNullValues; private final List parts; @JsonCreator public ColumnDescriptor( @JsonProperty("valueType") ValueType valueType, @JsonProperty("hasMultipleValues") boolean hasMultipleValues, + @JsonProperty("hasNullValues") boolean hasNullValues, @JsonProperty("parts") List parts ) { this.valueType = valueType; this.hasMultipleValues = hasMultipleValues; + this.hasNullValues = hasNullValues; this.parts = parts; } @@ -70,6 +73,12 @@ public boolean isHasMultipleValues() return hasMultipleValues; } + @JsonProperty + public boolean isHasNullValues() + { + return hasNullValues; + } + @JsonProperty public List getParts() { @@ -99,6 +108,7 @@ public Column read(ByteBuffer buffer, ColumnConfig columnConfig, SmooshedFileMap final ColumnBuilder builder = new ColumnBuilder() .setType(valueType) .setHasMultipleValues(hasMultipleValues) + .setHasNullValues(hasNullValues) .setFileMapper(smooshedFiles); for (ColumnPartSerde part : parts) { @@ -112,6 +122,7 @@ public static class Builder { private ValueType valueType = null; private Boolean hasMultipleValues = null; + private boolean hasNullValues = false; private final List parts = Lists.newArrayList(); @@ -135,6 +146,12 @@ public Builder setHasMultipleValues(boolean hasMultipleValues) return this; } + public Builder setHasNullValues(boolean hasNullValues) + { + this.hasNullValues = hasNullValues || this.hasNullValues; + return this; + } + public Builder addSerde(ColumnPartSerde serde) { parts.add(serde); @@ -144,7 +161,12 @@ public Builder addSerde(ColumnPartSerde serde) public ColumnDescriptor build() { Preconditions.checkNotNull(valueType, "must specify a valueType"); - return new ColumnDescriptor(valueType, hasMultipleValues == null ? false : hasMultipleValues, parts); + return new ColumnDescriptor( + valueType, + hasMultipleValues == null ? false : hasMultipleValues, + hasNullValues, + parts + ); } } } diff --git a/processing/src/main/java/io/druid/segment/column/DoubleColumn.java b/processing/src/main/java/io/druid/segment/column/DoubleColumn.java index 3d53cded9942..654495a3bd55 100644 --- a/processing/src/main/java/io/druid/segment/column/DoubleColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DoubleColumn.java @@ -20,6 +20,7 @@ package io.druid.segment.column; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.data.CompressedDoublesIndexedSupplier; public class DoubleColumn extends AbstractColumn @@ -29,11 +30,17 @@ public class DoubleColumn extends AbstractColumn private static final ColumnCapabilitiesImpl CAPABILITIES = new ColumnCapabilitiesImpl() .setType(ValueType.DOUBLE); + private static final ColumnCapabilitiesImpl CAPABILITIES_WITH_NULL = new ColumnCapabilitiesImpl() + .setType(ValueType.DOUBLE).setHasNullValues(true); + private final CompressedDoublesIndexedSupplier column; + private final ImmutableBitmap nullValueBitmap; + - public DoubleColumn(CompressedDoublesIndexedSupplier column) + public DoubleColumn(CompressedDoublesIndexedSupplier column, ImmutableBitmap nullValueBitmap) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override @@ -45,12 +52,13 @@ public int getLength() @Override public ColumnCapabilities getCapabilities() { - return CAPABILITIES; + return nullValueBitmap.isEmpty() ? CAPABILITIES : CAPABILITIES_WITH_NULL; } @Override public GenericColumn getGenericColumn() { - return new IndexedDoublesGenericColumn(column.get()); + return new IndexedDoublesGenericColumn(column.get(), nullValueBitmap); } + } diff --git a/processing/src/main/java/io/druid/segment/column/FloatColumn.java b/processing/src/main/java/io/druid/segment/column/FloatColumn.java index ad130807b2fb..8d16a8761b1a 100644 --- a/processing/src/main/java/io/druid/segment/column/FloatColumn.java +++ b/processing/src/main/java/io/druid/segment/column/FloatColumn.java @@ -19,6 +19,7 @@ package io.druid.segment.column; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.data.CompressedFloatsIndexedSupplier; /** @@ -29,18 +30,23 @@ public class FloatColumn extends AbstractColumn private static final ColumnCapabilitiesImpl CAPABILITIES = new ColumnCapabilitiesImpl() .setType(ValueType.FLOAT); + private static final ColumnCapabilitiesImpl CAPABILITIES_WITH_NULL = new ColumnCapabilitiesImpl() + .setType(ValueType.FLOAT).setHasNullValues(true); private final CompressedFloatsIndexedSupplier column; + private final ImmutableBitmap nullValueBitmap; - public FloatColumn(CompressedFloatsIndexedSupplier column) + public FloatColumn(CompressedFloatsIndexedSupplier column, ImmutableBitmap nullValueBitmap) { this.column = column; + this.nullValueBitmap = nullValueBitmap; + } @Override public ColumnCapabilities getCapabilities() { - return CAPABILITIES; + return nullValueBitmap.isEmpty() ? CAPABILITIES : CAPABILITIES_WITH_NULL; } @Override @@ -52,6 +58,6 @@ public int getLength() @Override public GenericColumn getGenericColumn() { - return new IndexedFloatsGenericColumn(column.get()); + return new IndexedFloatsGenericColumn(column.get(), nullValueBitmap); } } diff --git a/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java index 47fd8f50c4ae..b028e2903597 100644 --- a/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java @@ -19,6 +19,7 @@ package io.druid.segment.column; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DoubleColumnSelector; import io.druid.segment.FloatColumnSelector; @@ -30,10 +31,12 @@ public class IndexedDoublesGenericColumn implements GenericColumn { private final IndexedDoubles column; + private final ImmutableBitmap nullValueBitmap; - public IndexedDoublesGenericColumn(IndexedDoubles indexedDoubles) + public IndexedDoublesGenericColumn(IndexedDoubles indexedDoubles, ImmutableBitmap nullValueBitmap) { column = indexedDoubles; + this.nullValueBitmap = nullValueBitmap; } @Override @@ -69,7 +72,7 @@ public float getFloatSingleValueRow(int rowNum) @Override public FloatColumnSelector makeFloatSingleValueRowSelector(ReadableOffset offset) { - return column.makeFloatColumnSelector(offset); + return column.makeFloatColumnSelector(offset, nullValueBitmap); } @Override @@ -81,7 +84,7 @@ public long getLongSingleValueRow(int rowNum) @Override public LongColumnSelector makeLongSingleValueRowSelector(ReadableOffset offset) { - return column.makeLongColumnSelector(offset); + return column.makeLongColumnSelector(offset, nullValueBitmap); } @Override @@ -93,7 +96,7 @@ public double getDoubleSingleValueRow(int rowNum) @Override public DoubleColumnSelector makeDoubleSingleValueRowSelector(ReadableOffset offset) { - return column.makeDoubleColumnSelector(offset); + return column.makeDoubleColumnSelector(offset, nullValueBitmap); } @Override diff --git a/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java index 1841fbf2c13d..70fe5872d389 100644 --- a/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java @@ -19,6 +19,7 @@ package io.druid.segment.column; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DoubleColumnSelector; import io.druid.segment.FloatColumnSelector; @@ -31,10 +32,15 @@ public class IndexedFloatsGenericColumn implements GenericColumn { private final IndexedFloats column; + private final ImmutableBitmap nullValueBitmap; - public IndexedFloatsGenericColumn(final IndexedFloats column) + public IndexedFloatsGenericColumn( + final IndexedFloats column, + ImmutableBitmap nullValueBitmap + ) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override @@ -70,7 +76,7 @@ public float getFloatSingleValueRow(int rowNum) @Override public FloatColumnSelector makeFloatSingleValueRowSelector(ReadableOffset offset) { - return column.makeFloatColumnSelector(offset); + return column.makeFloatColumnSelector(offset, nullValueBitmap); } @Override @@ -82,7 +88,7 @@ public long getLongSingleValueRow(int rowNum) @Override public LongColumnSelector makeLongSingleValueRowSelector(ReadableOffset offset) { - return column.makeLongColumnSelector(offset); + return column.makeLongColumnSelector(offset, nullValueBitmap); } @Override @@ -94,7 +100,7 @@ public double getDoubleSingleValueRow(int rowNum) @Override public DoubleColumnSelector makeDoubleSingleValueRowSelector(ReadableOffset offset) { - return column.makeDoubleColumnSelector(offset); + return column.makeDoubleColumnSelector(offset, nullValueBitmap); } @Override diff --git a/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java index bb1a035affa7..14b758dc5130 100644 --- a/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java @@ -19,6 +19,7 @@ package io.druid.segment.column; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DoubleColumnSelector; import io.druid.segment.FloatColumnSelector; @@ -31,10 +32,15 @@ public class IndexedLongsGenericColumn implements GenericColumn { private final IndexedLongs column; + private final ImmutableBitmap nullValueBitmap; - public IndexedLongsGenericColumn(final IndexedLongs column) + public IndexedLongsGenericColumn( + final IndexedLongs column, + ImmutableBitmap nullValueBitmap + ) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override @@ -70,7 +76,7 @@ public float getFloatSingleValueRow(int rowNum) @Override public FloatColumnSelector makeFloatSingleValueRowSelector(ReadableOffset offset) { - return column.makeFloatColumnSelector(offset); + return column.makeFloatColumnSelector(offset, nullValueBitmap); } @Override @@ -82,7 +88,7 @@ public long getLongSingleValueRow(int rowNum) @Override public LongColumnSelector makeLongSingleValueRowSelector(ReadableOffset offset) { - return column.makeLongColumnSelector(offset); + return column.makeLongColumnSelector(offset, nullValueBitmap); } @Override @@ -94,7 +100,7 @@ public double getDoubleSingleValueRow(int rowNum) @Override public DoubleColumnSelector makeDoubleSingleValueRowSelector(ReadableOffset offset) { - return column.makeDoubleColumnSelector(offset); + return column.makeDoubleColumnSelector(offset, nullValueBitmap); } @Override diff --git a/processing/src/main/java/io/druid/segment/column/LongColumn.java b/processing/src/main/java/io/druid/segment/column/LongColumn.java index bad4736d4e7f..d481275de212 100644 --- a/processing/src/main/java/io/druid/segment/column/LongColumn.java +++ b/processing/src/main/java/io/druid/segment/column/LongColumn.java @@ -19,6 +19,7 @@ package io.druid.segment.column; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.data.CompressedLongsIndexedSupplier; /** @@ -30,17 +31,22 @@ public class LongColumn extends AbstractColumn private static final ColumnCapabilitiesImpl CAPABILITIES = new ColumnCapabilitiesImpl() .setType(ValueType.LONG); + private static final ColumnCapabilitiesImpl CAPABILITIES_WITH_NULL = new ColumnCapabilitiesImpl() + .setType(ValueType.LONG); + private final CompressedLongsIndexedSupplier column; + private final ImmutableBitmap nullValueBitmap; - public LongColumn(CompressedLongsIndexedSupplier column) + public LongColumn(CompressedLongsIndexedSupplier column, ImmutableBitmap nullValueBitmap) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override public ColumnCapabilities getCapabilities() { - return CAPABILITIES; + return nullValueBitmap.isEmpty() ? CAPABILITIES : CAPABILITIES_WITH_NULL; } @Override @@ -52,6 +58,7 @@ public int getLength() @Override public GenericColumn getGenericColumn() { - return new IndexedLongsGenericColumn(column.get()); + return new IndexedLongsGenericColumn(column.get(), nullValueBitmap); } + } diff --git a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java index 188b49750e53..2017d58c1774 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java @@ -20,6 +20,7 @@ package io.druid.segment.column; import com.google.common.base.Supplier; +import io.druid.collections.bitmap.ImmutableBitmap; /** */ @@ -32,6 +33,7 @@ class SimpleColumn implements Column private final Supplier complexColumn; private final Supplier bitmapIndex; private final Supplier spatialIndex; + private final Supplier nullValueBitmap; SimpleColumn( ColumnCapabilities capabilities, @@ -40,7 +42,8 @@ class SimpleColumn implements Column Supplier genericColumn, Supplier complexColumn, Supplier bitmapIndex, - Supplier spatialIndex + Supplier spatialIndex, + Supplier nullValueBitmap ) { this.capabilities = capabilities; @@ -50,6 +53,7 @@ class SimpleColumn implements Column this.complexColumn = complexColumn; this.bitmapIndex = bitmapIndex; this.spatialIndex = spatialIndex; + this.nullValueBitmap = nullValueBitmap; } @Override diff --git a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java index fb39014125a2..a91314db963d 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java @@ -21,7 +21,6 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.base.Strings; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.ValueMatcher; @@ -42,7 +41,7 @@ import java.util.BitSet; /** -*/ + */ public class SimpleDictionaryEncodedColumn implements DictionaryEncodedColumn { @@ -88,8 +87,7 @@ public IndexedInts getMultiValueRow(int rowNum) @Override public String lookupName(int id) { - //Empty to Null will ensure that null and empty are equivalent for extraction function - return Strings.emptyToNull(cachedLookups.get(id)); + return cachedLookups.get(id); } @Override diff --git a/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java b/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java index e0035aeaecdf..650b9a338848 100644 --- a/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java @@ -32,6 +32,9 @@ public class ByteBufferSerializer public static T read(ByteBuffer buffer, ObjectStrategy strategy) { int size = buffer.getInt(); + if (size < 0) { + return null; + } ByteBuffer bufferToUse = buffer.asReadOnlyBuffer(); bufferToUse.limit(bufferToUse.position() + size); buffer.position(bufferToUse.limit()); @@ -43,7 +46,10 @@ public static void writeToChannel(T obj, ObjectStrategy strategy, Writabl throws IOException { byte[] toWrite = strategy.toBytes(obj); - channel.write(ByteBuffer.allocate(Ints.BYTES).putInt(0, toWrite.length)); - channel.write(ByteBuffer.wrap(toWrite)); + int size = toWrite == null ? -1 : toWrite.length; + channel.write(ByteBuffer.allocate(Ints.BYTES).putInt(0, size)); + if (size > 0) { + channel.write(ByteBuffer.wrap(toWrite)); + } } } diff --git a/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java b/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java index 753455888013..3b94f0f66c7a 100644 --- a/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java +++ b/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java @@ -71,8 +71,11 @@ public void open() throws IOException public void write(T objectToWrite) throws IOException { byte[] bytesToWrite = strategy.toBytes(objectToWrite); - SerializerUtils.writeBigEndianIntToOutputStream(headerOut, bytesToWrite.length, helperBuffer); - valueOut.write(bytesToWrite); + int length = bytesToWrite == null ? -1 : bytesToWrite.length; + SerializerUtils.writeBigEndianIntToOutputStream(headerOut, length, helperBuffer); + if (bytesToWrite != null) { + valueOut.write(bytesToWrite); + } } private String makeFilename(String suffix) diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index e5a4f9a9bd33..4f2a56859c18 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import com.google.common.io.CountingOutputStream; import com.google.common.primitives.Ints; import io.druid.common.utils.SerializerUtils; import io.druid.io.ZeroCopyByteArrayOutputStream; @@ -28,7 +29,7 @@ import io.druid.java.util.common.guava.Comparators; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import it.unimi.dsi.fastutil.bytes.ByteArrays; +import io.druid.segment.NullHandlingHelper; import java.io.Closeable; import java.io.IOException; @@ -86,15 +87,15 @@ public Class getClazz() @Override public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) { - return StringUtils.fromUtf8(buffer, numBytes); + if (numBytes < 0) { + return null; + } + return NullHandlingHelper.defaultToNull(StringUtils.fromUtf8(buffer, numBytes)); } @Override public byte[] toBytes(String val) { - if (val == null) { - return ByteArrays.EMPTY_ARRAY; - } return StringUtils.toUtf8(val); } @@ -279,8 +280,6 @@ private int indexOf(Indexed indexed, T value) throw new UnsupportedOperationException("Reverse lookup not allowed."); } - value = (value != null && value.equals("")) ? null : value; - int minIndex = 0; int maxIndex = size - 1; while (minIndex <= maxIndex) { @@ -338,11 +337,8 @@ public GenericIndexed.BufferIndexed singleThreaded() private T copyBufferAndGet(ByteBuffer valueBuffer, int startOffset, int endOffset) { - final int size = endOffset - startOffset; - if (size == 0) { - return null; - } ByteBuffer copyValueBuffer = valueBuffer.asReadOnlyBuffer(); + final int size = endOffset > startOffset ? endOffset - startOffset : copyValueBuffer.get(startOffset - Ints.BYTES); copyValueBuffer.position(startOffset); // fromByteBuffer must not modify the buffer limit return strategy.fromByteBuffer(copyValueBuffer, size); @@ -381,11 +377,11 @@ public int size() T bufferedIndexedGet(ByteBuffer copyValueBuffer, int startOffset, int endOffset) { - final int size = endOffset - startOffset; + final int size = endOffset > startOffset + ? endOffset - startOffset + : copyValueBuffer.get(startOffset - Ints.BYTES); lastReadSize = size; - if (size == 0) { - return null; - } + // ObjectStrategy.fromByteBuffer() is allowed to reset the limit of the buffer. So if the limit is changed, // position() call in the next line could throw an exception, if the position is set beyond the new limit. clear() // sets the limit to the maximum possible, the capacity. It is safe to reset the limit to capacity, because the @@ -451,9 +447,9 @@ private static GenericIndexed fromIterableVersionOne(Iterable objectsI ZeroCopyByteArrayOutputStream headerBytes = new ZeroCopyByteArrayOutputStream(); ZeroCopyByteArrayOutputStream valueBytes = new ZeroCopyByteArrayOutputStream(); + CountingOutputStream valueCountingBytes = new CountingOutputStream(valueBytes); ByteBuffer helperBuffer = ByteBuffer.allocate(Ints.BYTES); try { - int offset = 0; T prevVal = null; do { count++; @@ -463,10 +459,19 @@ private static GenericIndexed fromIterableVersionOne(Iterable objectsI } final byte[] bytes = strategy.toBytes(next); - offset += Ints.BYTES + bytes.length; - SerializerUtils.writeBigEndianIntToOutputStream(headerBytes, offset, helperBuffer); - SerializerUtils.writeBigEndianIntToOutputStream(valueBytes, bytes.length, helperBuffer); - valueBytes.write(bytes); + + + int size = bytes == null ? -1 : bytes.length; + SerializerUtils.writeBigEndianIntToOutputStream(valueCountingBytes, size, helperBuffer); + if (bytes != null) { + valueCountingBytes.write(bytes); + } + + SerializerUtils.writeBigEndianIntToOutputStream( + headerBytes, + Ints.checkedCast(valueCountingBytes.getCount()), + helperBuffer + ); if (prevVal instanceof Closeable) { CloseQuietly.close((Closeable) prevVal); @@ -532,7 +537,7 @@ public T get(final int index) final int endOffset; if (index == 0) { - startOffset = 4; + startOffset = Ints.BYTES; endOffset = headerBuffer.getInt(0); } else { int headerPosition = (index - 1) * Ints.BYTES; diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java index 82177a0e12af..383ee2c0d0fb 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java @@ -138,10 +138,12 @@ public void write(T objectToWrite) throws IOException } byte[] bytesToWrite = strategy.toBytes(objectToWrite); - + int size = bytesToWrite == null ? -1 : bytesToWrite.length; ++numWritten; - SerializerUtils.writeBigEndianIntToOutputStream(valuesOut, bytesToWrite.length, sizeHelperBuffer); - valuesOut.write(bytesToWrite); + SerializerUtils.writeBigEndianIntToOutputStream(valuesOut, size, sizeHelperBuffer); + if (bytesToWrite != null) { + valuesOut.write(bytesToWrite); + } if (!requireMultipleFiles) { SerializerUtils.writeBigEndianIntToOutputStream(headerOut, Ints.checkedCast(valuesOut.getCount()), buf); diff --git a/processing/src/main/java/io/druid/segment/data/IndexedDoubles.java b/processing/src/main/java/io/druid/segment/data/IndexedDoubles.java index 222b8585bafc..bf60308a1d90 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedDoubles.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedDoubles.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DoubleColumnSelector; import io.druid.segment.FloatColumnSelector; @@ -30,13 +31,15 @@ public interface IndexedDoubles extends Closeable { public int size(); + public double get(int index); + public void fill(int index, double[] toFill); @Override void close(); - default DoubleColumnSelector makeDoubleColumnSelector(ReadableOffset offset) + default DoubleColumnSelector makeDoubleColumnSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalDoubleColumnSelector implements DoubleColumnSelector, HistoricalColumnSelector { @@ -52,6 +55,12 @@ public double getDouble(int offset) return IndexedDoubles.this.get(offset); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -62,10 +71,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) return new HistoricalDoubleColumnSelector(); } - default FloatColumnSelector makeFloatColumnSelector(ReadableOffset offset) + + default FloatColumnSelector makeFloatColumnSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalFloatColumnSelector implements FloatColumnSelector, HistoricalColumnSelector { + @Override public float getFloat() { @@ -78,6 +89,12 @@ public double getDouble(int offset) return IndexedDoubles.this.get(offset); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -88,7 +105,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) return new HistoricalFloatColumnSelector(); } - default LongColumnSelector makeLongColumnSelector(ReadableOffset offset) + default LongColumnSelector makeLongColumnSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalLongColumnSelector implements LongColumnSelector, HistoricalColumnSelector { @@ -104,6 +121,12 @@ public double getDouble(int offset) return IndexedDoubles.this.get(offset); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/data/IndexedFloats.java b/processing/src/main/java/io/druid/segment/data/IndexedFloats.java index 38ebc6a6c608..8a5d605b27eb 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedFloats.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedFloats.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DoubleColumnSelector; import io.druid.segment.FloatColumnSelector; @@ -33,13 +34,15 @@ public interface IndexedFloats extends Closeable { public int size(); + public float get(int index); + public void fill(int index, float[] toFill); @Override void close(); - default FloatColumnSelector makeFloatColumnSelector(ReadableOffset offset) + default FloatColumnSelector makeFloatColumnSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalFloatColumnSelector implements FloatColumnSelector, HistoricalColumnSelector { @@ -49,6 +52,12 @@ public float getFloat() return IndexedFloats.this.get(offset.getOffset()); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public double getDouble(int offset) { @@ -65,7 +74,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) return new HistoricalFloatColumnSelector(); } - default DoubleColumnSelector makeDoubleColumnSelector(ReadableOffset offset) + default DoubleColumnSelector makeDoubleColumnSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalDoubleColumnSelector implements DoubleColumnSelector, HistoricalColumnSelector { @@ -81,6 +90,12 @@ public double getDouble(int offset) return IndexedFloats.this.get(offset); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -91,7 +106,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) return new HistoricalDoubleColumnSelector(); } - default LongColumnSelector makeLongColumnSelector(ReadableOffset offset) + default LongColumnSelector makeLongColumnSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalLongColumnSelector implements LongColumnSelector, HistoricalColumnSelector { @@ -107,6 +122,12 @@ public double getDouble(int offset) return IndexedFloats.this.get(offset); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/data/IndexedLongs.java b/processing/src/main/java/io/druid/segment/data/IndexedLongs.java index 62bbf61af4da..0e976d7f3be7 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedLongs.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedLongs.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DoubleColumnSelector; import io.druid.segment.FloatColumnSelector; @@ -39,7 +40,7 @@ public interface IndexedLongs extends Closeable @Override void close(); - default LongColumnSelector makeLongColumnSelector(ReadableOffset offset) + default LongColumnSelector makeLongColumnSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalLongColumnSelector implements LongColumnSelector, HistoricalColumnSelector { @@ -55,6 +56,12 @@ public double getDouble(int offset) return IndexedLongs.this.get(offset); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -65,7 +72,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) return new HistoricalLongColumnSelector(); } - default FloatColumnSelector makeFloatColumnSelector(ReadableOffset offset) + default FloatColumnSelector makeFloatColumnSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalFloatColumnSelector implements FloatColumnSelector, HistoricalColumnSelector { @@ -75,6 +82,12 @@ public float getFloat() return (float) IndexedLongs.this.get(offset.getOffset()); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public double getDouble(int offset) { @@ -91,7 +104,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) return new HistoricalFloatColumnSelector(); } - default DoubleColumnSelector makeDoubleColumnSelector(ReadableOffset offset) + default DoubleColumnSelector makeDoubleColumnSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalDoubleColumnSelector implements DoubleColumnSelector, HistoricalColumnSelector { @@ -107,6 +120,12 @@ public double getDouble(int offset) return IndexedLongs.this.get(offset); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java index dc7c45ad70af..d31ec275a322 100644 --- a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java @@ -102,6 +102,9 @@ public Class getClazz() @Override public ImmutableBitmap fromByteBuffer(ByteBuffer buffer, int numBytes) { + if (numBytes == 0) { + return null; + } buffer.limit(buffer.position() + numBytes); return new WrappedImmutableRoaringBitmap(new ImmutableRoaringBitmap(buffer)); } diff --git a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java index 6586d2b491f3..8acede584b93 100644 --- a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java @@ -37,7 +37,10 @@ @JsonSubTypes.Type(name = "float", value = FloatGenericColumnPartSerde.class), @JsonSubTypes.Type(name = "long", value = LongGenericColumnPartSerde.class), @JsonSubTypes.Type(name = "double", value = DoubleGenericColumnPartSerde.class), - @JsonSubTypes.Type(name = "stringDictionary", value = DictionaryEncodedColumnPartSerde.class) + @JsonSubTypes.Type(name = "stringDictionary", value = DictionaryEncodedColumnPartSerde.class), + @JsonSubTypes.Type(name = "floatV2", value = FloatGenericColumnPartSerdeV2.class), + @JsonSubTypes.Type(name = "longV2", value = LongGenericColumnPartSerdeV2.class), + @JsonSubTypes.Type(name = "doubleV2", value = DoubleGenericColumnPartSerdeV2.class), }) public interface ColumnPartSerde { diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java index f6f78e202830..18e10e5386bc 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.DoubleColumnSerializer; +import io.druid.segment.IndexIO; import io.druid.segment.column.ValueType; import io.druid.segment.data.CompressedDoublesIndexedSupplier; @@ -74,7 +75,11 @@ public Deserializer getDeserializer() ); builder.setType(ValueType.DOUBLE) .setHasMultipleValues(false) - .setGenericColumn(new DoubleGenericColumnSupplier(column)); + .setGenericColumn(new DoubleGenericColumnSupplier( + column, + IndexIO.LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap() + )); }; } diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java new file mode 100644 index 000000000000..d9f048ef1280 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -0,0 +1,188 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Suppliers; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.DoubleColumnSerializer; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.BitmapSerde; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferWriter; +import io.druid.segment.data.CompressedDoublesIndexedSupplier; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.WritableByteChannel; + +public class DoubleGenericColumnPartSerdeV2 implements ColumnPartSerde +{ + static final byte VERSION_ONE = 0x1; + private final ByteOrder byteOrder; + private Serializer serialize; + private final BitmapSerdeFactory bitmapSerdeFactory; + + @JsonCreator + public static DoubleGenericColumnPartSerdeV2 getDoubleGenericColumnPartSerde( + @JsonProperty("byteOrder") ByteOrder byteOrder, + @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + ) + { + return new DoubleGenericColumnPartSerdeV2(byteOrder, + bitmapSerdeFactory != null + ? bitmapSerdeFactory + : new BitmapSerde.LegacyBitmapSerdeFactory(), null + ); + } + + @JsonProperty + public ByteOrder getByteOrder() + { + return byteOrder; + } + + + public DoubleGenericColumnPartSerdeV2( + ByteOrder byteOrder, + BitmapSerdeFactory bitmapSerdeFactory, + Serializer serialize + ) + { + this.byteOrder = byteOrder; + this.bitmapSerdeFactory = bitmapSerdeFactory; + this.serialize = serialize; + } + + @Override + public Serializer getSerializer() + { + return serialize; + } + + @Override + public Deserializer getDeserializer() + { + return (buffer, builder, columnConfig) -> { + byte versionFromBuffer = buffer.get(); + + if (VERSION_ONE == versionFromBuffer) { + + int offset = buffer.getInt(); + int initialPos = buffer.position(); + final CompressedDoublesIndexedSupplier column = CompressedDoublesIndexedSupplier.fromByteBuffer( + buffer, + byteOrder, + builder.getFileMapper() + ); + + buffer.position(initialPos + offset); + final ImmutableBitmap bitmap; + if (buffer.hasRemaining()) { + int bitmapSize = buffer.getInt(); + bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBuffer(buffer, bitmapSize); + builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); + } else { + bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); + } + builder.setType(ValueType.DOUBLE) + .setHasMultipleValues(false) + .setGenericColumn(new DoubleGenericColumnSupplier(column, bitmap)); + } else { + throw new IAE("Unknown version[%d]", (int) versionFromBuffer); + } + }; + } + + public static SerializerBuilder serializerBuilder() + { + return new SerializerBuilder(); + } + + public static class SerializerBuilder + { + private ByteOrder byteOrder = null; + private DoubleColumnSerializer delegate = null; + private BitmapSerdeFactory bitmapSerdeFactory = null; + private ByteBufferWriter nullValueBitmapWriter = null; + + public SerializerBuilder withByteOrder(final ByteOrder byteOrder) + { + this.byteOrder = byteOrder; + return this; + } + + public SerializerBuilder withDelegate(final DoubleColumnSerializer delegate) + { + this.delegate = delegate; + return this; + } + + public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFactory) + { + this.bitmapSerdeFactory = bitmapSerdeFactory; + return this; + } + + public SerializerBuilder withNullValueBitmapWriter(ByteBufferWriter nullValueBitmapWriter) + { + this.nullValueBitmapWriter = nullValueBitmapWriter; + return this; + } + + public DoubleGenericColumnPartSerdeV2 build() + { + return new DoubleGenericColumnPartSerdeV2( + byteOrder, + bitmapSerdeFactory, + new Serializer() + { + @Override + public long numBytes() + { + long size = delegate.getSerializedSize() + Ints.BYTES + Byte.BYTES; + if (nullValueBitmapWriter != null) { + size += nullValueBitmapWriter.getSerializedSize(); + } + return size; + } + + @Override + public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException + { + channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE})); + channel.write(ByteBuffer.wrap(Ints.toByteArray((int) delegate.getSerializedSize()))); + delegate.writeToChannel(channel, fileSmoosher); + if (nullValueBitmapWriter != null) { + nullValueBitmapWriter.writeToChannel(channel, fileSmoosher); + } + } + } + ); + } + } +} diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java index 4d9c2b2f72c9..fc2609121265 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java @@ -20,6 +20,7 @@ package io.druid.segment.serde; import com.google.common.base.Supplier; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.column.GenericColumn; import io.druid.segment.column.IndexedDoublesGenericColumn; import io.druid.segment.data.CompressedDoublesIndexedSupplier; @@ -28,15 +29,20 @@ public class DoubleGenericColumnSupplier implements Supplier { private final CompressedDoublesIndexedSupplier column; + private final ImmutableBitmap nullValueBitmap; - public DoubleGenericColumnSupplier(CompressedDoublesIndexedSupplier column) + public DoubleGenericColumnSupplier( + CompressedDoublesIndexedSupplier column, + ImmutableBitmap nullValueBitmap + ) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override public GenericColumn get() { - return new IndexedDoublesGenericColumn(column.get()); + return new IndexedDoublesGenericColumn(column.get(), nullValueBitmap); } } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java index fe2ca9705fe4..3f386b2b4a80 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java @@ -21,9 +21,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.FloatColumnSerializer; +import io.druid.segment.IndexIO; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; import io.druid.segment.column.ValueType; @@ -125,7 +125,12 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo ); builder.setType(ValueType.FLOAT) .setHasMultipleValues(false) - .setGenericColumn(new FloatGenericColumnSupplier(column, byteOrder)); + .setGenericColumn(new FloatGenericColumnSupplier( + column, + byteOrder, + IndexIO.LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap() + )); } }; } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java new file mode 100644 index 000000000000..ee2dcacdedcc --- /dev/null +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -0,0 +1,187 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Suppliers; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.FloatColumnSerializer; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.BitmapSerde; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferSerializer; +import io.druid.segment.data.ByteBufferWriter; +import io.druid.segment.data.CompressedFloatsIndexedSupplier; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.WritableByteChannel; + +/** + */ +public class FloatGenericColumnPartSerdeV2 implements ColumnPartSerde +{ + static final byte VERSION_ONE = 0x1; + + @JsonCreator + public static FloatGenericColumnPartSerdeV2 createDeserializer( + @JsonProperty("byteOrder") ByteOrder byteOrder, + @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + ) + { + return new FloatGenericColumnPartSerdeV2( + byteOrder, + bitmapSerdeFactory != null ? bitmapSerdeFactory : new BitmapSerde.LegacyBitmapSerdeFactory(), + null + ); + } + + private final ByteOrder byteOrder; + private final BitmapSerdeFactory bitmapSerdeFactory; + private Serializer serializer; + + private FloatGenericColumnPartSerdeV2( + ByteOrder byteOrder, + BitmapSerdeFactory bitmapSerdeFactory, + Serializer serializer + ) + { + this.byteOrder = byteOrder; + this.bitmapSerdeFactory = bitmapSerdeFactory; + this.serializer = serializer; + } + + @JsonProperty + public ByteOrder getByteOrder() + { + return byteOrder; + } + + public static SerializerBuilder serializerBuilder() + { + return new SerializerBuilder(); + } + + public static class SerializerBuilder + { + private ByteOrder byteOrder = null; + private FloatColumnSerializer delegate = null; + private BitmapSerdeFactory bitmapSerdeFactory = null; + private ByteBufferWriter nullValueBitmapWriter = null; + + public SerializerBuilder withByteOrder(final ByteOrder byteOrder) + { + this.byteOrder = byteOrder; + return this; + } + + public SerializerBuilder withDelegate(final FloatColumnSerializer delegate) + { + this.delegate = delegate; + return this; + } + + public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFactory) + { + this.bitmapSerdeFactory = bitmapSerdeFactory; + return this; + } + + public SerializerBuilder withNullValueBitmapWriter(ByteBufferWriter nullValueBitmapWriter) + { + this.nullValueBitmapWriter = nullValueBitmapWriter; + return this; + } + + public FloatGenericColumnPartSerdeV2 build() + { + return new FloatGenericColumnPartSerdeV2( + byteOrder, bitmapSerdeFactory, new Serializer() + { + @Override + public long numBytes() + { + long size = delegate.getSerializedSize() + Ints.BYTES + Byte.BYTES; + if (nullValueBitmapWriter != null) { + size += nullValueBitmapWriter.getSerializedSize(); + } + return size; + } + + @Override + public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException + { + channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE})); + channel.write(ByteBuffer.wrap(Ints.toByteArray((int) delegate.getSerializedSize()))); + delegate.writeToChannel(channel, fileSmoosher); + if (nullValueBitmapWriter != null) { + nullValueBitmapWriter.writeToChannel(channel, fileSmoosher); + } + } + } + ); + } + + + } + + @Override + public Serializer getSerializer() + { + return serializer; + } + + @Override + public Deserializer getDeserializer() + { + return (buffer, builder, columnConfig) -> { + byte versionFromBuffer = buffer.get(); + if (VERSION_ONE == versionFromBuffer) { + int offset = buffer.getInt(); + int initialPos = buffer.position(); + final CompressedFloatsIndexedSupplier column = CompressedFloatsIndexedSupplier.fromByteBuffer( + buffer, + byteOrder, + builder.getFileMapper() + ); + buffer.position(initialPos + offset); + final ImmutableBitmap bitmap; + if (buffer.hasRemaining()) { + bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); + builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); + } else { + bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); + } + builder.setType(ValueType.FLOAT) + .setHasMultipleValues(false) + .setGenericColumn(new FloatGenericColumnSupplier(column, byteOrder, bitmap)); + } else { + throw new IAE("Unknown version[%d]", (int) versionFromBuffer); + } + + }; + } +} diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java index 81c9a91e4672..6e3034ca3a7f 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java @@ -20,6 +20,7 @@ package io.druid.segment.serde; import com.google.common.base.Supplier; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.column.GenericColumn; import io.druid.segment.column.IndexedFloatsGenericColumn; import io.druid.segment.data.CompressedFloatsIndexedSupplier; @@ -32,16 +33,22 @@ public class FloatGenericColumnSupplier implements Supplier { private final CompressedFloatsIndexedSupplier column; private final ByteOrder byteOrder; + private final ImmutableBitmap nullValueBitmap; - public FloatGenericColumnSupplier(CompressedFloatsIndexedSupplier column, ByteOrder byteOrder) + public FloatGenericColumnSupplier( + CompressedFloatsIndexedSupplier column, + ByteOrder byteOrder, + ImmutableBitmap nullValueBitmap + ) { this.column = column; this.byteOrder = byteOrder; + this.nullValueBitmap = nullValueBitmap; } @Override public GenericColumn get() { - return new IndexedFloatsGenericColumn(column.get()); + return new IndexedFloatsGenericColumn(column.get(), nullValueBitmap); } } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java index e5f943be0bb5..f7572d891238 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.IndexIO; import io.druid.segment.LongColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; @@ -125,7 +125,11 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo ); builder.setType(ValueType.LONG) .setHasMultipleValues(false) - .setGenericColumn(new LongGenericColumnSupplier(column)); + .setGenericColumn(new LongGenericColumnSupplier( + column, + IndexIO.LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap() + )); } }; } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java new file mode 100644 index 000000000000..1cd86316a86e --- /dev/null +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -0,0 +1,185 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Suppliers; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.LongColumnSerializer; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.BitmapSerde; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferSerializer; +import io.druid.segment.data.ByteBufferWriter; +import io.druid.segment.data.CompressedLongsIndexedSupplier; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.WritableByteChannel; + +/** + */ +public class LongGenericColumnPartSerdeV2 implements ColumnPartSerde +{ + static final byte VERSION_ONE = 0x1; + + @JsonCreator + public static LongGenericColumnPartSerdeV2 createDeserializer( + @JsonProperty("byteOrder") ByteOrder byteOrder, + @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + ) + { + return new LongGenericColumnPartSerdeV2( + byteOrder, + bitmapSerdeFactory != null ? bitmapSerdeFactory : new BitmapSerde.LegacyBitmapSerdeFactory(), + null + ); + } + + private final ByteOrder byteOrder; + private final BitmapSerdeFactory bitmapSerdeFactory; + private Serializer serializer; + + private LongGenericColumnPartSerdeV2( + ByteOrder byteOrder, + BitmapSerdeFactory bitmapSerdeFactory, Serializer serializer + ) + { + this.byteOrder = byteOrder; + this.bitmapSerdeFactory = bitmapSerdeFactory; + this.serializer = serializer; + } + + @JsonProperty + public ByteOrder getByteOrder() + { + return byteOrder; + } + + public static SerializerBuilder serializerBuilder() + { + return new SerializerBuilder(); + } + + public static class SerializerBuilder + { + private ByteOrder byteOrder = null; + private LongColumnSerializer delegate = null; + private BitmapSerdeFactory bitmapSerdeFactory = null; + private ByteBufferWriter nullValueBitmapWriter = null; + + public SerializerBuilder withByteOrder(final ByteOrder byteOrder) + { + this.byteOrder = byteOrder; + return this; + } + + public SerializerBuilder withDelegate(final LongColumnSerializer delegate) + { + this.delegate = delegate; + return this; + } + + public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFactory) + { + this.bitmapSerdeFactory = bitmapSerdeFactory; + return this; + } + + public SerializerBuilder withNullValueBitmapWriter(ByteBufferWriter nullValueBitmapWriter) + { + this.nullValueBitmapWriter = nullValueBitmapWriter; + return this; + } + + public LongGenericColumnPartSerdeV2 build() + { + return new LongGenericColumnPartSerdeV2( + byteOrder, bitmapSerdeFactory, new Serializer() + { + @Override + public long numBytes() + { + long size = delegate.getSerializedSize() + Ints.BYTES + Byte.BYTES; + if (nullValueBitmapWriter != null) { + size += nullValueBitmapWriter.getSerializedSize(); + } + return size; + } + + @Override + public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + { + channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE})); + channel.write(ByteBuffer.wrap(Ints.toByteArray((int) delegate.getSerializedSize()))); + delegate.writeToChannel(channel, smoosher); + if (nullValueBitmapWriter != null) { + nullValueBitmapWriter.writeToChannel(channel, smoosher); + } + } + } + ); + } + } + + @Override + public Serializer getSerializer() + { + return serializer; + } + + @Override + public Deserializer getDeserializer() + { + return (buffer, builder, columnConfig) -> { + byte versionFromBuffer = buffer.get(); + + if (VERSION_ONE == versionFromBuffer) { + int offset = buffer.getInt(); + int initialPos = buffer.position(); + final CompressedLongsIndexedSupplier column = CompressedLongsIndexedSupplier.fromByteBuffer( + buffer, + byteOrder, + builder.getFileMapper() + ); + buffer.position(initialPos + offset); + final ImmutableBitmap bitmap; + if (buffer.hasRemaining()) { + bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); + builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); + } else { + bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); + } + builder.setType(ValueType.LONG) + .setHasMultipleValues(false) + .setGenericColumn(new LongGenericColumnSupplier(column, bitmap)); + } else { + throw new IAE("Unknown version[%d]", (int) versionFromBuffer); + } + + }; + } +} diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java index 170d19dcd55b..742722af9853 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java @@ -20,6 +20,7 @@ package io.druid.segment.serde; import com.google.common.base.Supplier; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.column.GenericColumn; import io.druid.segment.column.IndexedLongsGenericColumn; import io.druid.segment.data.CompressedLongsIndexedSupplier; @@ -29,15 +30,17 @@ public class LongGenericColumnSupplier implements Supplier { private final CompressedLongsIndexedSupplier column; + private final ImmutableBitmap nullValueBitmap; - public LongGenericColumnSupplier(CompressedLongsIndexedSupplier column) + public LongGenericColumnSupplier(CompressedLongsIndexedSupplier column, ImmutableBitmap nullValueBitmap) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override public GenericColumn get() { - return new IndexedLongsGenericColumn(column.get()); + return new IndexedLongsGenericColumn(column.get(), nullValueBitmap); } } From 7552a611f18fe460136f1b4953ef69d64d9e8718 Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 02:03:12 +0530 Subject: [PATCH 02/50] Supporting Nulls in Indexing layer --- .../java/io/druid/data/input/MapBasedRow.java | 12 +-- .../main/java/io/druid/data/input/Row.java | 9 ++- .../io/druid/data/input/MapBasedRowTest.java | 14 ++-- .../io/druid/indexer/IndexGeneratorJob.java | 6 +- .../java/io/druid/indexer/InputRowSerde.java | 41 ++++++---- .../druid/indexer/hadoop/SegmentInputRow.java | 6 +- .../indexer/IndexGeneratorCombinerTest.java | 6 +- .../io/druid/indexer/InputRowSerdeTest.java | 32 +++++++- .../IngestSegmentFirehoseFactoryTest.java | 2 +- .../druid/java/util/common/StringUtils.java | 9 +++ .../java/util/common/StringUtilsTest.java | 6 +- .../java/io/druid/guice/GuiceInjectors.java | 1 + .../io/druid/guice/NullHandlingModule.java | 37 +++++++++ .../RowBasedColumnSelectorFactory.java | 62 ++++++++++++--- .../druid/segment/DimensionHandlerUtils.java | 34 +++++--- .../io/druid/segment/DimensionSelector.java | 11 +++ .../druid/segment/DoubleDimensionIndexer.java | 49 ++++++++++-- .../segment/DoubleDimensionMergerV9.java | 43 ++++++++-- .../druid/segment/FloatDimensionHandler.java | 2 +- .../druid/segment/FloatDimensionIndexer.java | 38 +++++++-- .../druid/segment/FloatDimensionMergerV9.java | 42 ++++++++-- .../druid/segment/LongDimensionIndexer.java | 43 +++++++--- .../druid/segment/LongDimensionMergerV9.java | 45 +++++++++-- .../druid/segment/NullDimensionSelector.java | 3 +- .../druid/segment/StringDimensionIndexer.java | 78 ++++++++++++------- .../segment/StringDimensionMergerV9.java | 18 +++-- .../segment/incremental/IncrementalIndex.java | 11 ++- ...IncrementalIndexColumnSelectorFactory.java | 24 ++++++ .../incremental/OffheapIncrementalIndex.java | 9 +++ .../incremental/OnheapIncrementalIndex.java | 6 ++ .../SpatialDimensionRowTransformer.java | 6 +- 31 files changed, 550 insertions(+), 155 deletions(-) create mode 100644 processing/src/main/java/io/druid/guice/NullHandlingModule.java diff --git a/api/src/main/java/io/druid/data/input/MapBasedRow.java b/api/src/main/java/io/druid/data/input/MapBasedRow.java index 7a8ba07a44a1..2be503918e76 100644 --- a/api/src/main/java/io/druid/data/input/MapBasedRow.java +++ b/api/src/main/java/io/druid/data/input/MapBasedRow.java @@ -101,12 +101,12 @@ public Object getRaw(String dimension) } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { Object metricValue = event.get(metric); if (metricValue == null) { - return 0.0f; + return null; } if (metricValue instanceof Number) { @@ -124,12 +124,12 @@ public float getFloatMetric(String metric) } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { Object metricValue = event.get(metric); if (metricValue == null) { - return 0L; + return null; } if (metricValue instanceof Number) { @@ -148,12 +148,12 @@ public long getLongMetric(String metric) } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { Object metricValue = event.get(metric); if (metricValue == null) { - return 0.0d; + return null; } if (metricValue instanceof Number) { diff --git a/api/src/main/java/io/druid/data/input/Row.java b/api/src/main/java/io/druid/data/input/Row.java index f5462ff5bda7..94b8eb78a0d9 100644 --- a/api/src/main/java/io/druid/data/input/Row.java +++ b/api/src/main/java/io/druid/data/input/Row.java @@ -24,6 +24,7 @@ import io.druid.guice.annotations.PublicApi; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.List; /** @@ -71,7 +72,7 @@ public interface Row extends Comparable * * @return the value of the provided column name */ - public Object getRaw(String dimension); + public @Nullable Object getRaw(String dimension); /** * Returns the float value of the given metric column. @@ -81,7 +82,7 @@ public interface Row extends Comparable * * @return the float value for the provided column name. */ - public float getFloatMetric(String metric); + public @Nullable Float getFloatMetric(String metric); /** * Returns the long value of the given metric column. @@ -91,7 +92,7 @@ public interface Row extends Comparable * * @return the long value for the provided column name. */ - public long getLongMetric(String metric); + public @Nullable Long getLongMetric(String metric); /** * Returns the double value of the given metric column. @@ -101,5 +102,5 @@ public interface Row extends Comparable * * @return the double value for the provided column name. */ - public double getDoubleMetric(String metric); + public @Nullable Double getDoubleMetric(String metric); } diff --git a/api/src/test/java/io/druid/data/input/MapBasedRowTest.java b/api/src/test/java/io/druid/data/input/MapBasedRowTest.java index 05241f50e054..faac3bf2fd4b 100644 --- a/api/src/test/java/io/druid/data/input/MapBasedRowTest.java +++ b/api/src/test/java/io/druid/data/input/MapBasedRowTest.java @@ -42,12 +42,12 @@ public void testGetLongMetricFromString() .build() ); - Assert.assertEquals(-1, row.getLongMetric("k0")); - Assert.assertEquals(1, row.getLongMetric("k1")); - Assert.assertEquals(1, row.getLongMetric("k2")); - Assert.assertEquals(100000, row.getLongMetric("k3")); - Assert.assertEquals(9223372036854775806L, row.getLongMetric("k4")); - Assert.assertEquals(-9223372036854775807L, row.getLongMetric("k5")); - Assert.assertEquals(9223372036854775802L, row.getLongMetric("k6")); + Assert.assertEquals(-1, row.getLongMetric("k0").longValue()); + Assert.assertEquals(1, row.getLongMetric("k1").longValue()); + Assert.assertEquals(1, row.getLongMetric("k2").longValue()); + Assert.assertEquals(100000, row.getLongMetric("k3").longValue()); + Assert.assertEquals(9223372036854775806L, row.getLongMetric("k4").longValue()); + Assert.assertEquals(-9223372036854775807L, row.getLongMetric("k5").longValue()); + Assert.assertEquals(9223372036854775802L, row.getLongMetric("k6").longValue()); } } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java index f7194e892eb5..12e4ae4a11a6 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java @@ -426,19 +426,19 @@ public Object getRaw(String dimension) } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { return row.getFloatMetric(metric); } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return row.getLongMetric(metric); } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { return row.getDoubleMetric(metric); } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index f5c6f9462e6d..0a4c1b7ec6b2 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -105,18 +105,22 @@ public InputRow get() } String t = aggFactory.getTypeName(); - - if (t.equals("float")) { - out.writeFloat(agg.getFloat()); - } else if (t.equals("long")) { - WritableUtils.writeVLong(out, agg.getLong()); - } else if (t.equals("double")) { - out.writeDouble(agg.getDouble()); + if (agg.isNull()) { + out.writeByte((byte) 1); } else { - //its a complex metric - Object val = agg.get(); - ComplexMetricSerde serde = getComplexMetricSerde(t); - writeBytes(serde.toBytes(val), out); + out.writeByte((byte) 0); + if (t.equals("float")) { + out.writeFloat(agg.getFloat()); + } else if (t.equals("long")) { + WritableUtils.writeVLong(out, agg.getLong()); + } else if (t.equals("double")) { + out.writeDouble(agg.getDouble()); + } else { + //its a complex metric + Object val = agg.get(); + ComplexMetricSerde serde = getComplexMetricSerde(t); + writeBytes(serde.toBytes(val), out); + } } } } @@ -130,8 +134,11 @@ public InputRow get() private static void writeBytes(byte[] value, ByteArrayDataOutput out) throws IOException { - WritableUtils.writeVInt(out, value.length); - out.write(value, 0, value.length); + int length = value == null ? -1 : value.length; + WritableUtils.writeVInt(out, length); + if (value != null) { + out.write(value, 0, value.length); + } } private static void writeString(String value, ByteArrayDataOutput out) throws IOException @@ -160,6 +167,9 @@ private static String readString(DataInput in) throws IOException private static byte[] readBytes(DataInput in) throws IOException { int size = WritableUtils.readVInt(in); + if (size < 0) { + return null; + } byte[] result = new byte[size]; in.readFully(result, 0, size); return result; @@ -210,6 +220,11 @@ public static final InputRow fromBytes(byte[] data, AggregatorFactory[] aggs) for (int i = 0; i < metricSize; i++) { String metric = readString(in); String type = getType(metric, aggs, i); + byte metricNullability = in.readByte(); + if (metricNullability == (byte) 1) { + // metric value is null. + continue; + } if (type.equals("float")) { event.put(metric, in.readFloat()); } else if (type.equals("long")) { diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java index 93f6d59e784b..442d0bd75325 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java @@ -71,19 +71,19 @@ public Object getRaw(String dimension) } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { return delegate.getFloatMetric(metric); } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return delegate.getLongMetric(metric); } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { return delegate.getDoubleMetric(metric); } diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorCombinerTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorCombinerTest.java index 4e9a1afbb9df..1e634cedae43 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorCombinerTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorCombinerTest.java @@ -186,7 +186,7 @@ public void testMultipleRowsMerged() throws Exception Assert.assertEquals(Arrays.asList("host", "keywords"), capturedRow.getDimensions()); Assert.assertEquals(ImmutableList.of(), capturedRow.getDimension("host")); Assert.assertEquals(Arrays.asList("bar", "foo"), capturedRow.getDimension("keywords")); - Assert.assertEquals(15, capturedRow.getLongMetric("visited_sum")); + Assert.assertEquals(15, capturedRow.getLongMetric("visited_sum").longValue()); Assert.assertEquals( 2.0, (Double) HyperUniquesAggregatorFactory.estimateCardinality( @@ -256,7 +256,7 @@ public void testMultipleRowsNotMerged() throws Exception Assert.assertEquals(Arrays.asList("host", "keywords"), capturedRow1.getDimensions()); Assert.assertEquals(Collections.singletonList("host1"), capturedRow1.getDimension("host")); Assert.assertEquals(Arrays.asList("bar", "foo"), capturedRow1.getDimension("keywords")); - Assert.assertEquals(10, capturedRow1.getLongMetric("visited_sum")); + Assert.assertEquals(10, capturedRow1.getLongMetric("visited_sum").longValue()); Assert.assertEquals( 1.0, (Double) HyperUniquesAggregatorFactory.estimateCardinality(capturedRow1.getRaw("unique_hosts"), false), @@ -267,7 +267,7 @@ public void testMultipleRowsNotMerged() throws Exception Assert.assertEquals(Arrays.asList("host", "keywords"), capturedRow2.getDimensions()); Assert.assertEquals(Collections.singletonList("host2"), capturedRow2.getDimension("host")); Assert.assertEquals(Arrays.asList("bar", "foo"), capturedRow2.getDimension("keywords")); - Assert.assertEquals(5, capturedRow2.getLongMetric("visited_sum")); + Assert.assertEquals(5, capturedRow2.getLongMetric("visited_sum").longValue()); Assert.assertEquals( 1.0, (Double) HyperUniquesAggregatorFactory.estimateCardinality(capturedRow2.getRaw("unique_hosts"), false), diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java index dd86e561811a..112ca7b23915 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java @@ -33,6 +33,7 @@ import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; @@ -71,6 +72,7 @@ public void testSerde() { // Prepare the mocks & set close() call count expectation to 1 final Aggregator mockedAggregator = EasyMock.createMock(DoubleSumAggregator.class); + EasyMock.expect(mockedAggregator.isNull()).andReturn(false).times(1); EasyMock.expect(mockedAggregator.getDouble()).andReturn(0d).times(1); mockedAggregator.aggregate(); EasyMock.expectLastCall().times(1); @@ -78,6 +80,14 @@ public void testSerde() EasyMock.expectLastCall().times(1); EasyMock.replay(mockedAggregator); + final Aggregator mockedNullAggregator = EasyMock.createMock(DoubleSumAggregator.class); + EasyMock.expect(mockedNullAggregator.isNull()).andReturn(true).times(1); + mockedNullAggregator.aggregate(); + EasyMock.expectLastCall().times(1); + mockedNullAggregator.close(); + EasyMock.expectLastCall().times(1); + EasyMock.replay(mockedNullAggregator); + InputRow in = new MapBasedInputRow( timestamp, dims, @@ -96,6 +106,13 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) { return mockedAggregator; } + }, + new DoubleSumAggregatorFactory("mockedNullAggregator", "m5") { + @Override + public Aggregator factorize(ColumnSelectorFactory metricFactory) + { + return mockedNullAggregator; + } } }; @@ -108,13 +125,22 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) Assert.assertEquals(ImmutableList.of("d1v"), out.getDimension("d1")); Assert.assertEquals(ImmutableList.of("d2v1", "d2v2"), out.getDimension("d2")); - Assert.assertEquals(0.0f, out.getFloatMetric("agg_non_existing"), 0.00001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0.0f, out.getFloatMetric("agg_non_existing"), 0.00001); + Assert.assertEquals(0L, out.getLongMetric("unparseable").longValue()); + } else { + Assert.assertNull(out.getFloatMetric("agg_non_existing")); + Assert.assertNull(out.getLongMetric("unparseable")); + } Assert.assertEquals(5.0f, out.getFloatMetric("m1out"), 0.00001); - Assert.assertEquals(100L, out.getLongMetric("m2out")); + Assert.assertEquals(100L, out.getLongMetric("m2out").longValue()); Assert.assertEquals(1, ((HyperLogLogCollector) out.getRaw("m3out")).estimateCardinality(), 0.001); - Assert.assertEquals(0L, out.getLongMetric("unparseable")); + + Assert.assertEquals(null, out.getLongMetric("m5")); + EasyMock.verify(mockedAggregator); + EasyMock.verify(mockedNullAggregator); } @Test(expected = ParseException.class) diff --git a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java index 59f216400361..7165bb4d29f2 100644 --- a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java @@ -527,7 +527,7 @@ public void simpleFirehoseReadingTest() throws IOException InputRow row = firehose.nextRow(); Assert.assertArrayEquals(new String[]{DIM_NAME}, row.getDimensions().toArray()); Assert.assertArrayEquals(new String[]{DIM_VALUE}, row.getDimension(DIM_NAME).toArray()); - Assert.assertEquals(METRIC_LONG_VALUE.longValue(), row.getLongMetric(METRIC_LONG_NAME)); + Assert.assertEquals(METRIC_LONG_VALUE.longValue(), row.getLongMetric(METRIC_LONG_NAME).longValue()); Assert.assertEquals(METRIC_FLOAT_VALUE, row.getFloatMetric(METRIC_FLOAT_NAME), METRIC_FLOAT_VALUE * 0.0001); ++rowcount; } diff --git a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java index 78088963ad33..d4fc2a2f818e 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java @@ -22,6 +22,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; +import javax.annotation.Nullable; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -66,8 +67,12 @@ public static byte[] toUtf8WithNullToEmpty(final String string) return string == null ? EMPTY_BYTES : toUtf8(string); } + @Nullable public static String fromUtf8(final byte[] bytes) { + if (bytes == null) { + return null; + } try { return new String(bytes, UTF8_STRING); } @@ -89,8 +94,12 @@ public static String fromUtf8(final ByteBuffer buffer) return StringUtils.fromUtf8(buffer, buffer.remaining()); } + @Nullable public static byte[] toUtf8(final String string) { + if (string == null) { + return null; + } try { return string.getBytes(UTF8_STRING); } diff --git a/java-util/src/test/java/io/druid/java/util/common/StringUtilsTest.java b/java-util/src/test/java/io/druid/java/util/common/StringUtilsTest.java index 30cea273b722..e3010e8ba8cf 100644 --- a/java-util/src/test/java/io/druid/java/util/common/StringUtilsTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/StringUtilsTest.java @@ -85,10 +85,10 @@ public void testNullPointerByteBuffer() StringUtils.fromUtf8((ByteBuffer) null); } - @Test(expected = NullPointerException.class) - public void testNullPointerByteArray() + @Test + public void testNullByteArray() { - StringUtils.fromUtf8((byte[]) null); + Assert.assertNull(StringUtils.fromUtf8((byte[]) null)); } @Test diff --git a/processing/src/main/java/io/druid/guice/GuiceInjectors.java b/processing/src/main/java/io/druid/guice/GuiceInjectors.java index f3bac2362ce4..8adaae4dd25d 100644 --- a/processing/src/main/java/io/druid/guice/GuiceInjectors.java +++ b/processing/src/main/java/io/druid/guice/GuiceInjectors.java @@ -42,6 +42,7 @@ public static Collection makeDefaultStartupModules() new JacksonModule(), new PropertiesModule(Arrays.asList("common.runtime.properties", "runtime.properties")), new ConfigModule(), + new NullHandlingModule(), new Module() { @Override diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java new file mode 100644 index 000000000000..ba6ca343098d --- /dev/null +++ b/processing/src/main/java/io/druid/guice/NullHandlingModule.java @@ -0,0 +1,37 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.segment.NullHandlingHelper; +import io.druid.segment.NullValueHandlingConfig; + +/** + */ +public class NullHandlingModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.null.handling", NullValueHandlingConfig.class); + binder.requestStaticInjection(NullHandlingHelper.class); + } +} diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java index 80133c6120f3..839b8077eeaa 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java @@ -20,7 +20,6 @@ package io.druid.query.groupby; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import io.druid.data.input.Row; @@ -29,11 +28,13 @@ import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.DimensionSelector; import io.druid.segment.DoubleColumnSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.IdLookup; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.SingleValueDimensionSelector; import io.druid.segment.column.Column; @@ -215,13 +216,14 @@ public ValueMatcher makeValueMatcher(final String value) @Override public boolean matches() { - final List dimensionValues = row.get().getDimension(dimension); + Row row = RowBasedColumnSelectorFactory.this.row.get(); + final List dimensionValues = row.getDimension(dimension); if (dimensionValues == null || dimensionValues.isEmpty()) { return value == null; } for (String dimensionValue : dimensionValues) { - if (Objects.equals(Strings.emptyToNull(dimensionValue), value)) { + if (Objects.equals(NullHandlingHelper.defaultToNull(dimensionValue), value)) { return true; } } @@ -246,7 +248,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (Objects.equals(extractionFn.apply(Strings.emptyToNull(dimensionValue)), value)) { + if (Objects.equals(extractionFn.apply(NullHandlingHelper.defaultToNull(dimensionValue)), value)) { return true; } } @@ -279,7 +281,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (predicate.apply(Strings.emptyToNull(dimensionValue))) { + if (predicate.apply(NullHandlingHelper.defaultToNull(dimensionValue))) { return true; } } @@ -305,7 +307,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (predicate.apply(extractionFn.apply(Strings.emptyToNull(dimensionValue)))) { + if (predicate.apply(extractionFn.apply(NullHandlingHelper.defaultToNull(dimensionValue)))) { return true; } } @@ -331,7 +333,7 @@ public int getValueCardinality() @Override public String lookupName(int id) { - final String value = Strings.emptyToNull(row.get().getDimension(dimension).get(id)); + final String value = NullHandlingHelper.defaultToNull(row.get().getDimension(dimension).get(id)); return extractionFn == null ? value : extractionFn.apply(value); } @@ -377,6 +379,13 @@ public float getFloat() { return (float) row.get().getTimestampFromEpoch(); } + + @Override + public boolean isNull() + { + // Time column never has null values + return false; + } } return new TimeFloatColumnSelector(); } else { @@ -385,7 +394,13 @@ public float getFloat() @Override public float getFloat() { - return row.get().getFloatMetric(columnName); + return DimensionHandlerUtils.nullToZero(row.get().getFloatMetric(columnName)); + } + + @Override + public boolean isNull() + { + return row.get().getFloatMetric(columnName) == null; } }; } @@ -410,6 +425,13 @@ public long getLong() { return row.get().getTimestampFromEpoch(); } + + @Override + public boolean isNull() + { + // Time column never has null values + return false; + } } return new TimeLongColumnSelector(); } else { @@ -418,7 +440,13 @@ public long getLong() @Override public long getLong() { - return row.get().getLongMetric(columnName); + return DimensionHandlerUtils.nullToZero(row.get().getLongMetric(columnName)); + } + + @Override + public boolean isNull() + { + return row.get().getLongMetric(columnName) == null; } }; } @@ -479,6 +507,14 @@ public double getDouble() { return (double) row.get().getTimestampFromEpoch(); } + + @Override + public boolean isNull() + { + // Time column never has null values + // Time column never has null values + return false; + } } return new TimeDoubleColumnSelector(); } else { @@ -487,7 +523,13 @@ public double getDouble() @Override public double getDouble() { - return row.get().getDoubleMetric(columnName); + return DimensionHandlerUtils.nullToZero(row.get().getDoubleMetric(columnName)); + } + + @Override + public boolean isNull() + { + return row.get().getDoubleMetric(columnName) == null; } }; } diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java index ff526e6662a4..ff185a51bce1 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java @@ -237,10 +237,20 @@ private static Colu return strategyFactory.makeColumnSelectorStrategy(capabilities, selector); } - public static Long convertObjectToLong(@Nullable Object valObj) + public static @Nullable + String convertObjectToString(@Nullable Object valObj) { if (valObj == null) { - return ZERO_LONG; + return null; + } + return valObj.toString(); + } + + public static @Nullable + Long convertObjectToLong(@Nullable Object valObj) + { + if (valObj == null) { + return null; } if (valObj instanceof Long) { @@ -254,10 +264,11 @@ public static Long convertObjectToLong(@Nullable Object valObj) } } - public static Float convertObjectToFloat(@Nullable Object valObj) + public static @Nullable + Float convertObjectToFloat(@Nullable Object valObj) { if (valObj == null) { - return ZERO_FLOAT; + return null; } if (valObj instanceof Float) { @@ -271,10 +282,11 @@ public static Float convertObjectToFloat(@Nullable Object valObj) } } - public static Double convertObjectToDouble(@Nullable Object valObj) + public static @Nullable + Double convertObjectToDouble(@Nullable Object valObj) { if (valObj == null) { - return ZERO_DOUBLE; + return null; } if (valObj instanceof Double) { @@ -282,8 +294,8 @@ public static Double convertObjectToDouble(@Nullable Object valObj) } else if (valObj instanceof Number) { return ((Number) valObj).doubleValue(); } else if (valObj instanceof String) { - Double doubleValue = Doubles.tryParse((String) valObj); - return doubleValue == null ? ZERO_DOUBLE : doubleValue; + Double doubleVal = Doubles.tryParse((String) valObj); + return NullHandlingHelper.nullToDefault(doubleVal); } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } @@ -327,16 +339,16 @@ public static Long getExactLongFromDecimalString(String decimalStr) public static Double nullToZero(@Nullable Double number) { - return number == null ? ZERO_DOUBLE : number; + return number == null ? NullHandlingHelper.ZERO_DOUBLE : number; } public static Long nullToZero(@Nullable Long number) { - return number == null ? ZERO_LONG : number; + return number == null ? NullHandlingHelper.ZERO_LONG : number; } public static Float nullToZero(@Nullable Float number) { - return number == null ? ZERO_FLOAT : number; + return number == null ? NullHandlingHelper.ZERO_FLOAT : number; } } diff --git a/processing/src/main/java/io/druid/segment/DimensionSelector.java b/processing/src/main/java/io/druid/segment/DimensionSelector.java index fa56c30928b4..b5c26789dbaf 100644 --- a/processing/src/main/java/io/druid/segment/DimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelector.java @@ -94,6 +94,7 @@ public interface DimensionSelector extends ColumnValueSelector, HotLoopCallee * @return the field name for the given id */ @CalledFromHotLoop + @Nullable public String lookupName(int id); /** @@ -147,4 +148,14 @@ default long getLong() { throw new UnsupportedOperationException("DimensionSelector cannot be operated as numeric ColumnValueSelector"); } + + /** + * @deprecated always throws {@link UnsupportedOperationException} + */ + @Deprecated + @Override + default boolean isNull() + { + throw new UnsupportedOperationException("DimensionSelector cannot be operated as numeric ColumnValueSelector"); + } } diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java index b3b71e300056..503a4e978911 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java @@ -21,6 +21,7 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.ValueType; @@ -29,10 +30,14 @@ import io.druid.segment.incremental.TimeAndDimsHolder; import javax.annotation.Nullable; +import java.util.Comparator; import java.util.List; +import java.util.Objects; public class DoubleDimensionIndexer implements DimensionIndexer { + public static final Comparator DOUBLE_COMPARATOR = Comparators.naturalNullsFirst(); + @Override public ValueType getValueType() { @@ -108,11 +113,20 @@ public long getLong() if (dimIndex >= dims.length) { return 0L; } - - double doubleValue = (Double) dims[dimIndex]; + double doubleValue = DimensionHandlerUtils.nullToZero((Double) dims[dimIndex]); return (long) doubleValue; } + @Override + public boolean isNull() + { + if (NullHandlingHelper.useDefaultValuesForNull()) { + return false; + } + final Object[] dims = currEntry.getKey().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -137,11 +151,20 @@ public float getFloat() if (dimIndex >= dims.length) { return 0.0f; } - - double doubleValue = (Double) dims[dimIndex]; + double doubleValue = DimensionHandlerUtils.nullToZero((Double) dims[dimIndex]); return (float) doubleValue; } + @Override + public boolean isNull() + { + if (NullHandlingHelper.useDefaultValuesForNull()) { + return false; + } + final Object[] dims = currEntry.getKey().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -164,9 +187,19 @@ public double getDouble() final Object[] dims = currEntry.getKey().getDims(); if (dimIndex >= dims.length) { - return 0.0; + return 0; + } + return DimensionHandlerUtils.nullToZero((Double) dims[dimIndex]); + } + + @Override + public boolean isNull() + { + if (NullHandlingHelper.useDefaultValuesForNull()) { + return false; } - return (Double) dims[dimIndex]; + final Object[] dims = currEntry.getKey().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; } @Override @@ -182,13 +215,13 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public int compareUnsortedEncodedKeyComponents(@Nullable Double lhs, @Nullable Double rhs) { - return Double.compare(DimensionHandlerUtils.nullToZero(lhs), DimensionHandlerUtils.nullToZero(rhs)); + return DOUBLE_COMPARATOR.compare(lhs, rhs); } @Override public boolean checkUnsortedEncodedKeyComponentsEqual(@Nullable Double lhs, @Nullable Double rhs) { - return DimensionHandlerUtils.nullToZero(lhs).equals(DimensionHandlerUtils.nullToZero(rhs)); + return Objects.equals(lhs, rhs); } @Override diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java index 87520d41ac8d..3bf90855023f 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java @@ -19,14 +19,19 @@ package io.druid.segment; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.IOPeon; -import io.druid.segment.serde.DoubleGenericColumnPartSerde; +import io.druid.segment.serde.DoubleGenericColumnPartSerdeV2; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.IntBuffer; @@ -41,6 +46,10 @@ public class DoubleDimensionMergerV9 implements DimensionMergerV9 protected final File outDir; protected IOPeon ioPeon; private DoubleColumnSerializer serializer; + private MutableBitmap nullRowsBitmap; + private int rowCount = 0; + private ByteBufferWriter nullValueBitmapWriter; + public DoubleDimensionMergerV9( String dimensionName, @@ -57,6 +66,7 @@ public DoubleDimensionMergerV9( this.outDir = outDir; this.ioPeon = ioPeon; this.progress = progress; + this.nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); try { setupEncodedValueWriter(); @@ -79,11 +89,14 @@ public ColumnDescriptor makeColumnDescriptor() throws IOException serializer.close(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.DOUBLE); + builder.setHasNullValues(!nullRowsBitmap.isEmpty()); builder.addSerde( - DoubleGenericColumnPartSerde.serializerBuilder() - .withByteOrder(IndexIO.BYTE_ORDER) - .withDelegate(serializer) - .build() + DoubleGenericColumnPartSerdeV2.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withDelegate(serializer) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) + .withNullValueBitmapWriter(nullValueBitmapWriter) + .build() ); return builder.build(); } @@ -103,19 +116,35 @@ public Double convertSegmentRowValuesToMergedRowValues(Double segmentRow, int se @Override public void processMergedRow(Double rowValues) throws IOException { + if (rowValues == null) { + nullRowsBitmap.add(rowCount); + } serializer.serialize(rowValues); + rowCount++; } @Override public void writeIndexes(List segmentRowNumConversions, Closer closer) throws IOException { - // double columns do not have indexes + boolean hasNullValues = !nullRowsBitmap.isEmpty(); + if (hasNullValues) { + nullValueBitmapWriter = new ByteBufferWriter<>( + ioPeon, + StringUtils.format("%s.nullBitmap", dimensionName), + indexSpec.getBitmapSerdeFactory().getObjectStrategy() + ); + try (Closeable bitmapWriter = nullValueBitmapWriter) { + nullValueBitmapWriter.open(); + nullValueBitmapWriter.write(indexSpec.getBitmapSerdeFactory() + .getBitmapFactory() + .makeImmutableBitmap(nullRowsBitmap)); + } + } } @Override public boolean canSkip() { - // a double column can never be all null return false; } } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java index f95b0d96534d..e432e61e98dd 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java @@ -21,8 +21,8 @@ import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; -import io.druid.segment.column.GenericColumn; import io.druid.segment.column.FloatColumn; +import io.druid.segment.column.GenericColumn; import io.druid.segment.data.IOPeon; import io.druid.segment.data.Indexed; diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java index ce925e04ddfc..df63e9512ab3 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java @@ -21,6 +21,7 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.ValueType; @@ -29,10 +30,14 @@ import io.druid.segment.incremental.TimeAndDimsHolder; import javax.annotation.Nullable; +import java.util.Comparator; import java.util.List; +import java.util.Objects; public class FloatDimensionIndexer implements DimensionIndexer { + public static final Comparator FLOAT_COMPARATOR = Comparators.naturalNullsFirst(); + @Override public ValueType getValueType() { @@ -114,8 +119,14 @@ public long getLong() return 0L; } - float floatVal = (Float) dims[dimIndex]; - return (long) floatVal; + return DimensionHandlerUtils.nullToZero((Float) dims[dimIndex]).longValue(); + } + + @Override + public boolean isNull() + { + final Object[] dims = currEntry.getKey().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; } @Override @@ -146,7 +157,14 @@ public float getFloat() return 0.0f; } - return (Float) dims[dimIndex]; + return DimensionHandlerUtils.nullToZero((Float) dims[dimIndex]); + } + + @Override + public boolean isNull() + { + final Object[] dims = currEntry.getKey().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; } @Override @@ -176,8 +194,14 @@ public double getDouble() if (dimIndex >= dims.length) { return 0.0; } - float floatVal = (Float) dims[dimIndex]; - return (double) floatVal; + return DimensionHandlerUtils.nullToZero((Float) dims[dimIndex]); + } + + @Override + public boolean isNull() + { + final Object[] dims = currEntry.getKey().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; } @Override @@ -193,13 +217,13 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public int compareUnsortedEncodedKeyComponents(@Nullable Float lhs, @Nullable Float rhs) { - return DimensionHandlerUtils.nullToZero(lhs).compareTo(DimensionHandlerUtils.nullToZero(rhs)); + return FLOAT_COMPARATOR.compare(lhs, rhs); } @Override public boolean checkUnsortedEncodedKeyComponentsEqual(@Nullable Float lhs, @Nullable Float rhs) { - return DimensionHandlerUtils.nullToZero(lhs).equals(DimensionHandlerUtils.nullToZero(rhs)); + return Objects.equals(lhs, rhs); } @Override diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java index d65da367dc2c..31fb92121079 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java @@ -19,14 +19,19 @@ package io.druid.segment; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.IOPeon; -import io.druid.segment.serde.FloatGenericColumnPartSerde; +import io.druid.segment.serde.FloatGenericColumnPartSerdeV2; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.IntBuffer; @@ -42,6 +47,9 @@ public class FloatDimensionMergerV9 implements DimensionMergerV9 protected IOPeon ioPeon; private FloatColumnSerializer serializer; + private MutableBitmap nullRowsBitmap; + private int rowCount = 0; + private ByteBufferWriter nullValueBitmapWriter; public FloatDimensionMergerV9( String dimensionName, @@ -58,6 +66,7 @@ public FloatDimensionMergerV9( this.outDir = outDir; this.ioPeon = ioPeon; this.progress = progress; + this.nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); try { setupEncodedValueWriter(); @@ -89,19 +98,35 @@ public Float convertSegmentRowValuesToMergedRowValues(Float segmentRow, int segm @Override public void processMergedRow(Float rowValues) throws IOException { + if (rowValues == null) { + nullRowsBitmap.add(rowCount); + } serializer.serialize(rowValues); + rowCount++; } @Override public void writeIndexes(List segmentRowNumConversions, Closer closer) throws IOException { - // floats have no indices to write + boolean hasNullValues = !nullRowsBitmap.isEmpty(); + if (hasNullValues) { + nullValueBitmapWriter = new ByteBufferWriter<>( + ioPeon, + StringUtils.format("%s.nullBitmap", dimensionName), + indexSpec.getBitmapSerdeFactory().getObjectStrategy() + ); + try (Closeable bitmapWriter = nullValueBitmapWriter) { + nullValueBitmapWriter.open(); + nullValueBitmapWriter.write(indexSpec.getBitmapSerdeFactory() + .getBitmapFactory() + .makeImmutableBitmap(nullRowsBitmap)); + } + } } @Override public boolean canSkip() { - // a float column can never be all null return false; } @@ -111,11 +136,14 @@ public ColumnDescriptor makeColumnDescriptor() throws IOException serializer.close(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.FLOAT); + builder.setHasNullValues(!nullRowsBitmap.isEmpty()); builder.addSerde( - FloatGenericColumnPartSerde.serializerBuilder() - .withByteOrder(IndexIO.BYTE_ORDER) - .withDelegate(serializer) - .build() + FloatGenericColumnPartSerdeV2.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) + .withNullValueBitmapWriter(nullValueBitmapWriter) + .withDelegate(serializer) + .build() ); return builder.build(); } diff --git a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java index 0a189a50dfca..a8773c41167b 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java @@ -21,6 +21,7 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.ValueType; @@ -29,10 +30,13 @@ import io.druid.segment.incremental.TimeAndDimsHolder; import javax.annotation.Nullable; +import java.util.Comparator; import java.util.List; +import java.util.Objects; public class LongDimensionIndexer implements DimensionIndexer { + public static final Comparator LONG_COMPARATOR = Comparators.naturalNullsFirst(); @Override public ValueType getValueType() { @@ -111,10 +115,17 @@ public long getLong() final Object[] dims = currEntry.getKey().getDims(); if (dimIndex >= dims.length) { - return 0L; + return 0; } - return (Long) dims[dimIndex]; + return DimensionHandlerUtils.nullToZero((Long) dims[dimIndex]); + } + + @Override + public boolean isNull() + { + final Object[] dims = currEntry.getKey().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; } @Override @@ -142,11 +153,17 @@ public float getFloat() final Object[] dims = currEntry.getKey().getDims(); if (dimIndex >= dims.length) { - return 0.0f; + return 0; } - long longVal = (Long) dims[dimIndex]; - return (float) longVal; + return DimensionHandlerUtils.nullToZero((Long) dims[dimIndex]); + } + + @Override + public boolean isNull() + { + final Object[] dims = currEntry.getKey().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; } @Override @@ -174,11 +191,17 @@ public double getDouble() final Object[] dims = currEntry.getKey().getDims(); if (dimIndex >= dims.length) { - return 0.0; + return 0; } - long longVal = (Long) dims[dimIndex]; - return (double) longVal; + return DimensionHandlerUtils.nullToZero((Long) dims[dimIndex]); + } + + @Override + public boolean isNull() + { + final Object[] dims = currEntry.getKey().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; } @Override @@ -194,13 +217,13 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public int compareUnsortedEncodedKeyComponents(@Nullable Long lhs, @Nullable Long rhs) { - return DimensionHandlerUtils.nullToZero(lhs).compareTo(DimensionHandlerUtils.nullToZero(rhs)); + return LONG_COMPARATOR.compare(lhs, rhs); } @Override public boolean checkUnsortedEncodedKeyComponentsEqual(@Nullable Long lhs, @Nullable Long rhs) { - return DimensionHandlerUtils.nullToZero(lhs).equals(DimensionHandlerUtils.nullToZero(rhs)); + return Objects.equals(lhs, rhs); } @Override diff --git a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java index c1afc2f67467..5d5d7b755966 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java @@ -20,15 +20,21 @@ package io.druid.segment; import com.google.common.base.Throwables; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.Closer; +import io.druid.java.util.common.logger.Logger; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.IOPeon; -import io.druid.segment.serde.LongGenericColumnPartSerde; +import io.druid.segment.serde.LongGenericColumnPartSerdeV2; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.IntBuffer; @@ -36,6 +42,8 @@ public class LongDimensionMergerV9 implements DimensionMergerV9 { + private static final Logger log = new Logger(LongDimensionMergerV9.class); + protected String dimensionName; protected ProgressIndicator progress; protected final IndexSpec indexSpec; @@ -43,6 +51,9 @@ public class LongDimensionMergerV9 implements DimensionMergerV9 protected final File outDir; protected IOPeon ioPeon; protected LongColumnSerializer serializer; + private MutableBitmap nullRowsBitmap; + private int rowCount = 0; + private ByteBufferWriter nullValueBitmapWriter; public LongDimensionMergerV9( String dimensionName, @@ -59,6 +70,7 @@ public LongDimensionMergerV9( this.outDir = outDir; this.ioPeon = ioPeon; this.progress = progress; + this.nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); try { setupEncodedValueWriter(); @@ -91,19 +103,35 @@ public Long convertSegmentRowValuesToMergedRowValues(Long segmentRow, int segmen @Override public void processMergedRow(Long rowValues) throws IOException { + if (rowValues == null) { + nullRowsBitmap.add(rowCount); + } serializer.serialize(rowValues); + rowCount++; } @Override public void writeIndexes(List segmentRowNumConversions, Closer closer) throws IOException { - // longs have no indices to write + boolean hasNullValues = !nullRowsBitmap.isEmpty(); + if (hasNullValues) { + nullValueBitmapWriter = new ByteBufferWriter<>( + ioPeon, + StringUtils.format("%s.nullBitmap", dimensionName), + indexSpec.getBitmapSerdeFactory().getObjectStrategy() + ); + try (Closeable bitmapWriter = nullValueBitmapWriter) { + nullValueBitmapWriter.open(); + nullValueBitmapWriter.write(indexSpec.getBitmapSerdeFactory() + .getBitmapFactory() + .makeImmutableBitmap(nullRowsBitmap)); + } + } } @Override public boolean canSkip() { - // a long column can never be all null return false; } @@ -113,11 +141,14 @@ public ColumnDescriptor makeColumnDescriptor() throws IOException serializer.close(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.LONG); + builder.setHasNullValues(!nullRowsBitmap.isEmpty()); builder.addSerde( - LongGenericColumnPartSerde.serializerBuilder() - .withByteOrder(IndexIO.BYTE_ORDER) - .withDelegate(serializer) - .build() + LongGenericColumnPartSerdeV2.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) + .withNullValueBitmapWriter(nullValueBitmapWriter) + .withDelegate(serializer) + .build() ); return builder.build(); } diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index ef7fd5838ab2..2139e4673604 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -20,7 +20,6 @@ package io.druid.segment; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.IndexedInts; @@ -109,7 +108,7 @@ public IdLookup idLookup() @Override public int lookupId(String name) { - return Strings.isNullOrEmpty(name) ? 0 : -1; + return NullHandlingHelper.isNullOrDefault(name) ? 0 : -1; } @Override diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index d7d81d3ef546..707935f53f17 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -21,7 +21,6 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import io.druid.collections.bitmap.BitmapFactory; @@ -58,12 +57,17 @@ public class StringDimensionIndexer implements DimensionIndexer { - private static final Function STRING_TRANSFORMER = o -> o != null ? o.toString() : null; + private static final Function STRING_TRANSFORMER = o -> o != null + ? NullHandlingHelper.defaultToNull(o.toString()) + : null; + + private static final int ABSENT_VALUE_ID = -1; private static class DimensionDictionary { private String minValue = null; private String maxValue = null; + private int idForNull = ABSENT_VALUE_ID; private final Object2IntMap valueToId = new Object2IntOpenHashMap<>(); @@ -79,20 +83,30 @@ public DimensionDictionary() public int getId(String value) { synchronized (lock) { - return valueToId.getInt(Strings.nullToEmpty(value)); + if (value == null) { + return idForNull; + } + final int id = valueToId.getInt(value); + return id < 0 ? ABSENT_VALUE_ID : id; } } public String getValue(int id) { synchronized (lock) { - return Strings.emptyToNull(idToValue.get(id)); + if (id == idForNull) { + return null; + } + return idToValue.get(id); } } public boolean contains(String value) { synchronized (lock) { + if (value == null) { + return idForNull != ABSENT_VALUE_ID; + } return valueToId.containsKey(value); } } @@ -100,23 +114,29 @@ public boolean contains(String value) public int size() { synchronized (lock) { - return valueToId.size(); + return idToValue.size(); } } public int add(String originalValue) { - String value = Strings.nullToEmpty(originalValue); synchronized (lock) { - int prev = valueToId.getInt(value); + if (originalValue == null) { + if (idForNull == ABSENT_VALUE_ID) { + idForNull = size(); + idToValue.add(null); + } + return idForNull; + } + int prev = valueToId.getInt(originalValue); if (prev >= 0) { return prev; } final int index = size(); - valueToId.put(value, index); - idToValue.add(value); - minValue = minValue == null || minValue.compareTo(value) > 0 ? value : minValue; - maxValue = maxValue == null || maxValue.compareTo(value) < 0 ? value : maxValue; + valueToId.put(originalValue, index); + idToValue.add(originalValue); + minValue = minValue == null || minValue.compareTo(originalValue) > 0 ? originalValue : minValue; + maxValue = maxValue == null || maxValue.compareTo(originalValue) < 0 ? originalValue : maxValue; return index; } } @@ -151,15 +171,18 @@ private static class SortedDimensionDictionary public SortedDimensionDictionary(List idToValue, int length) { - Object2IntSortedMap sortedMap = new Object2IntRBTreeMap<>(); + Object2IntSortedMap sortedMap = new Object2IntRBTreeMap<>(Comparators.naturalNullsFirst()); for (int id = 0; id < length; id++) { - sortedMap.put(idToValue.get(id), id); + String value = idToValue.get(id); + sortedMap.put(value, id); } - this.sortedVals = Lists.newArrayList(sortedMap.keySet()); + + this.sortedVals = new ArrayList<>(length); this.idToIndex = new int[length]; this.indexToId = new int[length]; int index = 0; - for (IntIterator iterator = sortedMap.values().iterator(); iterator.hasNext();) { + sortedVals.addAll(sortedMap.keySet()); + for (IntIterator iterator = sortedMap.values().iterator(); iterator.hasNext(); ) { int id = iterator.nextInt(); idToIndex[id] = index; indexToId[index] = id; @@ -184,7 +207,7 @@ public int getSortedIdFromUnsortedId(int id) public String getValueFromSortedId(int index) { - return Strings.emptyToNull(sortedVals.get(index)); + return sortedVals.get(index); } } @@ -211,8 +234,11 @@ public int[] processRowValsToUnsortedEncodedKeyComponent(Object dimValues) final int oldDictSize = dimLookup.size(); if (dimValues == null) { - dimLookup.add(null); - encodedDimensionValues = null; + if (!dimLookup.contains(null)) { + encodedDimensionValues = new int[]{dimLookup.add(null)}; + } else { + encodedDimensionValues = new int[]{dimLookup.getId(null)}; + } } else if (dimValues instanceof List) { List dimValuesList = (List) dimValues; if (dimValuesList.size() == 1) { @@ -295,7 +321,7 @@ public String get(int index) public int indexOf(String value) { int id = getEncodedValue(value, false); - return id < 0 ? -1 : getSortedEncodedValueFromUnsorted(id); + return id < 0 ? ABSENT_VALUE_ID : getSortedEncodedValueFromUnsorted(id); } @Override @@ -403,7 +429,7 @@ public IndexedInts getRow() final int nullId = getEncodedValue(null, false); if (nullId > -1) { if (nullIdIntArray == null) { - nullIdIntArray = new int[] {nullId}; + nullIdIntArray = new int[]{nullId}; } row = nullIdIntArray; rowSize = 1; @@ -555,7 +581,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public LongColumnSelector makeLongColumnSelector(TimeAndDimsHolder currEntry, IncrementalIndex.DimensionDesc desc) { - return ZeroLongColumnSelector.instance(); + return ZeroLongColumnSelector.instance(); } @Override @@ -577,22 +603,18 @@ public Object convertUnsortedEncodedKeyComponentToActualArrayOrList(int[] key, b return null; } if (key.length == 1) { - String val = getActualValue(key[0], false); - val = Strings.nullToEmpty(val); - return val; + return getActualValue(key[0], false); } else { if (asList) { List rowVals = new ArrayList<>(key.length); for (int id : key) { - String val = getActualValue(id, false); - rowVals.add(Strings.nullToEmpty(val)); + rowVals.add(getActualValue(id, false)); } return rowVals; } else { String[] rowArray = new String[key.length]; for (int i = 0; i < key.length; i++) { - String val = getActualValue(key[i], false); - rowArray[i] = Strings.nullToEmpty(val); + rowArray[i] = getActualValue(key[i], false); } return rowArray; } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index 887620c36ac6..a4000118aafb 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -57,6 +57,7 @@ import it.unimi.dsi.fastutil.ints.AbstractIntIterator; import it.unimi.dsi.fastutil.ints.IntIterable; import it.unimi.dsi.fastutil.ints.IntIterator; +import org.apache.logging.log4j.util.Strings; import java.io.Closeable; import java.io.File; @@ -71,8 +72,11 @@ public class StringDimensionMergerV9 implements DimensionMergerV9 { private static final Logger log = new Logger(StringDimensionMergerV9.class); - protected static final Indexed EMPTY_STR_DIM_VAL = new ArrayIndexed<>(new String[]{""}, String.class); - protected static final int[] EMPTY_STR_DIM_ARRAY = new int[]{0}; + protected static final Indexed NULL_STR_DIM_VAL = new ArrayIndexed<>( + new String[]{(String) null}, + String.class + ); + protected static final int[] NULL_STR_DIM_ARRAY = new int[]{0}; protected static final Splitter SPLITTER = Splitter.on(","); private IndexedIntsWriter encodedValueWriter; @@ -146,7 +150,7 @@ public void writeMergedValueMetadata(List adapters) throws IOE convertMissingValues = dimHasValues && dimAbsentFromSomeIndex; /* - * Ensure the empty str is always in the dictionary if the dimension was missing from one index but + * Ensure the null str is always in the dictionary if the dimension was missing from one index but * has non-null values in another index. * This is done so that MMappedIndexRowIterable can convert null columns to empty strings * later on, to allow rows from indexes without a particular dimension to merge correctly with @@ -154,7 +158,7 @@ public void writeMergedValueMetadata(List adapters) throws IOE */ if (convertMissingValues && !hasNull) { hasNull = true; - dimValueLookups[adapters.size()] = dimValueLookup = EMPTY_STR_DIM_VAL; + dimValueLookups[adapters.size()] = dimValueLookup = NULL_STR_DIM_VAL; numMergeIndex++; } @@ -233,7 +237,7 @@ public int[] convertSegmentRowValuesToMergedRowValues(int[] segmentRow, int segm // For strings, convert missing values to null/empty if conversion flag is set // But if bitmap/dictionary is not used, always convert missing to 0 if (dimVals == null) { - return convertMissingValues ? EMPTY_STR_DIM_ARRAY : null; + return convertMissingValues ? NULL_STR_DIM_ARRAY : null; } int[] newDimVals = new int[dimVals.length]; @@ -401,7 +405,7 @@ static void mergeBitmaps( prevRow = row; } - if ((dictId == 0) && (Iterables.getFirst(dimVals, "") == null)) { + if ((dictId == 0) && Iterables.getFirst(dimVals, null) == null) { mergedIndexes.or(nullRowsBitmap); } @@ -409,7 +413,7 @@ static void mergeBitmaps( if (hasSpatial) { String dimVal = dimVals.get(dictId); - if (dimVal != null) { + if (Strings.isNotEmpty(dimVal)) { List stringCoords = Lists.newArrayList(SPLITTER.split(dimVal)); float[] coords = new float[stringCoords.size()]; for (int j = 0; j < coords.length; j++) { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 6c8daf2ad22c..c0912248a018 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -55,6 +55,7 @@ import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; import io.druid.segment.Metadata; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; @@ -449,6 +450,9 @@ protected abstract Integer addToFacts( protected abstract double getMetricDoubleValue(int rowOffset, int aggOffset); + protected abstract boolean isNull(int rowOffset, int aggOffset); + + @Override public void close() { @@ -551,15 +555,20 @@ TimeAndDims toTimeAndDims(InputRow row) throws IndexSizeExceededException DimensionHandler handler = DimensionHandlerUtils.getHandlerFromCapabilities(dimension, capabilities, null); desc = addNewDimension(dimension, capabilities, handler); } + Object raw = row.getRaw(dimension); DimensionHandler handler = desc.getHandler(); DimensionIndexer indexer = desc.getIndexer(); - Object dimsKey = indexer.processRowValsToUnsortedEncodedKeyComponent(row.getRaw(dimension)); + Object dimsKey = indexer.processRowValsToUnsortedEncodedKeyComponent(raw); // Set column capabilities as data is coming in if (!capabilities.hasMultipleValues() && dimsKey != null && handler.getLengthOfEncodedKeyComponent(dimsKey) > 1) { capabilities.setHasMultipleValues(true); } + if (!NullHandlingHelper.useDefaultValuesForNull() && !capabilities.hasNullValues() && raw == null) { + capabilities.setHasNullValues(true); + } + if (wasNewDim) { if (overflow == null) { overflow = Lists.newArrayList(); diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java index 6433b2774124..763617746e19 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java @@ -144,6 +144,12 @@ public float getFloat() return index.getMetricFloatValue(timeAndDimsHolder.getValue(), metricIndex); } + @Override + public boolean isNull() + { + return index.isNull(timeAndDimsHolder.getValue(), metricIndex); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -185,6 +191,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("index", index); } + + @Override + public boolean isNull() + { + return index.isNull(timeAndDimsHolder.getValue(), metricIndex); + } }; } @@ -209,6 +221,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { // nothing to inspect } + + @Override + public boolean isNull() + { + return false; + } } return new TimeLongColumnSelector(); } @@ -235,6 +253,12 @@ public long getLong() return index.getMetricLongValue(timeAndDimsHolder.getValue(), metricIndex); } + @Override + public boolean isNull() + { + return index.isNull(timeAndDimsHolder.getValue(), metricIndex); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java index 4bfa587a12f3..b2d288d72c64 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java @@ -310,6 +310,15 @@ public double getMetricDoubleValue(int rowOffset, int aggOffset) return agg.getDouble(bb, indexAndOffset[1] + aggOffsetInBuffer[aggOffset]); } + @Override + public boolean isNull(int rowOffset, int aggOffset) + { + BufferAggregator agg = getAggs()[aggOffset]; + int[] indexAndOffset = indexAndOffsets.get(rowOffset); + ByteBuffer bb = aggBuffers.get(indexAndOffset[0]).get(); + return agg.isNull(bb, indexAndOffset[1] + aggOffsetInBuffer[aggOffset]); + } + /** * NOTE: This is NOT thread-safe with add... so make sure all the adding is DONE before closing */ diff --git a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java index a530981a2d1d..18652cc5a0fd 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java @@ -286,6 +286,12 @@ protected double getMetricDoubleValue(int rowOffset, int aggOffset) return concurrentGet(rowOffset)[aggOffset].getDouble(); } + @Override + public boolean isNull(int rowOffset, int aggOffset) + { + return concurrentGet(rowOffset)[aggOffset].isNull(); + } + /** * Clear out maps to allow GC * NOTE: This is NOT thread-safe with add... so make sure all the adding is DONE before closing diff --git a/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowTransformer.java b/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowTransformer.java index 45548f98e4d4..dcf3ff935e8c 100644 --- a/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowTransformer.java +++ b/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowTransformer.java @@ -130,19 +130,19 @@ public Object getRaw(String dimension) } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return row.getLongMetric(metric); } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { return row.getDoubleMetric(metric); } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { return row.getFloatMetric(metric); } From 7f5fa83e0288e4f6f303bfb06265bc44c290bb21 Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 02:07:55 +0530 Subject: [PATCH 03/50] Querying part 1 : Aggregators support for Nulls --- .../DistinctCountAggregator.java | 6 +- .../DistinctCountBufferAggregator.java | 6 +- .../query/aggregation/AggregateCombiner.java | 6 ++ .../druid/query/aggregation/Aggregator.java | 12 +++ .../query/aggregation/BufferAggregator.java | 23 +++++ .../DoubleMaxAggregatorFactory.java | 17 +++- .../DoubleMinAggregatorFactory.java | 16 +++- .../DoubleSumAggregatorFactory.java | 16 +++- .../FloatMaxAggregatorFactory.java | 13 ++- .../FloatMinAggregatorFactory.java | 13 ++- .../FloatSumAggregatorFactory.java | 13 ++- .../aggregation/LongMaxAggregatorFactory.java | 16 +++- .../aggregation/LongMinAggregatorFactory.java | 18 +++- .../aggregation/LongSumAggregatorFactory.java | 14 ++- .../query/aggregation/NullableAggregator.java | 88 ++++++++++++++++++ .../aggregation/NullableBufferAggregator.java | 91 +++++++++++++++++++ .../NullableDoubleAggregateCombiner.java | 62 +++++++++++++ .../NullableLongAggregateCombiner.java | 61 +++++++++++++ .../NullableObjectAggregateCombiner.java | 67 ++++++++++++++ .../aggregation/ObjectAggregateCombiner.java | 5 + .../SimpleDoubleAggregatorFactory.java | 3 +- .../SimpleFloatAggregatorFactory.java | 3 +- ...alityAggregatorColumnSelectorStrategy.java | 30 ++++-- .../first/DoubleFirstAggregatorFactory.java | 29 +++--- .../first/FloatFirstAggregatorFactory.java | 28 +++--- .../first/LongFirstAggregatorFactory.java | 28 +++--- .../last/DoubleLastAggregatorFactory.java | 26 +++--- .../last/FloatLastAggregatorFactory.java | 29 +++--- .../last/LongLastAggregatorFactory.java | 28 +++--- 29 files changed, 650 insertions(+), 117 deletions(-) create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java index 13ab3fc2bba3..6a39a5826a28 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java @@ -22,6 +22,7 @@ import io.druid.collections.bitmap.MutableBitmap; import io.druid.query.aggregation.Aggregator; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; public class DistinctCountAggregator implements Aggregator { @@ -41,8 +42,11 @@ public DistinctCountAggregator( @Override public void aggregate() { + boolean countNulls = NullHandlingHelper.useDefaultValuesForNull(); for (final Integer index : selector.getRow()) { - mutableBitmap.add(index); + if (countNulls || selector.lookupName(index) != null) { + mutableBitmap.add(index); + } } } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index 5c21597177b1..63039ad9f978 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -24,6 +24,7 @@ import io.druid.query.aggregation.BufferAggregator; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; @@ -50,9 +51,12 @@ public void init(ByteBuffer buf, int position) @Override public void aggregate(ByteBuffer buf, int position) { + boolean countNulls = NullHandlingHelper.useDefaultValuesForNull(); MutableBitmap mutableBitmap = getMutableBitmap(buf, position); for (final Integer index : selector.getRow()) { - mutableBitmap.add(index); + if (countNulls || selector.lookupName(index) != null) { + mutableBitmap.add(index); + } } buf.putLong(position, mutableBitmap.size()); } diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java index e873c82ed94f..70865682e036 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java @@ -66,4 +66,10 @@ public interface AggregateCombiner extends ColumnValueSelector * @see AggregatorFactory#combine */ void fold(ColumnValueSelector selector); + + @Override + default boolean isNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java index d7fba1e3c5cd..397aeefa09cf 100644 --- a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java @@ -21,6 +21,7 @@ import io.druid.guice.annotations.ExtensionPoint; +import javax.annotation.Nullable; import java.io.Closeable; /** @@ -37,6 +38,8 @@ public interface Aggregator extends Closeable { void aggregate(); void reset(); + + @Nullable Object get(); float getFloat(); long getLong(); @@ -51,6 +54,15 @@ default double getDouble() return (double) getFloat(); } + /** + * returns true if the Aggregator support returning null values and the aggregated value is Null. + * The default implementation always return false to enable smooth backward compatibility. re-implement if your aggregator is nullable. + */ + default boolean isNull() + { + return false; + } + @Override void close(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java index cdddd7629f2d..9a9d3ce75d71 100644 --- a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java @@ -178,4 +178,27 @@ default void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, By { } + /** + * Returns true if the aggregator is nullable and the aggregated value is null + *

+ *

+ * Implementations must not change the position, limit or mark of the given buffer + *

+ * Implementations are only required to support this method if they the aggregator supports null values. + * If it doesn't support null always returning false is recommended. + *

+ * The default implementation always returns false. + * This default method is added to enable smooth backward compatibility, please re-implement it if your aggregators + * support null values + * + * @param buf byte buffer storing the byte array representation of the aggregate + * @param position offset within the byte buffer at which the aggregate value is stored + * + * @return true if the aggrgeated value is null otherwise false. + */ + default boolean isNull(ByteBuffer buf, int position) + { + return false; + } + } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index ff73f2f8c0eb..972bbece60f4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -25,6 +25,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DoubleColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Collections; @@ -35,7 +37,6 @@ */ public class DoubleMaxAggregatorFactory extends SimpleDoubleAggregatorFactory { - @JsonCreator public DoubleMaxAggregatorFactory( @JsonProperty("name") String name, @@ -55,13 +56,21 @@ public DoubleMaxAggregatorFactory(String name, String fieldName) @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleMaxAggregator(getDoubleColumnSelector(metricFactory, Double.NEGATIVE_INFINITY)); + DoubleColumnSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, Double.NEGATIVE_INFINITY); + return NullHandlingHelper.getNullableAggregator( + new DoubleMaxAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleMaxBufferAggregator(getDoubleColumnSelector(metricFactory, Double.NEGATIVE_INFINITY)); + DoubleColumnSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, Double.NEGATIVE_INFINITY); + return NullHandlingHelper.getNullableAggregator( + new DoubleMaxBufferAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override @@ -73,7 +82,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new DoubleMaxAggregateCombiner(); + return NullHandlingHelper.getNullableCombiner(new DoubleMaxAggregateCombiner()); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index 9577e89846c3..a57d3d423858 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -25,6 +25,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DoubleColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Arrays; @@ -54,13 +56,21 @@ public DoubleMinAggregatorFactory(String name, String fieldName) @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleMinAggregator(getDoubleColumnSelector(metricFactory, Double.POSITIVE_INFINITY)); + DoubleColumnSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, Double.POSITIVE_INFINITY); + return NullHandlingHelper.getNullableAggregator( + new DoubleMinAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleMinBufferAggregator(getDoubleColumnSelector(metricFactory, Double.POSITIVE_INFINITY)); + DoubleColumnSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, Double.POSITIVE_INFINITY); + return NullHandlingHelper.getNullableAggregator( + new DoubleMinBufferAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override @@ -72,7 +82,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new DoubleMinAggregateCombiner(); + return NullHandlingHelper.getNullableCombiner(new DoubleMinAggregateCombiner()); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index 50ec11bed556..97ad1ba24449 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -25,6 +25,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DoubleColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Arrays; @@ -55,13 +57,21 @@ public DoubleSumAggregatorFactory(String name, String fieldName) @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleSumAggregator(getDoubleColumnSelector(metricFactory, 0.0)); + DoubleColumnSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); + return NullHandlingHelper.getNullableAggregator( + new DoubleSumAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleSumBufferAggregator(getDoubleColumnSelector(metricFactory, 0.0)); + DoubleColumnSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); + return NullHandlingHelper.getNullableAggregator( + new DoubleSumBufferAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override @@ -73,7 +83,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new DoubleSumAggregateCombiner(); + return NullHandlingHelper.getNullableCombiner(new DoubleSumAggregateCombiner()); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java index 63a83da06d2d..f44e0b0c40fc 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -25,6 +25,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Collections; @@ -53,13 +55,18 @@ public FloatMaxAggregatorFactory(String name, String fieldName) @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new FloatMaxAggregator(getFloatColumnSelector(metricFactory, Float.NEGATIVE_INFINITY)); + FloatColumnSelector floatColumnSelector = getFloatColumnSelector(metricFactory, Float.NEGATIVE_INFINITY); + return NullHandlingHelper.getNullableAggregator(new FloatMaxAggregator(floatColumnSelector), floatColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new FloatMaxBufferAggregator(getFloatColumnSelector(metricFactory, Float.NEGATIVE_INFINITY)); + FloatColumnSelector floatColumnSelector = getFloatColumnSelector(metricFactory, Float.NEGATIVE_INFINITY); + return NullHandlingHelper.getNullableAggregator(new FloatMaxBufferAggregator(getFloatColumnSelector( + metricFactory, + Float.NEGATIVE_INFINITY + )), floatColumnSelector); } @Override @@ -71,7 +78,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new DoubleMaxAggregateCombiner(); + return NullHandlingHelper.getNullableCombiner(new DoubleMaxAggregateCombiner()); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java index 1129e15bc761..9d84dbb4d80a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java @@ -25,6 +25,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Arrays; @@ -53,13 +55,18 @@ public FloatMinAggregatorFactory(String name, String fieldName) @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new FloatMinAggregator(getFloatColumnSelector(metricFactory, Float.POSITIVE_INFINITY)); + FloatColumnSelector floatColumnSelector = getFloatColumnSelector(metricFactory, Float.POSITIVE_INFINITY); + return NullHandlingHelper.getNullableAggregator(new FloatMinAggregator(floatColumnSelector), floatColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new FloatMinBufferAggregator(getFloatColumnSelector(metricFactory, Float.POSITIVE_INFINITY)); + FloatColumnSelector floatColumnSelector = getFloatColumnSelector(metricFactory, Float.POSITIVE_INFINITY); + return NullHandlingHelper.getNullableAggregator( + new FloatMinBufferAggregator(floatColumnSelector), + floatColumnSelector + ); } @Override @@ -71,7 +78,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new DoubleMinAggregateCombiner(); + return NullHandlingHelper.getNullableCombiner(new DoubleMinAggregateCombiner()); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java index 4986a1ff6a3b..451b7124bc28 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java @@ -25,6 +25,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Arrays; @@ -53,13 +55,18 @@ public FloatSumAggregatorFactory(String name, String fieldName) @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new FloatSumAggregator(getFloatColumnSelector(metricFactory, 0.0f)); + FloatColumnSelector floatColumnSelector = getFloatColumnSelector(metricFactory, 0.0f); + return NullHandlingHelper.getNullableAggregator(new FloatSumAggregator(floatColumnSelector), floatColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new FloatSumBufferAggregator(getFloatColumnSelector(metricFactory, 0.0f)); + FloatColumnSelector floatColumnSelector = getFloatColumnSelector(metricFactory, 0.0f); + return NullHandlingHelper.getNullableAggregator( + new FloatSumBufferAggregator(floatColumnSelector), + floatColumnSelector + ); } @Override @@ -71,7 +78,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new DoubleSumAggregateCombiner(); + return NullHandlingHelper.getNullableCombiner(new DoubleSumAggregateCombiner()); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index 35c25cffeb30..89ddd8d1b4f7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -30,6 +30,7 @@ import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Arrays; @@ -75,13 +76,18 @@ public LongMaxAggregatorFactory(String name, String fieldName) @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongMaxAggregator(getLongColumnSelector(metricFactory)); + LongColumnSelector longColumnSelector = getLongColumnSelector(metricFactory); + return NullHandlingHelper.getNullableAggregator(new LongMaxAggregator(longColumnSelector), longColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongMaxBufferAggregator(getLongColumnSelector(metricFactory)); + LongColumnSelector longColumnSelector = getLongColumnSelector(metricFactory); + return NullHandlingHelper.getNullableAggregator( + new LongMaxBufferAggregator(longColumnSelector), + longColumnSelector + ); } private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) @@ -104,7 +110,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new LongAggregateCombiner() + return NullHandlingHelper.getNullableCombiner(new LongAggregateCombiner() { private long max; @@ -125,7 +131,7 @@ public long getLong() { return max; } - }; + }); } @Override @@ -212,7 +218,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Longs.BYTES; + return Longs.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index d024dc6afa1c..3524d83b4a54 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; import io.druid.java.util.common.StringUtils; @@ -30,6 +31,7 @@ import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Arrays; @@ -68,6 +70,7 @@ public LongMinAggregatorFactory( this.macroTable = macroTable; } + @VisibleForTesting public LongMinAggregatorFactory(String name, String fieldName) { this(name, fieldName, null, ExprMacroTable.nil()); @@ -76,13 +79,18 @@ public LongMinAggregatorFactory(String name, String fieldName) @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongMinAggregator(getLongColumnSelector(metricFactory)); + LongColumnSelector longColumnSelector = getLongColumnSelector(metricFactory); + return NullHandlingHelper.getNullableAggregator(new LongMinAggregator(longColumnSelector), longColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongMinBufferAggregator(getLongColumnSelector(metricFactory)); + LongColumnSelector longColumnSelector = getLongColumnSelector(metricFactory); + return NullHandlingHelper.getNullableAggregator( + new LongMinBufferAggregator(longColumnSelector), + longColumnSelector + ); } private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) @@ -105,7 +113,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new LongAggregateCombiner() + return NullHandlingHelper.getNullableCombiner(new LongAggregateCombiner() { private long min; @@ -126,7 +134,7 @@ public long getLong() { return min; } - }; + }); } @Override @@ -213,7 +221,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Longs.BYTES; + return Longs.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index cec708eb4a60..9942eb632f99 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -29,6 +29,7 @@ import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Arrays; @@ -74,13 +75,18 @@ public LongSumAggregatorFactory(String name, String fieldName) @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongSumAggregator(getLongColumnSelector(metricFactory)); + LongColumnSelector longColumnSelector = getLongColumnSelector(metricFactory); + return NullHandlingHelper.getNullableAggregator(new LongSumAggregator(longColumnSelector), longColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongSumBufferAggregator(getLongColumnSelector(metricFactory)); + LongColumnSelector longColumnSelector = getLongColumnSelector(metricFactory); + return NullHandlingHelper.getNullableAggregator( + new LongSumBufferAggregator(longColumnSelector), + longColumnSelector + ); } private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) @@ -103,7 +109,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new LongSumAggregateCombiner(); + return NullHandlingHelper.getNullableCombiner(new LongSumAggregateCombiner()); } @Override @@ -190,7 +196,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Longs.BYTES; + return Longs.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java new file mode 100644 index 000000000000..3a09ee46f54d --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -0,0 +1,88 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.ColumnValueSelector; + +public class NullableAggregator implements Aggregator +{ + private final Aggregator delegate; + private final ColumnValueSelector selector; + private boolean isNull; + + public NullableAggregator(Aggregator delegate, ColumnValueSelector selector) + { + this.delegate = delegate; + this.selector = selector; + reset(); + } + + @Override + public void aggregate() + { + if (isNull && !selector.isNull()) { + isNull = false; + } + delegate.aggregate(); + } + + @Override + public void reset() + { + isNull = true; + delegate.reset(); + } + + @Override + public Object get() + { + return delegate.get(); + } + + @Override + public float getFloat() + { + return delegate.getFloat(); + } + + @Override + public long getLong() + { + return delegate.getLong(); + } + + @Override + public double getDouble() + { + return delegate.getDouble(); + } + + @Override + public boolean isNull() + { + return isNull; + } + + @Override + public void close() + { + delegate.close(); + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java new file mode 100644 index 000000000000..2f70f086edc0 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -0,0 +1,91 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.ColumnValueSelector; + +import java.nio.ByteBuffer; + +public class NullableBufferAggregator implements BufferAggregator +{ + public static final byte IS_NULL_BYTE = (byte) 1; + public static final byte IS_NOT_NULL_BYTE = (byte) 0; + private final BufferAggregator delegate; + private final ColumnValueSelector selector; + + + public NullableBufferAggregator(BufferAggregator delegate, ColumnValueSelector selector) + { + this.delegate = delegate; + this.selector = selector; + } + + @Override + public void init(ByteBuffer buf, int position) + { + buf.put(position, IS_NULL_BYTE); + delegate.init(buf, position + Byte.BYTES); + } + + @Override + public void aggregate(ByteBuffer buf, int position) + { + if (buf.get(position) == IS_NULL_BYTE && !selector.isNull()) { + buf.put(position, IS_NOT_NULL_BYTE); + } + delegate.aggregate(buf, position + Byte.BYTES); + } + + @Override + public Object get(ByteBuffer buf, int position) + { + return delegate.get(buf, position + Byte.BYTES); + } + + @Override + public float getFloat(ByteBuffer buf, int position) + { + return delegate.getFloat(buf, position + Byte.BYTES); + } + + @Override + public long getLong(ByteBuffer buf, int position) + { + return delegate.getLong(buf, position + Byte.BYTES); + } + + @Override + public double getDouble(ByteBuffer buf, int position) + { + return delegate.getDouble(buf, position + Byte.BYTES); + } + + @Override + public boolean isNull(ByteBuffer buf, int position) + { + return buf.get() == IS_NULL_BYTE; + } + + @Override + public void close() + { + + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java new file mode 100644 index 000000000000..3d16cacfb649 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java @@ -0,0 +1,62 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.ColumnValueSelector; + +public class NullableDoubleAggregateCombiner extends DoubleAggregateCombiner +{ + private boolean isNull; + + private DoubleAggregateCombiner delegate; + + public NullableDoubleAggregateCombiner(DoubleAggregateCombiner delegate) + { + this.delegate = delegate; + } + + @Override + public void reset(ColumnValueSelector selector) + { + isNull = true; + delegate.reset(selector); + } + + @Override + public void fold(ColumnValueSelector selector) + { + if (isNull && !selector.isNull()) { + isNull = false; + } + delegate.fold(selector); + } + + @Override + public double getDouble() + { + return delegate.getDouble(); + } + + @Override + public boolean isNull() + { + return isNull; + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java new file mode 100644 index 000000000000..61ca61ea9a10 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java @@ -0,0 +1,61 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.ColumnValueSelector; + +public class NullableLongAggregateCombiner extends LongAggregateCombiner +{ + private boolean isNull; + + private LongAggregateCombiner delegate; + + public NullableLongAggregateCombiner(LongAggregateCombiner delegate) + { + this.delegate = delegate; + } + + @Override + public void reset(ColumnValueSelector selector) + { + isNull = true; + } + + @Override + public void fold(ColumnValueSelector selector) + { + if (isNull && !selector.isNull()) { + isNull = true; + } + delegate.fold(selector); + } + + @Override + public long getLong() + { + return delegate.getLong(); + } + + @Override + public boolean isNull() + { + return isNull; + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java new file mode 100644 index 000000000000..e644c9db5c13 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java @@ -0,0 +1,67 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.ColumnValueSelector; + +public class NullableObjectAggregateCombiner extends ObjectAggregateCombiner +{ + private boolean isNull; + + private ObjectAggregateCombiner delegate; + + public NullableObjectAggregateCombiner(ObjectAggregateCombiner delegate) + { + this.delegate = delegate; + } + + @Override + public void reset(ColumnValueSelector selector) + { + isNull = true; + } + + @Override + public void fold(ColumnValueSelector selector) + { + if (isNull && !selector.isNull()) { + isNull = true; + } + delegate.fold(selector); + } + + @Override + public Class classOfObject() + { + return delegate.classOfObject(); + } + + @Override + public Object get() + { + return delegate.get(); + } + + @Override + public boolean isNull() + { + return isNull; + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/ObjectAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/ObjectAggregateCombiner.java index 29bc0950982f..b268d568a3f1 100644 --- a/processing/src/main/java/io/druid/query/aggregation/ObjectAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/ObjectAggregateCombiner.java @@ -26,4 +26,9 @@ */ public abstract class ObjectAggregateCombiner implements AggregateCombiner, ObjectColumnSelector { + @Override + public boolean isNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index 24441d029028..72125cc43ceb 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -26,6 +26,7 @@ import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DoubleColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.util.Collections; import java.util.Comparator; @@ -81,7 +82,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Double.BYTES; + return Double.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java index 98461cb62c37..4f6337c436ed 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -25,6 +25,7 @@ import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.FloatColumnSelector; +import io.druid.segment.NullHandlingHelper; import java.util.Collections; import java.util.Comparator; @@ -80,7 +81,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Float.BYTES; + return Float.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java index fb78d6c2453f..6008e9f3d01b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java @@ -23,12 +23,14 @@ import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.IndexedInts; import it.unimi.dsi.fastutil.ints.IntIterator; import java.util.Arrays; -public class StringCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy +public class StringCardinalityAggregatorColumnSelectorStrategy + implements CardinalityAggregatorColumnSelectorStrategy { public static final String CARDINALITY_AGG_NULL_STRING = "\u0000"; public static final char CARDINALITY_AGG_SEPARATOR = '\u0001'; @@ -41,20 +43,28 @@ public void hashRow(DimensionSelector dimSelector, Hasher hasher) // nothing to add to hasher if size == 0, only handle size == 1 and size != 0 cases. if (size == 1) { final String value = dimSelector.lookupName(row.get(0)); - hasher.putUnencodedChars(nullToSpecial(value)); + if (NullHandlingHelper.useDefaultValuesForNull() || value != null) { + hasher.putUnencodedChars(nullToSpecial(value)); + } } else if (size != 0) { + boolean hasNonNullValue = false; final String[] values = new String[size]; for (int i = 0; i < size; ++i) { final String value = dimSelector.lookupName(row.get(i)); + if (!NullHandlingHelper.useDefaultValuesForNull() && !hasNonNullValue && value != null) { + hasNonNullValue = true; + } values[i] = nullToSpecial(value); } - // Values need to be sorted to ensure consistent multi-value ordering across different segments - Arrays.sort(values); - for (int i = 0; i < size; ++i) { - if (i != 0) { - hasher.putChar(CARDINALITY_AGG_SEPARATOR); + if (NullHandlingHelper.useDefaultValuesForNull() || hasNonNullValue) { + // Values need to be sorted to ensure consistent multi-value ordering across different segments + Arrays.sort(values); + for (int i = 0; i < size; ++i) { + if (i != 0) { + hasher.putChar(CARDINALITY_AGG_SEPARATOR); + } + hasher.putUnencodedChars(values[i]); } - hasher.putUnencodedChars(values[i]); } } } @@ -65,7 +75,9 @@ public void hashValues(DimensionSelector dimSelector, HyperLogLogCollector colle for (IntIterator rowIt = dimSelector.getRow().iterator(); rowIt.hasNext(); ) { int index = rowIt.nextInt(); final String value = dimSelector.lookupName(index); - collector.add(CardinalityAggregator.hashFn.hashUnencodedChars(nullToSpecial(value)).asBytes()); + if (NullHandlingHelper.useDefaultValuesForNull() || value != null) { + collector.add(CardinalityAggregator.hashFn.hashUnencodedChars(nullToSpecial(value)).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 783ba82d2b3b..fc0d270c347b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -27,14 +27,16 @@ import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; +import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorFactoryNotMergeableException; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; -import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DoubleColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; @@ -76,20 +78,23 @@ public DoubleFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleFirstAggregator( + + DoubleColumnSelector doubleColumnSelector = metricFactory.makeDoubleColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new DoubleFirstAggregator( name, metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeDoubleColumnSelector(fieldName) - ); + doubleColumnSelector + ), doubleColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleFirstBufferAggregator( + DoubleColumnSelector doubleColumnSelector = metricFactory.makeDoubleColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new DoubleFirstBufferAggregator( metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeDoubleColumnSelector(fieldName) - ); + doubleColumnSelector + ), doubleColumnSelector); } @Override @@ -119,7 +124,7 @@ public AggregatorFactory getCombiningFactory() public Aggregator factorize(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new DoubleFirstAggregator(name, null, null) + return NullHandlingHelper.getNullableAggregator(new DoubleFirstAggregator(name, null, null) { @Override public void aggregate() @@ -130,14 +135,14 @@ public void aggregate() firstValue = pair.rhs; } } - }; + }, selector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new DoubleFirstBufferAggregator(null, null) + return NullHandlingHelper.getNullableAggregator(new DoubleFirstBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -155,7 +160,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("selector", selector); } - }; + }, selector); } }; } @@ -228,7 +233,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES + Double.BYTES; + return Long.BYTES + Double.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index e26b0228f8a2..3af4462b3494 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -27,14 +27,16 @@ import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; +import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorFactoryNotMergeableException; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; -import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; @@ -76,20 +78,22 @@ public FloatFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new FloatFirstAggregator( + FloatColumnSelector floatColumnSelector = metricFactory.makeFloatColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new FloatFirstAggregator( name, metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeFloatColumnSelector(fieldName) - ); + floatColumnSelector + ), floatColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new FloatFirstBufferAggregator( + FloatColumnSelector floatColumnSelector = metricFactory.makeFloatColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new FloatFirstBufferAggregator( metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeFloatColumnSelector(fieldName) - ); + floatColumnSelector + ), floatColumnSelector); } @Override @@ -119,7 +123,7 @@ public AggregatorFactory getCombiningFactory() public Aggregator factorize(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new FloatFirstAggregator(name, null, null) + return NullHandlingHelper.getNullableAggregator(new FloatFirstAggregator(name, null, null) { @Override public void aggregate() @@ -130,14 +134,14 @@ public void aggregate() firstValue = pair.rhs; } } - }; + }, selector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new FloatFirstBufferAggregator(null, null) + return NullHandlingHelper.getNullableAggregator(new FloatFirstBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -155,7 +159,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("selector", selector); } - }; + }, selector); } }; } @@ -228,7 +232,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES + Float.BYTES; + return Long.BYTES + Float.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 96ff43d2792d..9fd80a908825 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -26,14 +26,16 @@ import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; +import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorFactoryNotMergeableException; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; -import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; @@ -69,20 +71,22 @@ public LongFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongFirstAggregator( + LongColumnSelector longColumnSelector = metricFactory.makeLongColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new LongFirstAggregator( name, metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeLongColumnSelector(fieldName) - ); + longColumnSelector + ), longColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongFirstBufferAggregator( + LongColumnSelector longColumnSelector = metricFactory.makeLongColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new LongFirstBufferAggregator( metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeLongColumnSelector(fieldName) - ); + longColumnSelector + ), longColumnSelector); } @Override @@ -112,7 +116,7 @@ public AggregatorFactory getCombiningFactory() public Aggregator factorize(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new LongFirstAggregator(name, null, null) + return NullHandlingHelper.getNullableAggregator(new LongFirstAggregator(name, null, null) { @Override public void aggregate() @@ -123,14 +127,14 @@ public void aggregate() firstValue = pair.rhs; } } - }; + }, selector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new LongFirstBufferAggregator(null, null) + return NullHandlingHelper.getNullableAggregator(new LongFirstBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -148,7 +152,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("selector", selector); } - }; + }, selector); } }; } @@ -221,7 +225,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES * 2; + return Long.BYTES * 2 + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index ddaec8698897..35f7c13888a5 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -36,6 +36,8 @@ import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DoubleColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; @@ -67,20 +69,22 @@ public DoubleLastAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleLastAggregator( + DoubleColumnSelector doubleColumnSelector = metricFactory.makeDoubleColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new DoubleLastAggregator( name, metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeDoubleColumnSelector(fieldName) - ); + doubleColumnSelector + ), doubleColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleLastBufferAggregator( + DoubleColumnSelector doubleColumnSelector = metricFactory.makeDoubleColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new DoubleLastBufferAggregator( metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeDoubleColumnSelector(fieldName) - ); + doubleColumnSelector + ), doubleColumnSelector); } @Override @@ -110,7 +114,7 @@ public AggregatorFactory getCombiningFactory() public Aggregator factorize(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new DoubleLastAggregator(name, null, null) + return NullHandlingHelper.getNullableAggregator(new DoubleLastAggregator(name, null, null) { @Override public void aggregate() @@ -121,14 +125,14 @@ public void aggregate() lastValue = pair.rhs; } } - }; + }, selector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new DoubleLastBufferAggregator(null, null) + return NullHandlingHelper.getNullableAggregator(new DoubleLastBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -146,7 +150,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("selector", selector); } - }; + }, selector); } }; } @@ -219,7 +223,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES + Double.BYTES; + return Long.BYTES + Double.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 34f818e490bf..44e613083333 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -26,16 +26,18 @@ import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; +import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorFactoryNotMergeableException; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; -import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.first.FloatFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; @@ -67,20 +69,23 @@ public FloatLastAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new FloatLastAggregator( + + FloatColumnSelector floatColumnSelector = metricFactory.makeFloatColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new FloatLastAggregator( name, metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeFloatColumnSelector(fieldName) - ); + floatColumnSelector + ), floatColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new FloatLastBufferAggregator( + FloatColumnSelector floatColumnSelector = metricFactory.makeFloatColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new FloatLastBufferAggregator( metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeFloatColumnSelector(fieldName) - ); + floatColumnSelector + ), floatColumnSelector); } @Override @@ -110,7 +115,7 @@ public AggregatorFactory getCombiningFactory() public Aggregator factorize(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new FloatLastAggregator(name, null, null) + return NullHandlingHelper.getNullableAggregator(new FloatLastAggregator(name, null, null) { @Override public void aggregate() @@ -121,14 +126,14 @@ public void aggregate() lastValue = pair.rhs; } } - }; + }, selector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new FloatLastBufferAggregator(null, null) + return NullHandlingHelper.getNullableAggregator(new FloatLastBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -146,7 +151,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("selector", selector); } - }; + }, selector); } }; } @@ -220,7 +225,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES + Float.BYTES; + return Long.BYTES + Float.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 23bd19345231..18ef42250d0a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -25,16 +25,18 @@ import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; +import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorFactoryNotMergeableException; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; -import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; @@ -65,20 +67,22 @@ public LongLastAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongLastAggregator( + LongColumnSelector longColumnSelector = metricFactory.makeLongColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new LongLastAggregator( name, metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeLongColumnSelector(fieldName) - ); + longColumnSelector + ), longColumnSelector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongLastBufferAggregator( + LongColumnSelector longColumnSelector = metricFactory.makeLongColumnSelector(fieldName); + return NullHandlingHelper.getNullableAggregator(new LongLastBufferAggregator( metricFactory.makeLongColumnSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeLongColumnSelector(fieldName) - ); + longColumnSelector + ), longColumnSelector); } @Override @@ -108,7 +112,7 @@ public AggregatorFactory getCombiningFactory() public Aggregator factorize(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new LongLastAggregator(name, null, null) + return NullHandlingHelper.getNullableAggregator(new LongLastAggregator(name, null, null) { @Override public void aggregate() @@ -119,14 +123,14 @@ public void aggregate() lastValue = pair.rhs; } } - }; + }, selector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { final ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(name); - return new LongLastBufferAggregator(null, null) + return NullHandlingHelper.getNullableAggregator(new LongLastBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -144,7 +148,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("selector", selector); } - }; + }, selector); } }; } @@ -217,7 +221,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES * 2; + return Long.BYTES * 2 + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); } @Override From 40b622867d8aaa164623b8cc2b1b64343fbbf63d Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 02:12:56 +0530 Subject: [PATCH 04/50] Querying part 2 : Filters + DimensionSpec + lookups --- .../dimension/ListFilteredDimensionSpec.java | 6 +-- .../query/dimension/LookupDimensionSpec.java | 6 ++- .../dimension/RegexFilteredDimensionSpec.java | 7 ++-- .../extraction/JavaScriptExtractionFn.java | 6 +-- .../extraction/RegexDimExtractionFn.java | 9 +++-- ...bleValueMatcherColumnSelectorStrategy.java | 29 ++++++++++---- .../query/filter/DruidDoublePredicate.java | 20 ++++++++++ .../query/filter/DruidFloatPredicate.java | 19 ++++++++++ .../query/filter/DruidLongPredicate.java | 20 ++++++++++ ...oatValueMatcherColumnSelectorStrategy.java | 29 ++++++++++---- .../io/druid/query/filter/InDimFilter.java | 38 ++++++++++--------- .../io/druid/query/filter/LikeDimFilter.java | 24 +++--------- ...ongValueMatcherColumnSelectorStrategy.java | 29 ++++++++++---- .../druid/query/filter/SelectorDimFilter.java | 26 ++++++++++--- ...ingValueMatcherColumnSelectorStrategy.java | 36 ++++++------------ .../segment/ZeroDoubleColumnSelector.java | 6 +++ .../segment/ZeroFloatColumnSelector.java | 5 +++ .../druid/segment/ZeroLongColumnSelector.java | 8 +++- .../io/druid/segment/filter/BoundFilter.java | 10 +++-- .../segment/filter/ExpressionFilter.java | 4 +- .../io/druid/segment/filter/InFilter.java | 5 +-- .../io/druid/segment/filter/LikeFilter.java | 6 ++- 22 files changed, 232 insertions(+), 116 deletions(-) diff --git a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java index c7ab3873b578..0af2c3cbd124 100644 --- a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java @@ -23,11 +23,11 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.base.Strings; import io.druid.java.util.common.StringUtils; import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; import io.druid.segment.IdLookup; +import io.druid.segment.NullHandlingHelper; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import javax.annotation.Nullable; @@ -106,7 +106,7 @@ private DimensionSelector filterWhiteList(DimensionSelector selector) } } else { for (int i = 0; i < selectorCardinality; i++) { - if (values.contains(Strings.nullToEmpty(selector.lookupName(i)))) { + if (values.contains(NullHandlingHelper.nullToDefault(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } @@ -137,7 +137,7 @@ public boolean apply(@Nullable String input) forwardMapping.defaultReturnValue(-1); final int[] reverseMapping = new int[maxPossibleFilteredCardinality]; for (int i = 0; i < selectorCardinality; i++) { - if (!values.contains(Strings.nullToEmpty(selector.lookupName(i)))) { + if (!values.contains(NullHandlingHelper.nullToDefault(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } diff --git a/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java index 97448a55cb10..d03ddb741688 100644 --- a/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -31,6 +31,7 @@ import io.druid.query.lookup.LookupExtractor; import io.druid.query.lookup.LookupReferencesManager; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.ValueType; import javax.annotation.Nullable; @@ -77,7 +78,7 @@ public LookupDimensionSpec( { this.retainMissingValue = retainMissingValue; this.optimize = optimize == null ? true : optimize; - this.replaceMissingValueWith = Strings.emptyToNull(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandlingHelper.defaultToNull(replaceMissingValueWith); this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be Null"); this.outputName = Preconditions.checkNotNull(outputName, "outputName can not be Null"); this.lookupReferencesManager = lookupReferencesManager; @@ -174,7 +175,7 @@ public byte[] getCacheKey() byte[] replaceWithBytes = StringUtils.toUtf8(Strings.nullToEmpty(replaceMissingValueWith)); - return ByteBuffer.allocate(6 + return ByteBuffer.allocate(7 + dimensionBytes.length + outputNameBytes.length + dimExtractionFnBytes.length @@ -189,6 +190,7 @@ public byte[] getCacheKey() .put(replaceWithBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(retainMissingValue ? (byte) 1 : (byte) 0) + .put(replaceMissingValueWith == null ? (byte) 0 : (byte) 1) .array(); } diff --git a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java index 4ebd407083f4..cee0d2ba5f78 100644 --- a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -22,10 +22,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import io.druid.java.util.common.StringUtils; import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; import it.unimi.dsi.fastutil.ints.Int2IntMap; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; @@ -76,7 +76,7 @@ public DimensionSelector decorate(final DimensionSelector selector) @Override public boolean apply(@Nullable String input) { - return compiledRegex.matcher(Strings.nullToEmpty(input)).matches(); + return compiledRegex.matcher(NullHandlingHelper.nullToDefault(input)).matches(); } } ); @@ -86,7 +86,8 @@ public boolean apply(@Nullable String input) final Int2IntOpenHashMap forwardMapping = new Int2IntOpenHashMap(); forwardMapping.defaultReturnValue(-1); for (int i = 0; i < selectorCardinality; i++) { - if (compiledRegex.matcher(Strings.nullToEmpty(selector.lookupName(i))).matches()) { + String val = NullHandlingHelper.nullToDefault(selector.lookupName(i)); + if (val != null && compiledRegex.matcher(val).matches()) { forwardMapping.put(i, count++); } } diff --git a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java index 51efd1368441..0a5012e45013 100644 --- a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java @@ -24,10 +24,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; import io.druid.js.JavaScriptConfig; +import io.druid.segment.NullHandlingHelper; import org.mozilla.javascript.Context; import org.mozilla.javascript.ContextFactory; import org.mozilla.javascript.ScriptableObject; @@ -119,14 +119,14 @@ public String apply(@Nullable Object value) throw new ISE("JavaScript is disabled"); } - return Strings.emptyToNull(fn.apply(value)); + return NullHandlingHelper.defaultToNull(fn.apply(value)); } @Override @Nullable public String apply(@Nullable String value) { - return this.apply((Object) Strings.emptyToNull(value)); + return this.apply((Object) NullHandlingHelper.defaultToNull(value)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java index 3a1b0a5cc293..1f5aceb3fc8f 100644 --- a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java @@ -22,9 +22,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.primitives.Ints; import io.druid.java.util.common.StringUtils; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -106,13 +106,14 @@ public byte[] getCacheKey() public String apply(@Nullable String dimValue) { final String retVal; - final Matcher matcher = pattern.matcher(Strings.nullToEmpty(dimValue)); - if (matcher.find()) { + String val = NullHandlingHelper.nullToDefault(dimValue); + final Matcher matcher = val == null ? null : pattern.matcher(val); + if (matcher != null && matcher.find()) { retVal = matcher.group(index); } else { retVal = replaceMissingValue ? replaceMissingValueWith : dimValue; } - return Strings.emptyToNull(retVal); + return NullHandlingHelper.defaultToNull(retVal); } @JsonProperty("expr") diff --git a/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java index f2c20e645ce5..d9de977f2126 100644 --- a/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java @@ -22,7 +22,6 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.DoubleColumnSelector; -import io.druid.segment.filter.BooleanValueMatcher; public class DoubleValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy @@ -32,7 +31,20 @@ public ValueMatcher makeValueMatcher(final DoubleColumnSelector selector, final { final Double matchVal = DimensionHandlerUtils.convertObjectToDouble(value); if (matchVal == null) { - return BooleanValueMatcher.of(false); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } final long matchValLongBits = Double.doubleToLongBits(matchVal); @@ -63,6 +75,9 @@ public ValueMatcher makeValueMatcher( @Override public boolean matches() { + if (selector.isNull()) { + return predicate.applyNull(); + } return predicate.applyDouble(selector.getDouble()); } @@ -78,13 +93,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueGetter makeValueGetter(final DoubleColumnSelector selector) { - return new ValueGetter() - { - @Override - public String[] get() - { - return new String[]{Double.toString(selector.getDouble())}; + return () -> { + if (selector.isNull()) { + return null; } + return new String[]{Double.toString(selector.getDouble())}; }; } } diff --git a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java index c28b8d376873..f1e2874ea6ee 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java @@ -26,5 +26,25 @@ public interface DruidDoublePredicate DruidDoublePredicate ALWAYS_TRUE = input -> true; + DruidDoublePredicate MATCH_NULL_ONLY = new DruidDoublePredicate() + { + @Override + public boolean applyDouble(double input) + { + return false; + } + + @Override + public boolean applyNull() + { + return true; + } + }; + boolean applyDouble(double input); + + default boolean applyNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java index f38a298c7597..9e7c0fdc5b54 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java @@ -28,6 +28,25 @@ public interface DruidFloatPredicate DruidFloatPredicate ALWAYS_TRUE = input -> true; + DruidFloatPredicate MATCH_NULL_ONLY = new DruidFloatPredicate() + { + @Override + public boolean applyFloat(float input) + { + return false; + } + + @Override + public boolean applyNull() + { + return true; + } + }; boolean applyFloat(float input); + + default boolean applyNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java index 2afca5c40e10..4175c6bcbe18 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java @@ -28,5 +28,25 @@ public interface DruidLongPredicate DruidLongPredicate ALWAYS_TRUE = input -> true; + DruidLongPredicate MATCH_NULL_ONLY = new DruidLongPredicate() + { + @Override + public boolean applyLong(long input) + { + return false; + } + + @Override + public boolean applyNull() + { + return true; + } + }; + boolean applyLong(long input); + + default boolean applyNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java index 209678de818b..0a887b4bb987 100644 --- a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java @@ -22,7 +22,6 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.FloatColumnSelector; -import io.druid.segment.filter.BooleanValueMatcher; public class FloatValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy { @@ -31,7 +30,20 @@ public ValueMatcher makeValueMatcher(final FloatColumnSelector selector, final S { final Float matchVal = DimensionHandlerUtils.convertObjectToFloat(value); if (matchVal == null) { - return BooleanValueMatcher.of(false); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } final int matchValIntBits = Float.floatToIntBits(matchVal); @@ -62,6 +74,9 @@ public ValueMatcher makeValueMatcher( @Override public boolean matches() { + if (selector.isNull()) { + return predicate.applyNull(); + } return predicate.applyFloat(selector.getFloat()); } @@ -77,13 +92,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueGetter makeValueGetter(final FloatColumnSelector selector) { - return new ValueGetter() - { - @Override - public String[] get() - { - return new String[]{Float.toString(selector.getFloat())}; + return () -> { + if (selector.isNull()) { + return null; } + return new String[]{Float.toString(selector.getFloat())}; }; } } diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index d12d0a552742..a7c10b6ceba2 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -21,12 +21,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; @@ -34,10 +32,12 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.extraction.ExtractionFn; import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.DimensionHandlerUtils; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.filter.InFilter; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; @@ -51,6 +51,8 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; public class InDimFilter implements DimFilter { @@ -58,7 +60,7 @@ public class InDimFilter implements DimFilter // Hashing threshold is not applied to String for now, String still uses ImmutableSortedSet public static final int NUMERIC_HASHING_THRESHOLD = 16; - private final ImmutableSortedSet values; + private final SortedSet values; private final String dimension; private final ExtractionFn extractionFn; private final Supplier longPredicateSupplier; @@ -74,19 +76,11 @@ public InDimFilter( { Preconditions.checkNotNull(dimension, "dimension can not be null"); Preconditions.checkArgument(values != null && !values.isEmpty(), "values can not be null or empty"); - this.values = ImmutableSortedSet.copyOf( - Iterables.transform( - values, new Function() - { - @Override - public String apply(String input) - { - return Strings.nullToEmpty(input); - } - } - ) - ); + this.values = new TreeSet<>(Comparators.naturalNullsFirst()); + for (String value : values) { + this.values.add(NullHandlingHelper.defaultToNull(value)); + } this.dimension = dimension; this.extractionFn = extractionFn; this.longPredicateSupplier = getLongPredicateSupplier(); @@ -119,14 +113,18 @@ public byte[] getCacheKey() final byte[][] valuesBytes = new byte[values.size()][]; int valuesBytesSize = 0; int index = 0; + boolean hasNull = false; for (String value : values) { + if (value == null) { + hasNull = true; + } valuesBytes[index] = StringUtils.toUtf8(Strings.nullToEmpty(value)); valuesBytesSize += valuesBytes[index].length + 1; ++index; } byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - ByteBuffer filterCacheKey = ByteBuffer.allocate(3 + ByteBuffer filterCacheKey = ByteBuffer.allocate(5 + dimensionBytes.length + valuesBytesSize + extractionFnBytes.length) @@ -134,6 +132,8 @@ public byte[] getCacheKey() .put(dimensionBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) + .put(DimFilterUtils.STRING_SEPARATOR) + .put(hasNull ? (byte) 1 : (byte) 0) .put(DimFilterUtils.STRING_SEPARATOR); for (byte[] bytes : valuesBytes) { filterCacheKey.put(bytes) @@ -175,7 +175,7 @@ private InDimFilter optimizeLookup() // there may be row values that match the selector value but are not included // in the lookup map. Match on the selector value as well. // If the selector value is overwritten in the lookup map, don't add selector value to keys. - if (exFn.isRetainMissingValue() && lookup.apply(convertedValue) == null) { + if (exFn.isRetainMissingValue() && NullHandlingHelper.isNullOrDefault(lookup.apply(convertedValue))) { keys.add(convertedValue); } } @@ -261,7 +261,9 @@ public String toString() builder.append(")"); } - builder.append(" IN (").append(Joiner.on(", ").join(values)).append(")"); + builder.append(" IN (") + .append(Joiner.on(", ").join(Iterables.transform(values, input -> Strings.nullToEmpty(input)))) + .append(")"); return builder.toString(); } diff --git a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java index 5a98bc5d2327..ad76da563771 100644 --- a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java @@ -29,6 +29,7 @@ import com.google.common.primitives.Chars; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.Indexed; import io.druid.segment.filter.LikeFilter; @@ -151,7 +152,8 @@ private static void addPatternCharacter(final StringBuilder patternBuilder, fina public boolean matches(@Nullable final String s) { - return pattern.matcher(Strings.nullToEmpty(s)).matches(); + String val = NullHandlingHelper.nullToDefault(s); + return val != null && pattern.matcher(val).matches(); } /** @@ -165,7 +167,7 @@ public boolean matchesSuffixOnly(final Indexed strings, final int i) return true; } else if (suffixMatch == SuffixMatch.MATCH_EMPTY) { final String s = strings.get(i); - return (s == null ? 0 : s.length()) == prefix.length(); + return s == null ? matches(null) : s.length() == prefix.length(); } else { // suffixMatch is MATCH_PATTERN final String s = strings.get(i); @@ -181,23 +183,9 @@ public DruidPredicateFactory predicateFactory(final ExtractionFn extractionFn) public Predicate makeStringPredicate() { if (extractionFn != null) { - return new Predicate() - { - @Override - public boolean apply(String input) - { - return matches(extractionFn.apply(input)); - } - }; + return input -> matches(extractionFn.apply(input)); } else { - return new Predicate() - { - @Override - public boolean apply(String input) - { - return matches(input); - } - }; + return input -> matches(input); } } diff --git a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java index 73ce3bbaff5e..51d1db81b015 100644 --- a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java @@ -22,7 +22,6 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.LongColumnSelector; -import io.druid.segment.filter.BooleanValueMatcher; public class LongValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy { @@ -31,7 +30,20 @@ public ValueMatcher makeValueMatcher(final LongColumnSelector selector, final St { final Long matchVal = DimensionHandlerUtils.convertObjectToLong(value); if (matchVal == null) { - return BooleanValueMatcher.of(false); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } final long matchValLong = matchVal; return new ValueMatcher() @@ -61,6 +73,9 @@ public ValueMatcher makeValueMatcher( @Override public boolean matches() { + if (selector.isNull()) { + return predicate.applyNull(); + } return predicate.applyLong(selector.getLong()); } @@ -76,13 +91,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueGetter makeValueGetter(final LongColumnSelector selector) { - return new ValueGetter() - { - @Override - public String[] get() - { - return new String[]{Long.toString(selector.getLong())}; + return () -> { + if (selector.isNull()) { + return null; } + return new String[]{Long.toString(selector.getLong())}; }; } } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index d13971a8ea28..56d86497d9b6 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -25,7 +25,6 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; @@ -34,10 +33,12 @@ import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.filter.DimensionPredicateFilter; import io.druid.segment.filter.SelectorFilter; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Objects; /** @@ -64,7 +65,7 @@ public SelectorDimFilter( Preconditions.checkArgument(dimension != null, "dimension must not be null"); this.dimension = dimension; - this.value = Strings.nullToEmpty(value); + this.value = NullHandlingHelper.defaultToNull(value); this.extractionFn = extractionFn; } @@ -75,8 +76,9 @@ public byte[] getCacheKey() byte[] valueBytes = (value == null) ? new byte[]{} : StringUtils.toUtf8(value); byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - return ByteBuffer.allocate(3 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) + return ByteBuffer.allocate(5 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) .put(DimFilterUtils.SELECTOR_CACHE_ID) + .put(value == null ? (byte) 1 : (byte) 0) .put(dimensionBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(valueBytes) @@ -88,7 +90,7 @@ public byte[] getCacheKey() @Override public DimFilter optimize() { - return new InDimFilter(dimension, ImmutableList.of(value), extractionFn).optimize(); + return new InDimFilter(dimension, Arrays.asList(value), extractionFn).optimize(); } @Override @@ -97,14 +99,13 @@ public Filter toFilter() if (extractionFn == null) { return new SelectorFilter(dimension, value); } else { - final String valueOrNull = Strings.emptyToNull(value); final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override public Predicate makeStringPredicate() { - return Predicates.equalTo(valueOrNull); + return Predicates.equalTo(value); } @Override @@ -211,6 +212,10 @@ private void initLongPredicate() if (longPredicate != null) { return; } + if (value == null) { + longPredicate = DruidLongPredicate.MATCH_NULL_ONLY; + return; + } final Long valueAsLong = GuavaUtils.tryParseLong(value); if (valueAsLong == null) { longPredicate = DruidLongPredicate.ALWAYS_FALSE; @@ -231,6 +236,11 @@ private void initFloatPredicate() if (floatPredicate != null) { return; } + + if (value == null) { + floatPredicate = DruidFloatPredicate.MATCH_NULL_ONLY; + return; + } final Float valueAsFloat = Floats.tryParse(value); if (valueAsFloat == null) { @@ -251,6 +261,10 @@ private void initDoublePredicate() if (druidDoublePredicate != null) { return; } + if (value == null) { + druidDoublePredicate = DruidDoublePredicate.MATCH_NULL_ONLY; + return; + } final Double aDouble = Doubles.tryParse(value); if (aDouble == null) { diff --git a/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java index e380b1162ee3..aa3dfe14aa4f 100644 --- a/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java @@ -20,7 +20,6 @@ package io.druid.query.filter; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.BooleanValueMatcher; @@ -28,19 +27,11 @@ public class StringValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy { private static final String[] NULL_VALUE = new String[]{null}; - private static final ValueGetter NULL_VALUE_GETTER = new ValueGetter() - { - @Override - public String[] get() - { - return NULL_VALUE; - } - }; + private static final ValueGetter NULL_VALUE_GETTER = () -> NULL_VALUE; @Override public ValueMatcher makeValueMatcher(final DimensionSelector selector, String value) { - value = Strings.emptyToNull(value); if (selector.getValueCardinality() == 0) { return BooleanValueMatcher.of(value == null); } else { @@ -68,22 +59,17 @@ public ValueGetter makeValueGetter(final DimensionSelector selector) if (selector.getValueCardinality() == 0) { return NULL_VALUE_GETTER; } else { - return new ValueGetter() - { - @Override - public String[] get() - { - final IndexedInts row = selector.getRow(); - final int size = row.size(); - if (size == 0) { - return NULL_VALUE; - } else { - String[] values = new String[size]; - for (int i = 0; i < size; ++i) { - values[i] = Strings.emptyToNull(selector.lookupName(row.get(i))); - } - return values; + return () -> { + final IndexedInts row = selector.getRow(); + final int size = row.size(); + if (size == 0) { + return NULL_VALUE; + } else { + String[] values = new String[size]; + for (int i = 0; i < size; ++i) { + values[i] = selector.lookupName(row.get(i)); } + return values; } }; } diff --git a/processing/src/main/java/io/druid/segment/ZeroDoubleColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroDoubleColumnSelector.java index 0e6544fe9997..fd1069879079 100644 --- a/processing/src/main/java/io/druid/segment/ZeroDoubleColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ZeroDoubleColumnSelector.java @@ -41,6 +41,12 @@ public double getDouble() return 0.0d; } + @Override + public boolean isNull() + { + return false; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java index 39d9eb13bc2d..585e4ce69efd 100644 --- a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java @@ -48,6 +48,11 @@ public double getDouble(int offset) return 0.0; } + public boolean isNull() + { + return false; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java index 7117b824a798..073964202227 100644 --- a/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java @@ -38,7 +38,13 @@ public static ZeroLongColumnSelector instance() @Override public long getLong() { - return 0; + return 0L; + } + + @Override + public boolean isNull() + { + return false; } @Override diff --git a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java index c9fc4262a1ee..bdc134d46f0e 100644 --- a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java @@ -37,6 +37,7 @@ import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.IntListUtils; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.BitmapIndex; import it.unimi.dsi.fastutil.ints.IntList; @@ -148,7 +149,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasLowerBound()) { startIndex = 0; } else { - final int found = bitmapIndex.getIndex(boundDimFilter.getLower()); + final int found = bitmapIndex.getIndex(NullHandlingHelper.defaultToNull(boundDimFilter.getLower())); if (found >= 0) { startIndex = boundDimFilter.isLowerStrict() ? found + 1 : found; } else { @@ -159,7 +160,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasUpperBound()) { endIndex = bitmapIndex.getCardinality(); } else { - final int found = bitmapIndex.getIndex(boundDimFilter.getUpper()); + final int found = bitmapIndex.getIndex(NullHandlingHelper.defaultToNull(boundDimFilter.getUpper())); if (found >= 0) { endIndex = boundDimFilter.isUpperStrict() ? found : found + 1; } else { @@ -249,9 +250,10 @@ private boolean doesMatch(String input) { if (input == null) { return (!boundDimFilter.hasLowerBound() - || (boundDimFilter.getLower().isEmpty() && !boundDimFilter.isLowerStrict())) // lower bound allows null + || (NullHandlingHelper.isNullOrDefault(boundDimFilter.getLower()) && !boundDimFilter.isLowerStrict())) + // lower bound allows null && (!boundDimFilter.hasUpperBound() - || !boundDimFilter.getUpper().isEmpty() + || !NullHandlingHelper.isNullOrDefault(boundDimFilter.getUpper()) || !boundDimFilter.isUpperStrict()); // upper bound allows null } int lowerComparing = 1; diff --git a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java index e63f718dfcfe..8073e19c8845 100644 --- a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java @@ -33,6 +33,7 @@ import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.virtual.ExpressionSelectors; import java.util.Set; @@ -107,7 +108,8 @@ public T getBitmapResult(final BitmapIndexSelector selector, final BitmapRes value -> expr.eval(identifierName -> { // There's only one binding, and it must be the single column, so it can safely be ignored in production. assert column.equals(identifierName); - return value; + // convert null to Empty before passing to expressions if needed. + return NullHandlingHelper.nullToDefault(value); }).asBoolean() ); } diff --git a/processing/src/main/java/io/druid/segment/filter/InFilter.java b/processing/src/main/java/io/druid/segment/filter/InFilter.java index 559c5b3a17cd..4babdd4d7cd4 100644 --- a/processing/src/main/java/io/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/InFilter.java @@ -20,7 +20,6 @@ package io.druid.segment.filter; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.BitmapResultFactory; @@ -166,9 +165,9 @@ private DruidPredicateFactory getPredicateFactory() public Predicate makeStringPredicate() { if (extractionFn != null) { - return input -> values.contains(Strings.nullToEmpty(extractionFn.apply(input))); + return input -> values.contains(extractionFn.apply(input)); } else { - return input -> values.contains(Strings.nullToEmpty(input)); + return input -> values.contains(input); } } diff --git a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java index 8965de29638a..20d9429ce194 100644 --- a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java @@ -30,6 +30,7 @@ import io.druid.query.filter.ValueMatcher; import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.BitmapIndex; import io.druid.segment.data.Indexed; import it.unimi.dsi.fastutil.ints.AbstractIntIterator; @@ -91,7 +92,10 @@ private Iterable getBitmapIterable(final BitmapIndexSelector se { if (isSimpleEquals()) { // Verify that dimension equals prefix. - return ImmutableList.of(selector.getBitmapIndex(dimension, likeMatcher.getPrefix())); + return ImmutableList.of(selector.getBitmapIndex( + dimension, + NullHandlingHelper.defaultToNull(likeMatcher.getPrefix()) + )); } else if (isSimplePrefix()) { // Verify that dimension startsWith prefix, and is accepted by likeMatcher.matchesSuffixOnly. final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension); From 6fe82f68d4a76ff53653e9b07ce43e2ea1074d66 Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 02:15:28 +0530 Subject: [PATCH 05/50] Querying : Expressions + SQL layer + lookup --- .../main/java/io/druid/math/expr/Expr.java | 8 +++-- .../java/io/druid/math/expr/ExprEval.java | 3 +- .../java/io/druid/math/expr/Function.java | 20 +++++++++--- .../java/io/druid/math/expr/EvalTest.java | 2 +- .../java/io/druid/math/expr/FunctionTest.java | 2 +- .../io/druid/query/expression/ExprUtils.java | 6 +++- .../druid/query/expression/LikeExprMacro.java | 4 +-- .../expression/RegexpExtractExprMacro.java | 7 +++-- .../extraction/FunctionalExtraction.java | 11 ++++--- .../query/lookup/LookupExtractionFn.java | 7 +++-- .../druid/query/lookup/LookupExtractor.java | 3 +- .../virtual/ExpressionObjectSelector.java | 13 +++++--- .../segment/virtual/ExpressionSelectors.java | 31 +++++++++++++++---- .../expression/CeilOperatorConversion.java | 4 +-- .../calcite/expression/DruidExpression.java | 5 +-- .../sql/calcite/expression/Expressions.java | 18 +++++------ .../TimeFloorOperatorConversion.java | 4 +-- .../druid/sql/calcite/planner/Calcites.java | 3 +- .../druid/sql/calcite/rel/DruidSemiJoin.java | 16 ++++++++-- .../io/druid/sql/calcite/rel/QueryMaker.java | 4 +-- 20 files changed, 112 insertions(+), 59 deletions(-) diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index 913f40244bc8..1630bf8e5fbf 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -20,7 +20,6 @@ package io.druid.math.expr; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.math.LongMath; import com.google.common.primitives.Ints; import io.druid.java.util.common.IAE; @@ -124,7 +123,7 @@ class StringExpr extends ConstantExpr public StringExpr(String value) { - this.value = Strings.emptyToNull(value); + this.value = value; } @Nullable @@ -491,7 +490,10 @@ class BinPlusExpr extends BinaryEvalOpExprBase @Override protected ExprEval evalString(@Nullable String left, @Nullable String right) { - return ExprEval.of(Strings.nullToEmpty(left) + Strings.nullToEmpty(right)); + if (left == null || right == null) { + return ExprEval.of(null); + } + return ExprEval.of(left + right); } @Override diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 630347982a22..676a616e0f49 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -20,7 +20,6 @@ package io.druid.math.expr; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.primitives.Doubles; import com.google.common.primitives.Ints; import io.druid.common.guava.GuavaUtils; @@ -230,7 +229,7 @@ private static class StringExprEval extends ExprEval { private StringExprEval(String value) { - super(Strings.emptyToNull(value)); + super(value); } @Override diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index e1510a9647f4..ac95075a7788 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -902,6 +902,9 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String s = args.get(i).eval(bindings).asString(); if (s != null) { builder.append(s); + } else { + // Result of concatenation is null if any of the Values is null. + return ExprEval.of(null); } } return ExprEval.of(builder.toString()); @@ -961,7 +964,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) return ExprEval.of(arg.substring(index)); } } else { - return ExprEval.of(null); + return ExprEval.of(""); } } } @@ -984,8 +987,11 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); final String pattern = args.get(1).eval(bindings).asString(); final String replacement = args.get(2).eval(bindings).asString(); + if (arg == null) { + return ExprEval.of(null); + } return ExprEval.of( - Strings.nullToEmpty(arg).replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) + arg.replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) ); } } @@ -1006,7 +1012,10 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } final String arg = args.get(0).eval(bindings).asString(); - return ExprEval.of(StringUtils.toLowerCase(Strings.nullToEmpty(arg))); + if (arg == null) { + return ExprEval.of(null); + } + return ExprEval.of(StringUtils.toLowerCase(arg)); } } @@ -1026,7 +1035,10 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } final String arg = args.get(0).eval(bindings).asString(); - return ExprEval.of(StringUtils.toUpperCase(Strings.nullToEmpty(arg))); + if (arg == null) { + return ExprEval.of(null); + } + return ExprEval.of(StringUtils.toUpperCase(arg)); } } } diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java index 75a06a03d6eb..250eaf96d63b 100644 --- a/common/src/test/java/io/druid/math/expr/EvalTest.java +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -140,7 +140,7 @@ public void testLongEval() Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01+09:00')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01.419+09:00')", bindings)); - Assert.assertEquals("NULL", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); + Assert.assertEquals("", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); Assert.assertEquals("x", eval("nvl(if(x == 9223372036854775806, '', 'x'), 'NULL')", bindings).asString()); } diff --git a/common/src/test/java/io/druid/math/expr/FunctionTest.java b/common/src/test/java/io/druid/math/expr/FunctionTest.java index 76f590ad875e..35185eeebbaf 100644 --- a/common/src/test/java/io/druid/math/expr/FunctionTest.java +++ b/common/src/test/java/io/druid/math/expr/FunctionTest.java @@ -55,7 +55,7 @@ public void testCaseSearched() public void testConcat() { assertExpr("concat(x,' ',y)", "foo 2"); - assertExpr("concat(x,' ',nonexistent,' ',y)", "foo 2"); + assertExpr("concat(x,' ',nonexistent,' ',y)", null); assertExpr("concat(z)", "3.1"); assertExpr("concat()", null); } diff --git a/processing/src/main/java/io/druid/query/expression/ExprUtils.java b/processing/src/main/java/io/druid/query/expression/ExprUtils.java index 74d9218b258e..d4d3e4ae816a 100644 --- a/processing/src/main/java/io/druid/query/expression/ExprUtils.java +++ b/processing/src/main/java/io/druid/query/expression/ExprUtils.java @@ -22,6 +22,7 @@ import io.druid.java.util.common.IAE; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.math.expr.Expr; +import org.apache.logging.log4j.util.Strings; import org.joda.time.Chronology; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -70,7 +71,10 @@ public static PeriodGranularity toPeriodGranularity( } else { Chronology chronology = timeZone == null ? ISOChronology.getInstanceUTC() : ISOChronology.getInstance(timeZone); final Object value = originArg.eval(bindings).value(); - origin = value != null ? new DateTime(value, chronology) : null; + origin = !(value == null || (value instanceof String && Strings.isBlank((String) value))) ? new DateTime( + value, + chronology + ) : null; } return new PeriodGranularity(period, origin, timeZone); diff --git a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java index c60400be33eb..ebdb15465c9e 100644 --- a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java @@ -19,13 +19,13 @@ package io.druid.query.expression; -import com.google.common.base.Strings; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.ExprType; import io.druid.query.filter.LikeDimFilter; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nonnull; import java.util.List; @@ -63,7 +63,7 @@ public Expr apply(final List args) } final LikeDimFilter.LikeMatcher likeMatcher = LikeDimFilter.LikeMatcher.from( - Strings.nullToEmpty((String) patternExpr.getLiteralValue()), + NullHandlingHelper.nullToDefault((String) patternExpr.getLiteralValue()), escapeChar ); diff --git a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java index f9a4273a05ae..7d48546ea32d 100644 --- a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java @@ -19,11 +19,11 @@ package io.druid.query.expression; -import com.google.common.base.Strings; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nonnull; import java.util.List; @@ -63,9 +63,10 @@ class RegexpExtractExpr implements Expr @Override public ExprEval eval(final ObjectBinding bindings) { - final Matcher matcher = pattern.matcher(Strings.nullToEmpty(arg.eval(bindings).asString())); + String s = arg.eval(bindings).asString(); + final Matcher matcher = pattern.matcher(NullHandlingHelper.nullToDefault(s)); final String retVal = matcher.find() ? matcher.group(index) : null; - return ExprEval.of(Strings.emptyToNull(retVal)); + return ExprEval.of(NullHandlingHelper.defaultToNull(retVal)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java index bd762574efd1..7cae1c31d7ec 100644 --- a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java +++ b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; @@ -52,9 +53,9 @@ public FunctionalExtraction( ) { this.retainMissingValue = retainMissingValue; - this.replaceMissingValueWith = Strings.emptyToNull(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandlingHelper.defaultToNull(replaceMissingValueWith); Preconditions.checkArgument( - !(this.retainMissingValue && !Strings.isNullOrEmpty(this.replaceMissingValueWith)), + !(this.retainMissingValue && !(this.replaceMissingValueWith == null)), "Cannot specify a [replaceMissingValueWith] and set [retainMissingValue] to true" ); @@ -79,8 +80,10 @@ public String apply(@Nullable String dimValue) @Override public String apply(@Nullable String dimValue) { - final String retval = extractionFunction.apply(dimValue); - return Strings.isNullOrEmpty(retval) ? FunctionalExtraction.this.replaceMissingValueWith : retval; + final String retval = NullHandlingHelper.defaultToNull(extractionFunction.apply(dimValue)); + return retval == null + ? FunctionalExtraction.this.replaceMissingValueWith + : retval; } }; } diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java index b0c65a7e098f..ae4752f56bf0 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java @@ -23,11 +23,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.base.Throwables; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionCacheHelper; import io.druid.query.extraction.FunctionalExtraction; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.io.ByteArrayOutputStream; @@ -56,9 +56,9 @@ public LookupExtractionFn( { @Nullable @Override - public String apply(String input) + public String apply(@Nullable String input) { - return lookup.apply(Strings.nullToEmpty(input)); + return NullHandlingHelper.defaultToNull(lookup.apply(NullHandlingHelper.nullToDefault(input))); } }, retainMissingValue, @@ -120,6 +120,7 @@ public byte[] getCacheKey() outputStream.write(isInjective() ? 1 : 0); outputStream.write(isRetainMissingValue() ? 1 : 0); outputStream.write(isOptimize() ? 1 : 0); + outputStream.write(getReplaceMissingValueWith() == null ? 1 : 0); return outputStream.toByteArray(); } catch (IOException ex) { diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java index 0400c21ae1e6..1a487fab63d5 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java @@ -25,7 +25,6 @@ import io.druid.query.extraction.MapLookupExtractor; import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -45,7 +44,7 @@ public abstract class LookupExtractor * @return The lookup, or null key cannot have the lookup applied to it and should be treated as missing. */ @Nullable - public abstract String apply(@NotNull String key); + public abstract String apply(@Nullable String key); /** * @param keys set of keys to apply lookup for each element diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java index bb0217958678..d8883f3634e9 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java @@ -29,6 +29,7 @@ import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; @@ -97,13 +98,13 @@ static Supplier supplierFromDimensionSelector(final DimensionSelector se final IndexedInts row = selector.getRow(); if (row.size() == 0) { // Treat empty multi-value rows as nulls. - return null; + return NullHandlingHelper.nullToDefault((String) null); } else if (row.size() == 1) { - return selector.lookupName(row.get(0)); + return NullHandlingHelper.nullToDefault(selector.lookupName(row.get(0))); } else { // Can't handle multi-value rows in expressions. // Treat them as nulls until we think of something better to do. - return null; + return NullHandlingHelper.nullToDefault((String) null); } }; } @@ -124,10 +125,12 @@ static Supplier supplierFromObjectSelector(final ObjectColumnSelector se // Might be Numbers and Strings. Use a selector that double-checks. return () -> { final Object val = selector.get(); - if (val instanceof Number || val instanceof String) { + if (val instanceof String) { + return NullHandlingHelper.nullToDefault((String) val); + } else if (val instanceof Number) { return val; } else { - return null; + return NullHandlingHelper.nullToDefault((String) null); } }; } else { diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 7733975007d3..d8a46b941067 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -19,7 +19,6 @@ package io.druid.segment.virtual; -import com.google.common.base.Strings; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; import io.druid.query.extraction.ExtractionFn; @@ -29,6 +28,7 @@ import io.druid.segment.DoubleColumnSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; public class ExpressionSelectors { @@ -48,7 +48,7 @@ public static ExpressionObjectSelector makeObjectColumnSelector( public static LongColumnSelector makeLongColumnSelector( final ColumnSelectorFactory columnSelectorFactory, final Expr expression, - final long nullValue + final Long nullValue ) { final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); @@ -61,6 +61,12 @@ public long getLong() return exprEval.isNull() ? nullValue : exprEval.asLong(); } + @Override + public boolean isNull() + { + return baseSelector.get().isNull() && nullValue == null; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -73,7 +79,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) public static FloatColumnSelector makeFloatColumnSelector( final ColumnSelectorFactory columnSelectorFactory, final Expr expression, - final float nullValue + final Float nullValue ) { final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); @@ -86,6 +92,12 @@ public float getFloat() return exprEval.isNull() ? nullValue : (float) exprEval.asDouble(); } + @Override + public boolean isNull() + { + return baseSelector.get().isNull() && nullValue == null; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -98,7 +110,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) public static DoubleColumnSelector makeDoubleColumnSelector( ColumnSelectorFactory columnSelectorFactory, Expr expression, - double nullValue + Double nullValue ) { final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); @@ -111,6 +123,13 @@ public double getDouble() return exprEval.isNull() ? nullValue : exprEval.asDouble(); } + @Override + public boolean isNull() + { + final ExprEval exprEval = baseSelector.get(); + return exprEval.isNull() && nullValue == null; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -134,7 +153,7 @@ class DefaultExpressionDimensionSelector extends BaseSingleValueDimensionSelecto @Override protected String getValue() { - return Strings.emptyToNull(baseSelector.get().asString()); + return NullHandlingHelper.defaultToNull(baseSelector.get().asString()); } @Override @@ -150,7 +169,7 @@ class ExtractionExpressionDimensionSelector extends BaseSingleValueDimensionSele @Override protected String getValue() { - return extractionFn.apply(Strings.emptyToNull(baseSelector.get().asString())); + return extractionFn.apply(NullHandlingHelper.defaultToNull(baseSelector.get().asString())); } @Override diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/CeilOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/CeilOperatorConversion.java index 582d7397c523..61a48ea6c25c 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/CeilOperatorConversion.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/CeilOperatorConversion.java @@ -19,7 +19,6 @@ package io.druid.sql.calcite.expression; -import com.google.common.collect.ImmutableList; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.sql.calcite.planner.PlannerContext; @@ -31,6 +30,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import java.util.Arrays; import java.util.stream.Collectors; public class CeilOperatorConversion implements SqlOperatorConversion @@ -76,7 +76,7 @@ public DruidExpression toDruidExpression( // So there is no simple extraction for this operator. return DruidExpression.fromFunctionCall( "timestamp_ceil", - ImmutableList.of( + Arrays.asList( druidExpression.getExpression(), DruidExpression.stringLiteral(granularity.getPeriod().toString()), DruidExpression.numberLiteral( diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java index 86475a390492..8df6075aab1c 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java @@ -26,6 +26,7 @@ import io.druid.math.expr.Expr; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; @@ -55,7 +56,7 @@ public class DruidExpression private DruidExpression(final SimpleExtraction simpleExtraction, final String expression) { this.simpleExtraction = simpleExtraction; - this.expression = Preconditions.checkNotNull(expression); + this.expression = expression; } public static DruidExpression of(final SimpleExtraction simpleExtraction, final String expression) @@ -90,7 +91,7 @@ public static String stringLiteral(final String s) public static String nullLiteral() { - return "''"; + return NullHandlingHelper.useDefaultValuesForNull() ? "''" : null; } public static String functionCall(final String functionName, final List args) diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java index 52d2176b0f5a..9a46b0b47866 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java @@ -33,14 +33,15 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.TimeFormatExtractionFn; import io.druid.query.filter.AndDimFilter; -import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; import io.druid.query.filter.ExpressionDimFilter; import io.druid.query.filter.LikeDimFilter; import io.druid.query.filter.NotDimFilter; import io.druid.query.filter.OrDimFilter; +import io.druid.query.filter.SelectorDimFilter; import io.druid.query.ordering.StringComparator; import io.druid.query.ordering.StringComparators; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; import io.druid.sql.calcite.filtration.BoundRefKey; @@ -98,8 +99,8 @@ public class Expressions .build(); private static final Map UNARY_SUFFIX_OPERATOR_MAP = ImmutableMap.builder() - .put(SqlStdOperatorTable.IS_NULL, "== ''") - .put(SqlStdOperatorTable.IS_NOT_NULL, "!= ''") + .put(SqlStdOperatorTable.IS_NULL, "== " + DruidExpression.nullLiteral()) + .put(SqlStdOperatorTable.IS_NOT_NULL, "!= " + DruidExpression.nullLiteral()) .put(SqlStdOperatorTable.IS_FALSE, "<= 0") // Matches Evals.asBoolean .put(SqlStdOperatorTable.IS_NOT_TRUE, "<= 0") // Matches Evals.asBoolean .put(SqlStdOperatorTable.IS_TRUE, "> 0") // Matches Evals.asBoolean @@ -470,13 +471,10 @@ private static DimFilter toSimpleLeafFilter( return null; } - final BoundDimFilter equalFilter = Bounds.equalTo( - new BoundRefKey( - druidExpression.getSimpleExtraction().getColumn(), - druidExpression.getSimpleExtraction().getExtractionFn(), - StringComparators.LEXICOGRAPHIC - ), - "" + final DimFilter equalFilter = new SelectorDimFilter( + druidExpression.getSimpleExtraction().getColumn(), + NullHandlingHelper.nullToDefault((String) null), + druidExpression.getSimpleExtraction().getExtractionFn() ); return kind == SqlKind.IS_NOT_NULL ? new NotDimFilter(equalFilter) : equalFilter; diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/TimeFloorOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/TimeFloorOperatorConversion.java index 5871d190be49..3ddbf9e7b0e0 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/TimeFloorOperatorConversion.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/TimeFloorOperatorConversion.java @@ -20,7 +20,6 @@ package io.druid.sql.calcite.expression; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.sql.calcite.planner.Calcites; import io.druid.sql.calcite.planner.PlannerContext; @@ -38,6 +37,7 @@ import org.joda.time.DateTimeZone; import org.joda.time.Period; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -63,7 +63,7 @@ public static DruidExpression applyTimestampFloor( simpleExtraction -> simpleExtraction.cascade(ExtractionFns.fromQueryGranularity(granularity)), expression -> DruidExpression.functionCall( "timestamp_floor", - ImmutableList.of( + Arrays.asList( expression, DruidExpression.stringLiteral(granularity.getPeriod().toString()), DruidExpression.numberLiteral( diff --git a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java index 83f6337ef160..7bfd1836cbd9 100644 --- a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java +++ b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java @@ -28,6 +28,7 @@ import io.druid.query.ordering.StringComparator; import io.druid.query.ordering.StringComparators; import io.druid.segment.column.ValueType; +import io.druid.sql.calcite.expression.DruidExpression; import io.druid.sql.calcite.schema.DruidSchema; import io.druid.sql.calcite.schema.InformationSchema; import org.apache.calcite.jdbc.CalciteSchema; @@ -89,7 +90,7 @@ public static SchemaPlus createRootSchema(final Schema druidSchema) public static String escapeStringLiteral(final String s) { if (s == null) { - return "''"; + return DruidExpression.nullLiteral(); } else { boolean isPlainAscii = true; final StringBuilder builder = new StringBuilder("'"); diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java index bd011a4590e5..58744a6bf2eb 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java @@ -35,6 +35,9 @@ import io.druid.query.filter.DimFilter; import io.druid.query.filter.ExpressionDimFilter; import io.druid.query.filter.OrDimFilter; +import io.druid.query.filter.SelectorDimFilter; +import io.druid.segment.DimensionHandlerUtils; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.VirtualColumn; import io.druid.segment.virtual.ExpressionVirtualColumn; import io.druid.sql.calcite.expression.DruidExpression; @@ -287,7 +290,7 @@ public List accumulate(final List theFilters, final Object for (int i : rightKeys) { final Object value = row[i]; - final String stringValue = value != null ? String.valueOf(value) : ""; + final String stringValue = DimensionHandlerUtils.convertObjectToString(value); values.add(stringValue); if (values.size() > maxSemiJoinRowsInMemory) { throw new ResourceLimitExceededException( @@ -301,11 +304,18 @@ public List accumulate(final List theFilters, final Object for (int i = 0; i < values.size(); i++) { final DruidExpression leftExpression = leftExpressions.get(i); if (leftExpression.isSimpleExtraction()) { + String valToFilter = values.get(i); bounds.add( + valToFilter == null ? + new SelectorDimFilter( + leftExpression.getSimpleExtraction().getColumn(), + NullHandlingHelper.nullToDefault((String) null), + leftExpression.getSimpleExtraction().getExtractionFn() + ) : new BoundDimFilter( leftExpression.getSimpleExtraction().getColumn(), - values.get(i), - values.get(i), + valToFilter, + valToFilter, false, false, null, diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java index 6a4bac009bf6..e3f24353d617 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; @@ -47,6 +46,7 @@ import io.druid.query.topn.TopNQuery; import io.druid.query.topn.TopNResultValue; import io.druid.segment.DimensionHandlerUtils; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import io.druid.server.QueryLifecycleFactory; import io.druid.sql.calcite.planner.Calcites; @@ -400,7 +400,7 @@ private Object coerce(final Object value, final SqlTypeName sqlType) if (SqlTypeName.CHAR_TYPES.contains(sqlType)) { if (value == null || value instanceof String) { - coercedValue = Strings.nullToEmpty((String) value); + coercedValue = NullHandlingHelper.nullToDefault((String) value); } else if (value instanceof NlsString) { coercedValue = ((NlsString) value).getValue(); } else if (value instanceof Number) { From 3c1e6af55df1212cd02677e30790614811b509bc Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 02:17:41 +0530 Subject: [PATCH 06/50] Querying remaining changes + groupie --- .../io/druid/query/groupby/GroupByQuery.java | 19 +-- .../epinephelinae/GroupByQueryEngineV2.java | 5 +- .../epinephelinae/RowBasedGrouperHelper.java | 139 +++++++++++------- ...ngStringGroupByColumnSelectorStrategy.java | 3 +- .../StringGroupByColumnSelectorStrategy.java | 3 +- .../groupby/having/EqualToHavingSpec.java | 4 + .../groupby/having/GreaterThanHavingSpec.java | 4 + .../groupby/having/LessThanHavingSpec.java | 4 + .../java/io/druid/query/search/SearchHit.java | 3 +- .../druid/query/search/SearchQueryRunner.java | 3 +- .../query/search/UseIndexesStrategy.java | 3 +- .../java/io/druid/query/topn/TopNMapFn.java | 12 +- 12 files changed, 117 insertions(+), 85 deletions(-) diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index 31d3548f8c2c..8c35494da8a6 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -60,6 +60,7 @@ import io.druid.query.ordering.StringComparators; import io.druid.query.spec.LegacySegmentSpec; import io.druid.query.spec.QuerySegmentSpec; +import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.VirtualColumn; import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; @@ -572,19 +573,19 @@ private static int compareDims(List dimensions, Row lhs, Row rhs) for (DimensionSpec dimension : dimensions) { final int dimCompare; if (dimension.getOutputType() == ValueType.LONG) { - dimCompare = Long.compare( - ((Number) lhs.getRaw(dimension.getOutputName())).longValue(), - ((Number) rhs.getRaw(dimension.getOutputName())).longValue() + dimCompare = Comparators.naturalNullsFirst().compare( + DimensionHandlerUtils.convertObjectToLong(lhs.getRaw(dimension.getOutputName())), + DimensionHandlerUtils.convertObjectToLong(rhs.getRaw(dimension.getOutputName())) ); } else if (dimension.getOutputType() == ValueType.FLOAT) { - dimCompare = Float.compare( - ((Number) lhs.getRaw(dimension.getOutputName())).floatValue(), - ((Number) rhs.getRaw(dimension.getOutputName())).floatValue() + dimCompare = Comparators.naturalNullsFirst().compare( + DimensionHandlerUtils.convertObjectToFloat(lhs.getRaw(dimension.getOutputName())), + DimensionHandlerUtils.convertObjectToFloat(rhs.getRaw(dimension.getOutputName())) ); } else if (dimension.getOutputType() == ValueType.DOUBLE) { - dimCompare = Double.compare( - ((Number) lhs.getRaw(dimension.getOutputName())).doubleValue(), - ((Number) rhs.getRaw(dimension.getOutputName())).doubleValue() + dimCompare = Comparators.naturalNullsFirst().compare( + DimensionHandlerUtils.convertObjectToDouble(lhs.getRaw(dimension.getOutputName())), + DimensionHandlerUtils.convertObjectToDouble(rhs.getRaw(dimension.getOutputName())) ); } else { dimCompare = ((Ordering) Comparators.naturalNullsFirst()).compare( diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java index ffa614d4727c..c96423676302 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java @@ -684,19 +684,16 @@ private static void convertRowTypesToOutputTypes(List dimensionSp (dimName, baseVal) -> { switch (outputType) { case STRING: - baseVal = baseVal == null ? "" : baseVal.toString(); + baseVal = DimensionHandlerUtils.convertObjectToString(baseVal); break; case LONG: baseVal = DimensionHandlerUtils.convertObjectToLong(baseVal); - baseVal = baseVal == null ? 0L : baseVal; break; case FLOAT: baseVal = DimensionHandlerUtils.convertObjectToFloat(baseVal); - baseVal = baseVal == null ? 0.f : baseVal; break; case DOUBLE: baseVal = DimensionHandlerUtils.convertObjectToDouble(baseVal); - baseVal = baseVal == null ? 0.d : baseVal; break; default: throw new IAE("Unsupported type: " + outputType); diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index ab6486559f57..5bc6e750f207 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -40,6 +39,7 @@ import io.druid.java.util.common.Pair; import io.druid.java.util.common.granularity.AllGranularity; import io.druid.java.util.common.guava.Accumulator; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.BaseQuery; import io.druid.query.ColumnSelectorPlus; import io.druid.query.aggregation.AggregatorFactory; @@ -61,6 +61,7 @@ import io.druid.segment.DoubleColumnSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; @@ -75,6 +76,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; // this class contains shared code between GroupByMergingQueryRunnerV2 and GroupByRowProcessor @@ -419,7 +421,7 @@ public Row apply(Grouper.Entry entry) Object dimVal = entry.getKey().getKey()[i]; theMap.put( query.getDimensions().get(i - dimStart).getOutputName(), - dimVal instanceof String ? Strings.emptyToNull((String) dimVal) : dimVal + dimVal instanceof String ? NullHandlingHelper.defaultToNull((String) dimVal) : dimVal ); } @@ -508,18 +510,11 @@ private static class StringInputRawSupplierColumnSelectorStrategy @Override public Supplier makeInputRawSupplier(DimensionSelector selector) { - return new Supplier() - { - @Override - public Comparable get() - { - final String value; - IndexedInts index = selector.getRow(); - value = index.size() == 0 - ? "" - : selector.lookupName(index.get(0)); - return Strings.nullToEmpty(value); - } + return () -> { + IndexedInts index = selector.getRow(); + return index.size() == 0 + ? null + : selector.lookupName(index.get(0)); }; } } @@ -585,28 +580,19 @@ private static Function[] makeValueConvertFunctions( type = type == null ? ValueType.STRING : type; switch (type) { case STRING: - functions[i] = input -> input == null ? "" : input.toString(); + functions[i] = input -> DimensionHandlerUtils.convertObjectToString(input); break; case LONG: - functions[i] = input -> { - final Long val = DimensionHandlerUtils.convertObjectToLong(input); - return val == null ? 0L : val; - }; + functions[i] = input -> DimensionHandlerUtils.convertObjectToLong(input); break; case FLOAT: - functions[i] = input -> { - final Float val = DimensionHandlerUtils.convertObjectToFloat(input); - return val == null ? 0.f : val; - }; + functions[i] = input -> DimensionHandlerUtils.convertObjectToFloat(input); break; case DOUBLE: - functions[i] = input -> { - Double val = DimensionHandlerUtils.convertObjectToDouble(input); - return val == null ? 0.0 : val; - }; + functions[i] = input -> DimensionHandlerUtils.convertObjectToDouble(input); break; default: throw new IAE("invalid type: [%s]", type); @@ -830,7 +816,10 @@ public int compare(Grouper.Entry entry1, Grouper.Entry private static int compareDimsInRows(RowBasedKey key1, RowBasedKey key2, int dimStart) { for (int i = dimStart; i < key1.getKey().length; i++) { - final int cmp = ((Comparable) key1.getKey()[i]).compareTo(key2.getKey()[i]); + final int cmp = Comparators.naturalNullsFirst().compare( + (Comparable) key1.getKey()[i], + (Comparable) key2.getKey()[i] + ); if (cmp != 0) { return cmp; } @@ -879,9 +868,10 @@ private static int compareDimsInRowsWithAggs( if (isNumericField.get(i) && comparator == StringComparators.NUMERIC) { // use natural comparison - cmp = lhs.compareTo(rhs); + cmp = Comparators.naturalNullsFirst().compare(lhs, rhs); } else { - cmp = comparator.compare(lhs.toString(), rhs.toString()); + cmp = comparator.compare(DimensionHandlerUtils.convertObjectToString(lhs), + DimensionHandlerUtils.convertObjectToString(rhs)); } if (cmp != 0) { @@ -1002,7 +992,7 @@ public RowBasedKey fromByteBuffer(ByteBuffer buffer, int position) public Grouper.BufferComparator bufferComparator() { if (sortableIds == null) { - Map sortedMap = Maps.newTreeMap(); + Map sortedMap = Maps.newTreeMap(Comparators.naturalNullsFirst()); for (int id = 0; id < dictionary.size(); id++) { sortedMap.put(dictionary.get(id), id); } @@ -1323,7 +1313,8 @@ private int addToDictionary(final String s) { Integer idx = reverseDictionary.get(s); if (idx == null) { - final long additionalEstimatedSize = (long) s.length() * Chars.BYTES + ROUGH_OVERHEAD_PER_DICTIONARY_ENTRY; + final long additionalEstimatedSize = (long) (s == null ? 0 : s.length()) * Chars.BYTES + + ROUGH_OVERHEAD_PER_DICTIONARY_ENTRY; if (currentEstimatedSize + additionalEstimatedSize > maxDictionarySize) { return -1; } @@ -1504,6 +1495,8 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { + int index = buffer.getInt(initialOffset + keyBufferPosition); + String str = dictionary.get(index); dimValues[dimValIdx] = dictionary.get(buffer.getInt(initialOffset + keyBufferPosition)); } @@ -1548,28 +1541,39 @@ public LongRowBasedKeySerdeHelper(int keyBufferPosition) @Override public int getKeyBufferValueSize() { - return Longs.BYTES; + return Longs.BYTES + Byte.BYTES; } @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - keyBuffer.putLong((Long) key.getKey()[idx]); + Long aLong = (Long) key.getKey()[idx]; + keyBuffer.put(aLong == null ? (byte) 1 : (byte) 0); + keyBuffer.putLong(DimensionHandlerUtils.nullToZero(aLong)); return true; } + protected Long readFromBuffer(ByteBuffer buffer, int initialOffset) + { + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + return NullHandlingHelper.nullToDefault((Long) null); + } + return buffer.getLong(initialOffset + keyBufferPosition + Byte.BYTES); + } + @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = buffer.getLong(initialOffset + keyBufferPosition); + dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); } @Override public int compare(ByteBuffer lhsBuffer, ByteBuffer rhsBuffer, int lhsPosition, int rhsPosition) { - return Longs.compare( - lhsBuffer.getLong(lhsPosition + keyBufferPosition), - rhsBuffer.getLong(rhsPosition + keyBufferPosition) + return Objects.compare( + readFromBuffer(lhsBuffer, lhsPosition), + readFromBuffer(rhsBuffer, rhsPosition), + Comparators.naturalNullsFirst() ); } } @@ -1587,9 +1591,8 @@ public LimitPushDownLongRowBasedKeySerdeHelper(int keyBufferPosition, StringComp @Override public int compare(ByteBuffer lhsBuffer, ByteBuffer rhsBuffer, int lhsPosition, int rhsPosition) { - long lhs = lhsBuffer.getLong(lhsPosition + keyBufferPosition); - long rhs = rhsBuffer.getLong(rhsPosition + keyBufferPosition); - + Long lhs = readFromBuffer(lhsBuffer, lhsPosition); + Long rhs = readFromBuffer(rhsBuffer, rhsPosition); return cmp.compare(String.valueOf(lhs), String.valueOf(rhs)); } } @@ -1606,28 +1609,39 @@ public FloatRowBasedKeySerdeHelper(int keyBufferPosition) @Override public int getKeyBufferValueSize() { - return Floats.BYTES; + return Floats.BYTES + Byte.BYTES; } @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - keyBuffer.putFloat((Float) key.getKey()[idx]); + Float aFloat = (Float) key.getKey()[idx]; + keyBuffer.put(aFloat == null ? (byte) 1 : (byte) 0); + keyBuffer.putFloat(DimensionHandlerUtils.nullToZero(aFloat)); return true; } + protected Float readFromBuffer(ByteBuffer buffer, int initialOffset) + { + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + return NullHandlingHelper.nullToDefault((Float) null); + } + return buffer.getFloat(initialOffset + keyBufferPosition + Byte.BYTES); + } + @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = buffer.getFloat(initialOffset + keyBufferPosition); + dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); } @Override public int compare(ByteBuffer lhsBuffer, ByteBuffer rhsBuffer, int lhsPosition, int rhsPosition) { - return Float.compare( - lhsBuffer.getFloat(lhsPosition + keyBufferPosition), - rhsBuffer.getFloat(rhsPosition + keyBufferPosition) + return Objects.compare( + readFromBuffer(lhsBuffer, lhsPosition), + readFromBuffer(rhsBuffer, rhsPosition), + Comparators.naturalNullsFirst() ); } } @@ -1645,8 +1659,8 @@ public LimitPushDownFloatRowBasedKeySerdeHelper(int keyBufferPosition, StringCom @Override public int compare(ByteBuffer lhsBuffer, ByteBuffer rhsBuffer, int lhsPosition, int rhsPosition) { - float lhs = lhsBuffer.getFloat(lhsPosition + keyBufferPosition); - float rhs = rhsBuffer.getFloat(rhsPosition + keyBufferPosition); + Float lhs = readFromBuffer(lhsBuffer, lhsPosition); + Float rhs = readFromBuffer(rhsBuffer, rhsPosition); return cmp.compare(String.valueOf(lhs), String.valueOf(rhs)); } } @@ -1663,28 +1677,39 @@ public DoubleRowBasedKeySerdeHelper(int keyBufferPosition) @Override public int getKeyBufferValueSize() { - return Doubles.BYTES; + return Doubles.BYTES + Byte.BYTES; } @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - keyBuffer.putDouble((Double) key.getKey()[idx]); + Double aDouble = (Double) key.getKey()[idx]; + keyBuffer.put(aDouble == null ? (byte) 1 : (byte) 0); + keyBuffer.putDouble(DimensionHandlerUtils.nullToZero(aDouble)); return true; } + protected Double readFromBuffer(ByteBuffer buffer, int initialOffset) + { + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + return NullHandlingHelper.nullToDefault((Double) null); + } + return buffer.getDouble(initialOffset + keyBufferPosition + Byte.BYTES); + } + @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = buffer.getDouble(initialOffset + keyBufferPosition); + dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); } @Override public int compare(ByteBuffer lhsBuffer, ByteBuffer rhsBuffer, int lhsPosition, int rhsPosition) { - return Double.compare( - lhsBuffer.getDouble(lhsPosition + keyBufferPosition), - rhsBuffer.getDouble(rhsPosition + keyBufferPosition) + return Objects.compare( + readFromBuffer(lhsBuffer, lhsPosition), + readFromBuffer(rhsBuffer, rhsPosition), + Comparators.naturalNullsFirst() ); } } @@ -1702,8 +1727,8 @@ public LimitPushDownDoubleRowBasedKeySerdeHelper(int keyBufferPosition, StringCo @Override public int compare(ByteBuffer lhsBuffer, ByteBuffer rhsBuffer, int lhsPosition, int rhsPosition) { - double lhs = lhsBuffer.getDouble(lhsPosition + keyBufferPosition); - double rhs = rhsBuffer.getDouble(rhsPosition + keyBufferPosition); + Double lhs = readFromBuffer(lhsBuffer, lhsPosition); + Double rhs = readFromBuffer(rhsBuffer, rhsPosition); return cmp.compare(String.valueOf(lhs), String.valueOf(rhs)); } } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java index 503f80b6bd2e..91818ee5a0f8 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.ArrayBasedIndexedInts; import io.druid.segment.data.IndexedInts; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; @@ -59,7 +60,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, value ); } else { - resultMap.put(selectorPlus.getOutputName(), ""); + resultMap.put(selectorPlus.getOutputName(), NullHandlingHelper.nullToDefault((String) null)); } } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java index 9cabaa5aa1c7..43cedcecc613 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java @@ -23,6 +23,7 @@ import com.google.common.primitives.Ints; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.IndexedInts; import java.nio.ByteBuffer; @@ -48,7 +49,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, ((DimensionSelector) selectorPlus.getSelector()).lookupName(id) ); } else { - resultMap.put(selectorPlus.getOutputName(), ""); + resultMap.put(selectorPlus.getOutputName(), NullHandlingHelper.nullToDefault((String) null)); } } diff --git a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java index 89698285b802..e3354be361aa 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java @@ -57,6 +57,10 @@ public String getAggregationName() @Override public boolean eval(Row row) { + Object metricVal = row.getRaw(aggregationName); + if (metricVal == null || value == null) { + return metricVal == null && value == null; + } return HavingSpecMetricComparator.compare(row, aggregationName, value) == 0; } diff --git a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java index d75333b74fae..3efbd72ab814 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java @@ -57,6 +57,10 @@ public Number getValue() @Override public boolean eval(Row row) { + Object metricVal = row.getRaw(aggregationName); + if (metricVal == null || value == null) { + return false; + } return HavingSpecMetricComparator.compare(row, aggregationName, value) > 0; } diff --git a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java index a60d5adc123e..a913054682ee 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java @@ -55,6 +55,10 @@ public Number getValue() @Override public boolean eval(Row row) { + Object metricVal = row.getRaw(aggregationName); + if (metricVal == null || value == null) { + return false; + } return HavingSpecMetricComparator.compare(row, aggregationName, value) < 0; } diff --git a/processing/src/main/java/io/druid/query/search/SearchHit.java b/processing/src/main/java/io/druid/query/search/SearchHit.java index 52e6692f989a..a9aae2dbe663 100644 --- a/processing/src/main/java/io/druid/query/search/SearchHit.java +++ b/processing/src/main/java/io/druid/query/search/SearchHit.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.segment.NullHandlingHelper; /** */ @@ -39,7 +40,7 @@ public SearchHit( ) { this.dimension = Preconditions.checkNotNull(dimension); - this.value = Preconditions.checkNotNull(value); + this.value = NullHandlingHelper.nullToDefault(value); this.count = count; } diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index dcdcef9321e5..bfe2f1fd1720 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -20,7 +20,6 @@ package io.druid.query.search; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -132,7 +131,7 @@ public void updateSearchResultSet( for (int i = 0; i < vals.size(); ++i) { final String dimVal = selector.lookupName(vals.get(i)); if (searchQuerySpec.accept(dimVal)) { - set.addTo(new SearchHit(outputName, Strings.nullToEmpty(dimVal)), 1); + set.addTo(new SearchHit(outputName, dimVal), 1); if (set.size() >= limit) { return; } diff --git a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java index 2597747e8ec5..84a369d68a8b 100644 --- a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java @@ -20,7 +20,6 @@ package io.druid.query.search; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import io.druid.collections.bitmap.BitmapFactory; @@ -261,7 +260,7 @@ public Object2IntRBTreeMap execute(int limit) extractionFn = IdentityExtractionFn.getInstance(); } for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { - String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); + String dimVal = extractionFn.apply(bitmapIndex.getValue(i)); if (!searchQuerySpec.accept(dimVal)) { continue; } diff --git a/processing/src/main/java/io/druid/query/topn/TopNMapFn.java b/processing/src/main/java/io/druid/query/topn/TopNMapFn.java index 24d05fa37775..3d4807628ec9 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNMapFn.java +++ b/processing/src/main/java/io/druid/query/topn/TopNMapFn.java @@ -56,14 +56,10 @@ public static Function getValueTransformer(ValueType outputType) return longVal == null ? DimensionHandlerUtils.ZERO_LONG : longVal; }; - private static Function FLOAT_TRANSFORMER = input -> { - final Float floatVal = DimensionHandlerUtils.convertObjectToFloat(input); - return floatVal == null ? DimensionHandlerUtils.ZERO_FLOAT : floatVal; - }; - private static Function DOUBLE_TRANSFORMER = input -> { - final Double doubleValue = DimensionHandlerUtils.convertObjectToDouble(input); - return doubleValue == null ? DimensionHandlerUtils.ZERO_DOUBLE : doubleValue; - }; + private static Function FLOAT_TRANSFORMER = input -> DimensionHandlerUtils.convertObjectToFloat(input); + + private static Function DOUBLE_TRANSFORMER = input -> DimensionHandlerUtils.convertObjectToDouble( + input); private static final TopNColumnSelectorStrategyFactory STRATEGY_FACTORY = new TopNColumnSelectorStrategyFactory(); From 920139f0904d4dc6659f77f3073e1ec5df17f926 Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 02:15:55 +0530 Subject: [PATCH 07/50] Enable test matrix in travis --- .travis.yml | 13 +++++++++++++ pom.xml | 2 ++ 2 files changed, 15 insertions(+) diff --git a/.travis.yml b/.travis.yml index 078b4cb96949..f9a3ec4d2dac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,19 @@ matrix: before_script: - unset _JAVA_OPTIONS script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl '!processing' + # processing module test with SQL Compatible Null Handling + - sudo: false + install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B + before_script: + - unset _JAVA_OPTIONS + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.null.handling.useDefaultValueForNull=false -pl processing + + # non-processing modules test with SQL Compatible Null Handling + - sudo: false + install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B + before_script: + - unset _JAVA_OPTIONS + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.null.handling.useDefaultValueForNull=false -pl '!processing' # run integration tests - sudo: required diff --git a/pom.xml b/pom.xml index 65b5d92443d2..2ef439940923 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ 1.10.77 + true @@ -1027,6 +1028,7 @@ -Xmx3000m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 -Duser.timezone=UTC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + -Ddruid.null.handling.useDefaultValueForNull=${druid.null.handling.useDefaultValueForNull} true From 6387e4bec455c7a88b81b2eca1e0a7fb4b230f2e Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 02:18:27 +0530 Subject: [PATCH 08/50] Fixing tests --- .../DistinctCountGroupByQueryTest.java | 137 ++++++- .../input/AvroStreamInputRowParserTest.java | 4 +- .../VarianceAggregatorCollectorTest.java | 7 + .../NullHandlingHelperInjectionTest.java | 65 ++++ .../aggregation/AggregationTestHelper.java | 27 +- .../aggregation/DoubleMaxAggregationTest.java | 10 +- .../aggregation/DoubleMinAggregationTest.java | 10 +- .../aggregation/FilteredAggregatorTest.java | 6 + .../aggregation/LongMaxAggregationTest.java | 10 +- .../aggregation/LongMinAggregationTest.java | 10 +- .../aggregation/MetricManipulatorFnsTest.java | 12 + .../TestDoubleColumnSelectorImpl.java | 6 + .../aggregation/TestFloatColumnSelector.java | 6 + .../aggregation/TestLongColumnSelector.java | 6 + .../CardinalityAggregatorTest.java | 15 +- .../first/DoubleFirstAggregationTest.java | 14 +- .../first/FloatFirstAggregationTest.java | 14 +- .../first/LongFirstAggregationTest.java | 14 +- .../last/DoubleLastAggregationTest.java | 14 +- .../last/FloatLastAggregationTest.java | 14 +- .../last/LongLastAggregationTest.java | 14 +- .../query/cache/CacheKeyBuilderTest.java | 2 +- .../druid/query/expression/ExprMacroTest.java | 20 +- .../extraction/FunctionalExtractionTest.java | 19 +- .../JavaScriptExtractionFnTest.java | 7 +- .../extraction/RegexDimExtractionFnTest.java | 17 +- .../query/groupby/GroupByQueryRunnerTest.java | 142 +++++-- .../epinephelinae/BufferHashGrouperTest.java | 9 +- .../LimitedBufferHashGrouperTest.java | 106 ++++-- .../TestColumnSelectorFactory.java | 21 +- .../LookupExtractionFnExpectationsTest.java | 7 +- .../metadata/SegmentMetadataQueryTest.java | 2 +- .../query/search/SearchQueryRunnerTest.java | 7 +- .../druid/query/topn/TopNQueryRunnerTest.java | 20 +- .../ConstantDimensionSelectorTest.java | 2 +- .../io/druid/segment/IndexMergerTestBase.java | 223 +++++++---- .../IndexMergerV9CompatibilityTest.java | 2 +- .../segment/TestDoubleColumnSelector.java | 6 + .../java/io/druid/segment/TestHelper.java | 16 +- .../segment/data/GenericIndexedTest.java | 36 +- .../segment/data/IncrementalIndexTest.java | 20 +- .../druid/segment/filter/BaseFilterTest.java | 2 +- .../druid/segment/filter/BoundFilterTest.java | 225 ++++++++--- .../filter/ColumnComparisonFilterTest.java | 35 +- .../segment/filter/ExpressionFilterTest.java | 11 +- .../segment/filter/FilterPartitionTest.java | 183 ++++++--- .../io/druid/segment/filter/InFilterTest.java | 135 +++++-- .../segment/filter/JavaScriptFilterTest.java | 15 +- .../druid/segment/filter/LikeFilterTest.java | 16 +- .../druid/segment/filter/RegexFilterTest.java | 24 +- .../segment/filter/SearchQueryFilterTest.java | 53 ++- .../segment/filter/SelectorFilterTest.java | 75 +++- .../IncrementalIndexStorageAdapterTest.java | 11 +- .../incremental/IncrementalIndexTest.java | 23 +- .../virtual/ExpressionObjectSelectorTest.java | 8 +- .../virtual/ExpressionVirtualColumnTest.java | 7 +- .../segment/virtual/VirtualColumnsTest.java | 12 + .../firehose/WikipediaIrcDecoder.java | 6 +- .../CombiningFirehoseFactoryTest.java | 6 +- .../segment/realtime/RealtimeManagerTest.java | 8 +- .../plumber/RealtimePlumberSchoolTest.java | 20 +- .../segment/realtime/plumber/SinkTest.java | 16 +- .../HashBasedNumberedShardSpecTest.java | 8 +- .../druid/sql/avatica/DruidStatementTest.java | 22 ++ .../druid/sql/calcite/CalciteQueryTest.java | 349 ++++++++++++++++-- .../calcite/expression/ExpressionsTest.java | 18 +- .../sql/calcite/http/SqlResourceTest.java | 10 + .../sql/calcite/planner/CalcitesTest.java | 3 +- 68 files changed, 1835 insertions(+), 565 deletions(-) create mode 100644 processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java diff --git a/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java b/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java index 5dfd1c176b1c..5dbf8b929f0b 100644 --- a/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java +++ b/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java @@ -35,23 +35,50 @@ import io.druid.query.groupby.GroupByQueryRunnerTestHelper; import io.druid.query.groupby.orderby.DefaultLimitSpec; import io.druid.query.groupby.orderby.OrderByColumnSpec; +import io.druid.query.groupby.strategy.GroupByStrategySelector; import io.druid.segment.IncrementalIndexSegment; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.Segment; import io.druid.segment.TestHelper; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.List; +@RunWith(Parameterized.class) public class DistinctCountGroupByQueryTest { + @Parameterized.Parameters(name = "{0}") + public static Collection constructorFeeder() throws IOException + { + final List constructors = Lists.newArrayList(); + for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) { + constructors.add(new Object[]{config}); + } + + return constructors; + } + + private GroupByQueryConfig config; + + public DistinctCountGroupByQueryTest( + GroupByQueryConfig config + ) + { + this.config = config; + } @Test public void testGroupByWithDistinctCountAgg() throws Exception { - final GroupByQueryConfig config = new GroupByQueryConfig(); + // groupBy V2 skips adding null entries while doing serde. + boolean nullsInResult = config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V2); config.setMaxIntermediateRows(10000); final GroupByQueryRunnerFactory factory = GroupByQueryRunnerTest.makeQueryRunnerFactory(config); @@ -90,6 +117,36 @@ public void testGroupByWithDistinctCountAgg() throws Exception ImmutableMap.of(visitor_id, "2", client_type, "android") ) ); + // Row with Null values + index.add( + new MapBasedInputRow( + timestamp + 2, + Lists.newArrayList(visitor_id, client_type), + ImmutableMap.of() + ) + ); + index.add( + new MapBasedInputRow( + timestamp + 2, + Lists.newArrayList(visitor_id, client_type), + ImmutableMap.of(visitor_id, "2") + ) + ); + // Row with empty values + index.add( + new MapBasedInputRow( + timestamp + 2, + Lists.newArrayList(visitor_id, client_type), + ImmutableMap.of(visitor_id, "", client_type, "") + ) + ); + index.add( + new MapBasedInputRow( + timestamp + 2, + Lists.newArrayList(visitor_id, client_type), + ImmutableMap.of(visitor_id, "4", client_type, "") + ) + ); GroupByQuery query = new GroupByQuery.Builder() .setDataSource(QueryRunnerTestHelper.dataSource) @@ -128,20 +185,70 @@ public void testGroupByWithDistinctCountAgg() throws Exception query ); - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow( - "1970-01-01T00:00:00.000Z", - client_type, "iphone", - "UV", 2L, - "rows", 2L - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "1970-01-01T00:00:00.000Z", - client_type, "android", - "UV", 1L, - "rows", 1L - ) - ); + List expectedResults; + if (NullHandlingHelper.useDefaultValuesForNull()) { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "iphone", + "UV", 2L, + "rows", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "android", + "UV", 1L, + "rows", 1L + ), + // Both rows with null and empty string gets rolled up so rowCount is 3 + nullsInResult ? + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "UV", 3L, + "rows", 3L + ) : + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "client_type", null, + "UV", 3L, + "rows", 3L + ) + ); + } else { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "iphone", + "UV", 2L, + "rows", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "android", + "UV", 1L, + "rows", 1L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "", + "UV", 2L, + "rows", 2L + ), + // client_type is null is this row + nullsInResult ? + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "UV", 1L, + "rows", 2L + ) : + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "client_type", null, + "UV", 1L, + "rows", 2L + ) + ); + } TestHelper.assertExpectedObjects(expectedResults, results, "distinct-count"); } } diff --git a/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java b/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java index f062a3acd951..968e5d028ace 100644 --- a/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java +++ b/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java @@ -243,8 +243,8 @@ public Integer apply(@Nullable String input) // test metrics assertEquals(SOME_FLOAT_VALUE, inputRow.getFloatMetric("someFloat"), 0); - assertEquals(SOME_LONG_VALUE, inputRow.getLongMetric("someLong")); - assertEquals(SOME_INT_VALUE, inputRow.getLongMetric("someInt")); + assertEquals(SOME_LONG_VALUE, inputRow.getLongMetric("someLong").longValue()); + assertEquals(SOME_INT_VALUE, inputRow.getLongMetric("someInt").longValue()); } public static GenericRecord buildSomeAvroDatum() throws IOException diff --git a/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceAggregatorCollectorTest.java b/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceAggregatorCollectorTest.java index 0c4aaaa36de8..fdc668f833c2 100644 --- a/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceAggregatorCollectorTest.java +++ b/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceAggregatorCollectorTest.java @@ -150,6 +150,13 @@ public float getFloat() { return v; } + + @Override + public boolean isNull() + { + return false; + } + } private static class ObjectHandOver implements ObjectColumnSelector diff --git a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java new file mode 100644 index 000000000000..ad29938cdbaf --- /dev/null +++ b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.guice; + +import com.google.inject.Injector; +import io.druid.segment.NullHandlingHelper; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class NullHandlingHelperInjectionTest +{ + public static String NULL_HANDLING_CONFIG_STRING = ("druid.null.handling.useDefaultValueForNull"); + + @Test + public void testNullHandlingHelperUseDefaultValues() + { + String prev = System.getProperty(NULL_HANDLING_CONFIG_STRING); + try { + System.setProperty(NULL_HANDLING_CONFIG_STRING, "true"); + Injector injector = GuiceInjectors.makeStartupInjector(); + Assert.assertEquals(true, NullHandlingHelper.useDefaultValuesForNull()); + } + finally { + if (prev != null) { + System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); + } + } + } + + @Test + public void testNullHandlingHelperNoDefaultValues() + { + String prev = System.getProperty(NULL_HANDLING_CONFIG_STRING); + try { + System.setProperty(NULL_HANDLING_CONFIG_STRING, "false"); + Injector injector = GuiceInjectors.makeStartupInjector(); + Assert.assertEquals(false, NullHandlingHelper.useDefaultValuesForNull()); + } + finally { + if (prev != null) { + System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); + } + } + } + +} diff --git a/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java b/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java index 0c4d9b620018..1515d9926869 100644 --- a/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java +++ b/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java @@ -525,22 +525,17 @@ public Sequence runQueryOnSegmentsObjs(final List segments, final MoreExecutors.sameThreadExecutor(), Lists.transform( segments, - new Function() - { - @Override - public QueryRunner apply(final Segment segment) - { - try { - return makeStringSerdeQueryRunner( - mapper, - toolChest, - query, - factory.createRunner(segment) - ); - } - catch (Exception ex) { - throw Throwables.propagate(ex); - } + segment -> { + try { + return makeStringSerdeQueryRunner( + mapper, + toolChest, + query, + factory.createRunner(segment) + ); + } + catch (Exception ex) { + throw Throwables.propagate(ex); } } ) diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java index ec602d483e1d..f6737ac4e4bb 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java @@ -57,7 +57,7 @@ public void setup() @Test public void testDoubleMaxAggregator() { - DoubleMaxAggregator agg = (DoubleMaxAggregator) doubleMaxAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleMaxAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -75,9 +75,9 @@ public void testDoubleMaxAggregator() @Test public void testDoubleMaxBufferAggregator() { - DoubleMaxBufferAggregator agg = (DoubleMaxBufferAggregator) doubleMaxAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = doubleMaxAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Doubles.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Doubles.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -109,13 +109,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMaxAggregator agg) + private void aggregate(TestDoubleColumnSelectorImpl selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMaxBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestDoubleColumnSelectorImpl selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java index bd4479736f96..f5c1607bd882 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java @@ -57,7 +57,7 @@ public void setup() @Test public void testDoubleMinAggregator() { - DoubleMinAggregator agg = (DoubleMinAggregator) doubleMinAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleMinAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -75,9 +75,9 @@ public void testDoubleMinAggregator() @Test public void testDoubleMinBufferAggregator() { - DoubleMinBufferAggregator agg = (DoubleMinBufferAggregator) doubleMinAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = doubleMinAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Doubles.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Doubles.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -109,13 +109,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMinAggregator agg) + private void aggregate(TestDoubleColumnSelectorImpl selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMinBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestDoubleColumnSelectorImpl selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index d3ddb3640ca8..d1b16ef78f59 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -212,6 +212,12 @@ public double getDouble() return ((ColumnValueSelector) selector).getDouble(); } + @Override + public boolean isNull() + { + return selector.isNull(); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java index 25cd857e294f..19b6cbc819cc 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java @@ -57,7 +57,7 @@ public void setup() @Test public void testLongMaxAggregator() { - LongMaxAggregator agg = (LongMaxAggregator) longMaxAggFactory.factorize(colSelectorFactory); + Aggregator agg = longMaxAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -75,9 +75,9 @@ public void testLongMaxAggregator() @Test public void testLongMaxBufferAggregator() { - LongMaxBufferAggregator agg = (LongMaxBufferAggregator) longMaxAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = longMaxAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Longs.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Longs.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -109,13 +109,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestLongColumnSelector selector, LongMaxAggregator agg) + private void aggregate(TestLongColumnSelector selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestLongColumnSelector selector, LongMaxBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestLongColumnSelector selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java index 73357564c42b..265a455457ca 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java @@ -57,7 +57,7 @@ public void setup() @Test public void testLongMinAggregator() { - LongMinAggregator agg = (LongMinAggregator) longMinAggFactory.factorize(colSelectorFactory); + Aggregator agg = longMinAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -75,9 +75,9 @@ public void testLongMinAggregator() @Test public void testLongMinBufferAggregator() { - LongMinBufferAggregator agg = (LongMinBufferAggregator) longMinAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = longMinAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Longs.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Longs.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -109,13 +109,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestLongColumnSelector selector, LongMinAggregator agg) + private void aggregate(TestLongColumnSelector selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestLongColumnSelector selector, LongMinBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestLongColumnSelector selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java b/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java index 967086f155f8..8721a5ad7780 100644 --- a/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java @@ -50,6 +50,12 @@ public long getLong() { return longVal; } + + @Override + public boolean isNull() + { + return false; + } } ); LongMinAggregatorFactory longMinAggregatorFactory = new LongMinAggregatorFactory(NAME, FIELD); @@ -89,6 +95,12 @@ public long getLong() { return longVal; } + + @Override + public boolean isNull() + { + return false; + } } ); constructorArrays.add( diff --git a/processing/src/test/java/io/druid/query/aggregation/TestDoubleColumnSelectorImpl.java b/processing/src/test/java/io/druid/query/aggregation/TestDoubleColumnSelectorImpl.java index 8e8cf5097817..f85a70c38679 100644 --- a/processing/src/test/java/io/druid/query/aggregation/TestDoubleColumnSelectorImpl.java +++ b/processing/src/test/java/io/druid/query/aggregation/TestDoubleColumnSelectorImpl.java @@ -39,6 +39,12 @@ public double getDouble() return doubles[index]; } + @Override + public boolean isNull() + { + return false; + } + public void increment() { ++index; diff --git a/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java b/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java index 8c0aa532d8f5..dc1c68caeb03 100644 --- a/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java +++ b/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java @@ -38,6 +38,12 @@ public float getFloat() return floats[index]; } + @Override + public boolean isNull() + { + return false; + } + public void increment() { ++index; diff --git a/processing/src/test/java/io/druid/query/aggregation/TestLongColumnSelector.java b/processing/src/test/java/io/druid/query/aggregation/TestLongColumnSelector.java index 2a78bb500521..aca323a66c8b 100644 --- a/processing/src/test/java/io/druid/query/aggregation/TestLongColumnSelector.java +++ b/processing/src/test/java/io/druid/query/aggregation/TestLongColumnSelector.java @@ -38,6 +38,12 @@ public long getLong() return longs[index]; } + @Override + public boolean isNull() + { + return false; + } + public void increment() { ++index; diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index 362f14285a2a..21274362c75b 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -47,6 +47,7 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.IdLookup; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.IndexedInts; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntIterators; @@ -430,8 +431,8 @@ public void testAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { aggregate(selectorList, agg); } - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.05); - Assert.assertEquals(7L, rowAggregatorFactoryRounded.finalizeComputation(agg.get())); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.05); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get())); } @Test @@ -474,8 +475,8 @@ public void testBufferAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { bufferAggregate(selectorList, agg, buf, pos); } - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); - Assert.assertEquals(7L, rowAggregatorFactoryRounded.finalizeComputation(agg.get(buf, pos))); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get(buf, pos))); } @Test @@ -554,11 +555,11 @@ public void testCombineValues() aggregate(selector2, agg2); } - Assert.assertEquals(4.0, (Double) valueAggregatorFactory.finalizeComputation(agg1.get()), 0.05); - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg2.get()), 0.05); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 4.0 : 3.0, (Double) valueAggregatorFactory.finalizeComputation(agg1.get()), 0.05); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg2.get()), 0.05); Assert.assertEquals( - 7.0, + NullHandlingHelper.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) rowAggregatorFactory.finalizeComputation( rowAggregatorFactory.combine( agg1.get(), diff --git a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java index 7de0b2a75ec1..8ef75b2bf51e 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestDoubleColumnSelectorImpl; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleFirstAggregator() { - DoubleFirstAggregator agg = (DoubleFirstAggregator) doubleFirstAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleFirstAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -92,7 +94,7 @@ public void testDoubleFirstAggregator() @Test public void testDoubleFirstBufferAggregator() { - DoubleFirstBufferAggregator agg = (DoubleFirstBufferAggregator) doubleFirstAggFactory.factorizeBuffered( + BufferAggregator agg = doubleFirstAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleFirstAggFactory.getMaxIntermediateSize()]); @@ -122,7 +124,7 @@ public void testCombine() @Test public void testDoubleFirstCombiningAggregator() { - DoubleFirstAggregator agg = (DoubleFirstAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -144,7 +146,7 @@ public void testDoubleFirstCombiningAggregator() @Test public void testDoubleFirstCombiningBufferAggregator() { - DoubleFirstBufferAggregator agg = (DoubleFirstBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleFirstAggFactory.getMaxIntermediateSize()]); @@ -174,7 +176,7 @@ public void testSerde() throws Exception } private void aggregate( - DoubleFirstAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -184,7 +186,7 @@ private void aggregate( } private void aggregate( - DoubleFirstBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java index d5775797dda7..47bbd172d3fa 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestFloatColumnSelector; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleFirstAggregator() { - FloatFirstAggregator agg = (FloatFirstAggregator) floatFirstAggregatorFactory.factorize(colSelectorFactory); + Aggregator agg = floatFirstAggregatorFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -92,7 +94,7 @@ public void testDoubleFirstAggregator() @Test public void testDoubleFirstBufferAggregator() { - FloatFirstBufferAggregator agg = (FloatFirstBufferAggregator) floatFirstAggregatorFactory.factorizeBuffered( + BufferAggregator agg = floatFirstAggregatorFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatFirstAggregatorFactory.getMaxIntermediateSize()]); @@ -122,7 +124,7 @@ public void testCombine() @Test public void testDoubleFirstCombiningAggregator() { - FloatFirstAggregator agg = (FloatFirstAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -144,7 +146,7 @@ public void testDoubleFirstCombiningAggregator() @Test public void testDoubleFirstCombiningBufferAggregator() { - FloatFirstBufferAggregator agg = (FloatFirstBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatFirstAggregatorFactory.getMaxIntermediateSize()]); @@ -174,7 +176,7 @@ public void testSerde() throws Exception } private void aggregate( - FloatFirstAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -184,7 +186,7 @@ private void aggregate( } private void aggregate( - FloatFirstBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java index f9e3be856ddc..9731e299569a 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; @@ -70,7 +72,7 @@ public void setup() @Test public void testLongFirstAggregator() { - LongFirstAggregator agg = (LongFirstAggregator) longFirstAggFactory.factorize(colSelectorFactory); + Aggregator agg = longFirstAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -91,7 +93,7 @@ public void testLongFirstAggregator() @Test public void testLongFirstBufferAggregator() { - LongFirstBufferAggregator agg = (LongFirstBufferAggregator) longFirstAggFactory.factorizeBuffered( + BufferAggregator agg = longFirstAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longFirstAggFactory.getMaxIntermediateSize()]); @@ -121,7 +123,7 @@ public void testCombine() @Test public void testLongFirstCombiningAggregator() { - LongFirstAggregator agg = (LongFirstAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -143,7 +145,7 @@ public void testLongFirstCombiningAggregator() @Test public void testLongFirstCombiningBufferAggregator() { - LongFirstBufferAggregator agg = (LongFirstBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longFirstAggFactory.getMaxIntermediateSize()]); @@ -173,7 +175,7 @@ public void testSerde() throws Exception } private void aggregate( - LongFirstAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -183,7 +185,7 @@ private void aggregate( } private void aggregate( - LongFirstBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java index 0202449eac40..6ff8547f6fc8 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestDoubleColumnSelectorImpl; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleLastAggregator() { - DoubleLastAggregator agg = (DoubleLastAggregator) doubleLastAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleLastAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -92,7 +94,7 @@ public void testDoubleLastAggregator() @Test public void testDoubleLastBufferAggregator() { - DoubleLastBufferAggregator agg = (DoubleLastBufferAggregator) doubleLastAggFactory.factorizeBuffered( + BufferAggregator agg = doubleLastAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleLastAggFactory.getMaxIntermediateSize()]); @@ -122,7 +124,7 @@ public void testCombine() @Test public void testDoubleLastCombiningAggregator() { - DoubleLastAggregator agg = (DoubleLastAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -144,7 +146,7 @@ public void testDoubleLastCombiningAggregator() @Test public void testDoubleLastCombiningBufferAggregator() { - DoubleLastBufferAggregator agg = (DoubleLastBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleLastAggFactory.getMaxIntermediateSize()]); @@ -174,7 +176,7 @@ public void testSerde() throws Exception } private void aggregate( - DoubleLastAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -184,7 +186,7 @@ private void aggregate( } private void aggregate( - DoubleLastBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java index 480e7b583239..217467cb7a02 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestFloatColumnSelector; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleLastAggregator() { - FloatLastAggregator agg = (FloatLastAggregator) floatLastAggregatorFactory.factorize(colSelectorFactory); + Aggregator agg = floatLastAggregatorFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -92,7 +94,7 @@ public void testDoubleLastAggregator() @Test public void testDoubleLastBufferAggregator() { - FloatLastBufferAggregator agg = (FloatLastBufferAggregator) floatLastAggregatorFactory.factorizeBuffered( + BufferAggregator agg = floatLastAggregatorFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatLastAggregatorFactory.getMaxIntermediateSize()]); @@ -122,7 +124,7 @@ public void testCombine() @Test public void testDoubleLastCombiningAggregator() { - FloatLastAggregator agg = (FloatLastAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -144,7 +146,7 @@ public void testDoubleLastCombiningAggregator() @Test public void testDoubleLastCombiningBufferAggregator() { - FloatLastBufferAggregator agg = (FloatLastBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatLastAggregatorFactory.getMaxIntermediateSize()]); @@ -174,7 +176,7 @@ public void testSerde() throws Exception } private void aggregate( - FloatLastAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -184,7 +186,7 @@ private void aggregate( } private void aggregate( - FloatLastBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java index 8b530df3f0ff..5ca9c26053e6 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; @@ -70,7 +72,7 @@ public void setup() @Test public void testLongLastAggregator() { - LongLastAggregator agg = (LongLastAggregator) longLastAggFactory.factorize(colSelectorFactory); + Aggregator agg = longLastAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -91,7 +93,7 @@ public void testLongLastAggregator() @Test public void testLongLastBufferAggregator() { - LongLastBufferAggregator agg = (LongLastBufferAggregator) longLastAggFactory.factorizeBuffered( + BufferAggregator agg = longLastAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longLastAggFactory.getMaxIntermediateSize()]); @@ -121,7 +123,7 @@ public void testCombine() @Test public void testLongLastCombiningAggregator() { - LongLastAggregator agg = (LongLastAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -143,7 +145,7 @@ public void testLongLastCombiningAggregator() @Test public void testLongLastCombiningBufferAggregator() { - LongLastBufferAggregator agg = (LongLastBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longLastAggFactory.getMaxIntermediateSize()]); @@ -173,7 +175,7 @@ public void testSerde() throws Exception } private void aggregate( - LongLastAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -183,7 +185,7 @@ private void aggregate( } private void aggregate( - LongLastBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/cache/CacheKeyBuilderTest.java b/processing/src/test/java/io/druid/query/cache/CacheKeyBuilderTest.java index 9ad874985a00..e4515852ab0b 100644 --- a/processing/src/test/java/io/druid/query/cache/CacheKeyBuilderTest.java +++ b/processing/src/test/java/io/druid/query/cache/CacheKeyBuilderTest.java @@ -24,8 +24,8 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; -import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.Cacheable; +import io.druid.java.util.common.StringUtils; import org.junit.Test; import java.nio.ByteBuffer; diff --git a/processing/src/test/java/io/druid/query/expression/ExprMacroTest.java b/processing/src/test/java/io/druid/query/expression/ExprMacroTest.java index 10b42157c95e..b5741d629c9d 100644 --- a/processing/src/test/java/io/druid/query/expression/ExprMacroTest.java +++ b/processing/src/test/java/io/druid/query/expression/ExprMacroTest.java @@ -84,8 +84,8 @@ public void testRegexpExtract() public void testTimestampCeil() { assertExpr("timestamp_ceil(t, 'P1M')", DateTimes.of("2000-03-01").getMillis()); - assertExpr("timestamp_ceil(t, 'P1D','','America/Los_Angeles')", DateTimes.of("2000-02-03T08").getMillis()); - assertExpr("timestamp_ceil(t, 'P1D','',CityOfAngels)", DateTimes.of("2000-02-03T08").getMillis()); + assertExpr("timestamp_ceil(t, 'P1D', null,'America/Los_Angeles')", DateTimes.of("2000-02-03T08").getMillis()); + assertExpr("timestamp_ceil(t, 'P1D', null,CityOfAngels)", DateTimes.of("2000-02-03T08").getMillis()); assertExpr("timestamp_ceil(t, 'P1D','1970-01-01T01','Etc/UTC')", DateTimes.of("2000-02-04T01").getMillis()); } @@ -93,8 +93,8 @@ public void testTimestampCeil() public void testTimestampFloor() { assertExpr("timestamp_floor(t, 'P1M')", DateTimes.of("2000-02-01").getMillis()); - assertExpr("timestamp_floor(t, 'P1D','','America/Los_Angeles')", DateTimes.of("2000-02-02T08").getMillis()); - assertExpr("timestamp_floor(t, 'P1D','',CityOfAngels)", DateTimes.of("2000-02-02T08").getMillis()); + assertExpr("timestamp_floor(t, 'P1D', null,'America/Los_Angeles')", DateTimes.of("2000-02-02T08").getMillis()); + assertExpr("timestamp_floor(t, 'P1D', null,CityOfAngels)", DateTimes.of("2000-02-02T08").getMillis()); assertExpr("timestamp_floor(t, 'P1D','1970-01-01T01','Etc/UTC')", DateTimes.of("2000-02-03T01").getMillis()); } @@ -139,36 +139,36 @@ public void testTimestampFormat() @Test public void testTrim() { - assertExpr("trim('')", null); + assertExpr("trim('')", ""); assertExpr("trim(concat(' ',x,' '))", "foo"); assertExpr("trim(spacey)", "hey there"); assertExpr("trim(spacey, '')", " hey there "); assertExpr("trim(spacey, 'he ')", "y ther"); - assertExpr("trim(spacey, spacey)", null); + assertExpr("trim(spacey, spacey)", ""); assertExpr("trim(spacey, substring(spacey, 0, 4))", "y ther"); } @Test public void testLTrim() { - assertExpr("ltrim('')", null); + assertExpr("ltrim('')", ""); assertExpr("ltrim(concat(' ',x,' '))", "foo "); assertExpr("ltrim(spacey)", "hey there "); assertExpr("ltrim(spacey, '')", " hey there "); assertExpr("ltrim(spacey, 'he ')", "y there "); - assertExpr("ltrim(spacey, spacey)", null); + assertExpr("ltrim(spacey, spacey)", ""); assertExpr("ltrim(spacey, substring(spacey, 0, 4))", "y there "); } @Test public void testRTrim() { - assertExpr("rtrim('')", null); + assertExpr("rtrim('')", ""); assertExpr("rtrim(concat(' ',x,' '))", " foo"); assertExpr("rtrim(spacey)", " hey there"); assertExpr("rtrim(spacey, '')", " hey there "); assertExpr("rtrim(spacey, 'he ')", " hey ther"); - assertExpr("rtrim(spacey, spacey)", null); + assertExpr("rtrim(spacey, spacey)", ""); assertExpr("rtrim(spacey, substring(spacey, 0, 4))", " hey ther"); } diff --git a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java index a1308e41d20c..493cb9e2b741 100644 --- a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java +++ b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -163,7 +164,11 @@ public void testReplaceMissing() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? MISSING : out, exFn.apply(in)); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? MISSING : out, exFn.apply(in)); + } else { + Assert.assertEquals(out == null ? MISSING : out, exFn.apply(in)); + } } @@ -178,7 +183,11 @@ public void testReplaceMissingBlank() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + } else { + Assert.assertEquals(out == null ? "" : out, exFn.apply(in)); + } } @Test @@ -192,7 +201,11 @@ public void testOnlyOneValuePresent() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + } else { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? "" : out, exFn.apply(in)); + } } @Test diff --git a/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java index dc4494dadbde..302ff3f84152 100644 --- a/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java @@ -26,6 +26,7 @@ import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.DateTimes; import io.druid.js.JavaScriptConfig; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -131,7 +132,11 @@ public void testJavascriptIsNull() Assert.assertEquals("yes", extractionFn.apply((String) null)); Assert.assertEquals("yes", extractionFn.apply((Object) null)); - Assert.assertEquals("yes", extractionFn.apply("")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals("yes", extractionFn.apply("")); + } else { + Assert.assertEquals("no", extractionFn.apply("")); + } Assert.assertEquals("no", extractionFn.apply("abc")); Assert.assertEquals("no", extractionFn.apply(new Object())); Assert.assertEquals("no", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java index ce6e468ccd93..6652802d5dd6 100644 --- a/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -145,11 +146,11 @@ public void testNullAndEmpty() String regex = "(.*)/.*/.*"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, false, null); // no match, map empty input value to null - Assert.assertEquals(null, extractionFn.apply("")); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); // null value, returns null Assert.assertEquals(null, extractionFn.apply(null)); // empty match, map empty result to null - Assert.assertEquals(null, extractionFn.apply("/a/b")); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("/a/b")); } @Test @@ -168,8 +169,8 @@ public void testMissingValueReplacementWhenPatternMatchesNull() { String regex = "^()$"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, true, "NO MATCH"); - Assert.assertEquals(null, extractionFn.apply("")); - Assert.assertEquals(null, extractionFn.apply(null)); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "NO MATCH", extractionFn.apply(null)); Assert.assertEquals("NO MATCH", extractionFn.apply("abc")); } @@ -178,10 +179,10 @@ public void testMissingValueReplacementToEmpty() { String regex = "(bob)"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, true, ""); - Assert.assertEquals(null, extractionFn.apply(null)); - Assert.assertEquals(null, extractionFn.apply("")); - Assert.assertEquals(null, extractionFn.apply("abc")); - Assert.assertEquals(null, extractionFn.apply("123")); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply(null)); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("abc")); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("123")); Assert.assertEquals("bob", extractionFn.apply("bobby")); } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 35ba74a9a39b..3b208bc1540a 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -124,6 +124,7 @@ import io.druid.query.ordering.StringComparators; import io.druid.query.search.ContainsSearchQuerySpec; import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; @@ -6943,12 +6944,23 @@ public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() .setGranularity(QueryRunnerTestHelper.dayGran) .setDimFilter(new ExtractionDimFilter("quality", "", lookupExtractionFn, null)) .build(); - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) - ); + + List expectedResults; + + if (NullHandlingHelper.useDefaultValuesForNull()) { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) + ); + } else { + // Only empty string should match, nulls will not match + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) + ); + } Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); @@ -6996,11 +7008,19 @@ public void testGroupByWithExtractionDimFilterWhenSearchValueNotInTheMap() @Test public void testGroupByWithExtractionDimFilterKeyisNull() { + Map extractionMap = new HashMap<>(); - extractionMap.put("", "NULLorEMPTY"); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + LookupExtractionFn lookupExtractionFn; + if (NullHandlingHelper.useDefaultValuesForNull()) { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + extractionMap.put("", "REPLACED_VALUE"); + } else { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "REPLACED_VALUE", true, false); + extractionMap.put("", "NOT_USED"); + } GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) @@ -7022,7 +7042,7 @@ public void testGroupByWithExtractionDimFilterKeyisNull() .setDimFilter( new ExtractionDimFilter( "null_column", - "NULLorEMPTY", + "REPLACED_VALUE", lookupExtractionFn, null ) @@ -7151,7 +7171,14 @@ public void testGroupByWithExtractionDimFilterNullDims() extractionMap.put("", "EMPTY"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + LookupExtractionFn lookupExtractionFn; + if (NullHandlingHelper.useDefaultValuesForNull()) { + extractionMap.put("", "EMPTY"); + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + } else { + extractionMap.put("", "SHOULD_NOT_BE_USED"); + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "EMPTY", true, true); + } GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) @@ -8128,21 +8155,39 @@ public void testGroupByNumericStringsAsNumericWithDecoration() .setGranularity(QueryRunnerTestHelper.allGran) .build(); - // "entertainment" rows are excluded by the decorated specs, they become empty rows - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 0L, - "qf", 0.0, - "count", 2L - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 170000L, - "qf", 170000.0, - "count", 2L - ) - ); + List expectedResults; + if (NullHandlingHelper.useDefaultValuesForNull()) { + // "entertainment" rows are excluded by the decorated specs, they become empty rows + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 0L, + "qf", 0.0, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 170000L, + "qf", 170000.0, + "count", 2L + ) + ); + } else { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", null, + "qf", null, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 170000L, + "qf", 170000.0, + "count", 2L + ) + ); + } Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); @@ -8185,21 +8230,38 @@ public void testGroupByDecorationOnNumerics() ) .setGranularity(QueryRunnerTestHelper.allGran) .build(); - - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 0L, - "qf", 0.0, - "count", 2L - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 1700L, - "qf", 17000.0, - "count", 2L - ) - ); + List expectedResults; + if (NullHandlingHelper.useDefaultValuesForNull()) { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 0L, + "qf", 0.0, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 1700L, + "qf", 17000.0, + "count", 2L + ) + ); + } else { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", null, + "qf", null, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 1700L, + "qf", 17000.0, + "count", 2L + ) + ); + } Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java index d3bd541a66b2..b612fafd8dda 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java @@ -33,6 +33,7 @@ import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.segment.CloserRule; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -111,7 +112,7 @@ public void testGrowing() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 10000, 2); - final int expectedMaxSize = 219; + final int expectedMaxSize = NullHandlingHelper.useDefaultValuesForNull() ? 219 : 210; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -139,7 +140,7 @@ public void testGrowing2() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 2_000_000_000, 2); - final int expectedMaxSize = 40988516; + final int expectedMaxSize = NullHandlingHelper.useDefaultValuesForNull() ? 40988516 : 39141224; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -153,7 +154,7 @@ public void testGrowing3() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, Integer.MAX_VALUE, 2); - final int expectedMaxSize = 44938972; + final int expectedMaxSize = NullHandlingHelper.useDefaultValuesForNull() ? 44938972 : 42955456; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -167,7 +168,7 @@ public void testNoGrowing() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 10000, Integer.MAX_VALUE); - final int expectedMaxSize = 267; + final int expectedMaxSize = NullHandlingHelper.useDefaultValuesForNull() ? 267 : 258; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java index 0fb72ba00045..c6cf93f77714 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java @@ -27,6 +27,7 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -53,20 +54,37 @@ public void testLimitAndBufferSwapping() for (int i = 0; i < numRows; i++) { Assert.assertTrue(String.valueOf(i + keyBase), grouper.aggregate(i + keyBase).isOk()); } + if (NullHandlingHelper.useDefaultValuesForNull()) { + // bucket size is hash(int) + key(int) + aggs(2 longs) + heap offset(int) = 28 bytes + // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes + // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes + // table arena is split in halves when doing push down, so each half is 9798 bytes + // each table arena half can hold 9798 / 28 = 349 buckets, with load factor of 0.5 max buckets per half is 174 + // First buffer swap occurs when we hit 174 buckets + // Subsequent buffer swaps occur after every 74 buckets, since we keep 100 buckets due to the limit + // With 1000 keys inserted, this results in one swap at the first 174 buckets, then 11 swaps afterwards. + // After the last swap, we have 100 keys + 12 new keys inserted. + Assert.assertEquals(12, grouper.getGrowthCount()); + Assert.assertEquals(112, grouper.getSize()); + Assert.assertEquals(349, grouper.getBuckets()); + Assert.assertEquals(174, grouper.getMaxSize()); + } else { + // With Nullability enabled + // bucket size is hash(int) + key(int) + aggs(2 longs + 1 bytes for Long Agg nullability) + heap offset(int) = 29 bytes + // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes + // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes + // table arena is split in halves when doing push down, so each half is 9798 bytes + // each table arena half can hold 9798 / 29 = 337 buckets, with load factor of 0.5 max buckets per half is 168 + // First buffer swap occurs when we hit 168 buckets + // Subsequent buffer swaps occur after every 68 buckets, since we keep 100 buckets due to the limit + // With 1000 keys inserted, this results in one swap at the first 169 buckets, then 12 swaps afterwards. + // After the last swap, we have 100 keys + 16 new keys inserted. + Assert.assertEquals(13, grouper.getGrowthCount()); + Assert.assertEquals(116, grouper.getSize()); + Assert.assertEquals(337, grouper.getBuckets()); + Assert.assertEquals(168, grouper.getMaxSize()); + } - // bucket size is hash(int) + key(int) + aggs(2 longs) + heap offset(int) = 28 bytes - // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes - // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes - // table arena is split in halves when doing push down, so each half is 9798 bytes - // each table arena half can hold 9798 / 28 = 349 buckets, with load factor of 0.5 max buckets per half is 174 - // First buffer swap occurs when we hit 174 buckets - // Subsequent buffer swaps occur after every 74 buckets, since we keep 100 buckets due to the limit - // With 1000 keys inserted, this results in one swap at the first 174 buckets, then 11 swaps afterwards. - // After the last swap, we have 100 keys + 12 new keys inserted. - Assert.assertEquals(12, grouper.getGrowthCount()); - Assert.assertEquals(112, grouper.getSize()); - Assert.assertEquals(349, grouper.getBuckets()); - Assert.assertEquals(174, grouper.getMaxSize()); Assert.assertEquals(100, grouper.getLimit()); // Aggregate slightly different row @@ -77,14 +95,27 @@ public void testLimitAndBufferSwapping() Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk()); } - // we added another 1000 unique keys - // previous size is 112, so next swap occurs after 62 rows - // after that, there are 1000 - 62 = 938 rows, 938 / 74 = 12 additional swaps after the first, - // with 50 keys being added after the final swap. - Assert.assertEquals(25, grouper.getGrowthCount()); - Assert.assertEquals(150, grouper.getSize()); - Assert.assertEquals(349, grouper.getBuckets()); - Assert.assertEquals(174, grouper.getMaxSize()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + // we added another 1000 unique keys + // previous size is 112, so next swap occurs after 62 rows + // after that, there are 1000 - 62 = 938 rows, 938 / 74 = 12 additional swaps after the first, + // with 50 keys being added after the final swap. + Assert.assertEquals(25, grouper.getGrowthCount()); + Assert.assertEquals(150, grouper.getSize()); + Assert.assertEquals(349, grouper.getBuckets()); + Assert.assertEquals(174, grouper.getMaxSize()); + } else { + // With Nullable Aggregator + // we added another 1000 unique keys + // previous size is 116, so next swap occurs after 52 rows + // after that, there are 1000 - 52 = 948 rows, 948 / 68 = 13 additional swaps after the first, + // with 64 keys being added after the final swap. + Assert.assertEquals(27, grouper.getGrowthCount()); + Assert.assertEquals(164, grouper.getSize()); + Assert.assertEquals(337, grouper.getBuckets()); + Assert.assertEquals(168, grouper.getMaxSize()); + } + Assert.assertEquals(100, grouper.getLimit()); final List> expected = Lists.newArrayList(); @@ -110,7 +141,7 @@ public void testMinBufferSize() final int limit = 100; final int keyBase = 100000; final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); - final LimitedBufferHashGrouper grouper = makeGrouper(columnSelectorFactory, 11716, 2, limit); + final LimitedBufferHashGrouper grouper = makeGrouper(columnSelectorFactory, 12120, 2, limit); final int numRows = 1000; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); @@ -119,10 +150,17 @@ public void testMinBufferSize() } // With minimum buffer size, after the first swap, every new key added will result in a swap - Assert.assertEquals(899, grouper.getGrowthCount()); - Assert.assertEquals(101, grouper.getSize()); - Assert.assertEquals(202, grouper.getBuckets()); - Assert.assertEquals(101, grouper.getMaxSize()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(224, grouper.getGrowthCount()); + Assert.assertEquals(104, grouper.getSize()); + Assert.assertEquals(209, grouper.getBuckets()); + Assert.assertEquals(104, grouper.getMaxSize()); + } else { + Assert.assertEquals(899, grouper.getGrowthCount()); + Assert.assertEquals(101, grouper.getSize()); + Assert.assertEquals(202, grouper.getBuckets()); + Assert.assertEquals(101, grouper.getMaxSize()); + } Assert.assertEquals(100, grouper.getLimit()); // Aggregate slightly different row @@ -132,11 +170,17 @@ public void testMinBufferSize() for (int i = 0; i < numRows; i++) { Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk()); } - - Assert.assertEquals(1899, grouper.getGrowthCount()); - Assert.assertEquals(101, grouper.getSize()); - Assert.assertEquals(202, grouper.getBuckets()); - Assert.assertEquals(101, grouper.getMaxSize()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(474, grouper.getGrowthCount()); + Assert.assertEquals(104, grouper.getSize()); + Assert.assertEquals(209, grouper.getBuckets()); + Assert.assertEquals(104, grouper.getMaxSize()); + } else { + Assert.assertEquals(1899, grouper.getGrowthCount()); + Assert.assertEquals(101, grouper.getSize()); + Assert.assertEquals(202, grouper.getBuckets()); + Assert.assertEquals(101, grouper.getMaxSize()); + } Assert.assertEquals(100, grouper.getLimit()); final List> expected = Lists.newArrayList(); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java index e6c68eaf867d..1149ad17387a 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java @@ -57,6 +57,12 @@ public float getFloat() { return row.get().getFloatMetric(columnName); } + + @Override + public boolean isNull() + { + return false; + } }; } @@ -70,6 +76,12 @@ public long getLong() { return row.get().getLongMetric(columnName); } + + @Override + public boolean isNull() + { + return row.get().getLongMetric(columnName) == null; + } }; } @@ -78,6 +90,7 @@ public ObjectColumnSelector makeObjectColumnSelector(final String columnName) { return new ObjectColumnSelector() { + @Override public Class classOfObject() { @@ -106,7 +119,13 @@ public DoubleColumnSelector makeDoubleColumnSelector(String columnName) @Override public double getDouble() { - return row.get().getFloatMetric(columnName); + return row.get().getFloatMetric(columnName).doubleValue(); + } + + @Override + public boolean isNull() + { + return false; } }; } diff --git a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java index a363f7606039..e32e268227ff 100644 --- a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java +++ b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import io.druid.query.extraction.MapLookupExtractor; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -65,7 +66,11 @@ public void testNullKeyIsMappable() false, false ); - Assert.assertEquals("bar", lookupExtractionFn.apply(null)); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals("bar", lookupExtractionFn.apply(null)); + } else { + Assert.assertEquals("REPLACE", lookupExtractionFn.apply(null)); + } } @Test diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java index deaebcc734ec..d66a3fca7054 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java @@ -26,9 +26,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.MoreExecutors; -import io.druid.java.util.common.Intervals; import io.druid.data.input.impl.TimestampSpec; import io.druid.jackson.DefaultObjectMapper; +import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Sequences; import io.druid.query.BySegmentResultValue; diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 5a3ed480256c..6a1846f683d8 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -50,6 +50,7 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.ordering.StringComparators; import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.QueryableIndexSegment; import io.druid.segment.TestHelper; import io.druid.segment.TestIndex; @@ -781,7 +782,11 @@ public void testSearchWithNullValueInDimension() throws Exception QueryRunner runner = factory.createRunner(new QueryableIndexSegment("asdf", TestIndex.persistRealtimeAndLoadMMapped(index))); List expectedHits = Lists.newLinkedList(); expectedHits.add(new SearchHit("table", "table", 1)); - expectedHits.add(new SearchHit("table", "", 1)); + if (NullHandlingHelper.useDefaultValuesForNull()) { + expectedHits.add(new SearchHit("table", "", 1)); + } else { + expectedHits.add(new SearchHit("table", null, 1)); + } checkSearchQuery(searchQuery, runner, expectedHits); } diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 998a43349198..6b531343226b 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -91,6 +91,7 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.ordering.StringComparators; import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; @@ -4251,10 +4252,15 @@ public void testTopNWithExtractionFilter() public void testTopNWithExtractionFilterAndFilteredAggregatorCaseNoExistingValue() { Map extractionMap = new HashMap<>(); - extractionMap.put("", "NULL"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + LookupExtractionFn lookupExtractionFn; + if (NullHandlingHelper.useDefaultValuesForNull()) { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + extractionMap.put("", "NULL"); + } else { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "NULL", true, false); + } DimFilter extractionFilter = new ExtractionDimFilter("null_column", "NULL", lookupExtractionFn, null); TopNQueryBuilder topNQueryBuilder = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) @@ -4324,10 +4330,16 @@ private Sequence> runWithPreMergeAndMerge(TopNQuery quer public void testTopNWithExtractionFilterNoExistingValue() { Map extractionMap = new HashMap<>(); - extractionMap.put("", "NULL"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + LookupExtractionFn lookupExtractionFn; + if (NullHandlingHelper.useDefaultValuesForNull()) { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + extractionMap.put("", "NULL"); + } else { + extractionMap.put("", "NOT_USED"); + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "NULL", true, true); + } DimFilter extractionFilter = new ExtractionDimFilter("null_column", "NULL", lookupExtractionFn, null); TopNQueryBuilder topNQueryBuilder = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) diff --git a/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java b/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java index 344c4938c8f4..71c1da965c78 100644 --- a/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java @@ -75,7 +75,7 @@ public void testLookupName() throws Exception public void testLookupId() throws Exception { Assert.assertEquals(0, NULL_SELECTOR.idLookup().lookupId(null)); - Assert.assertEquals(0, NULL_SELECTOR.idLookup().lookupId("")); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 0 : -1, NULL_SELECTOR.idLookup().lookupId("")); Assert.assertEquals(-1, NULL_SELECTOR.idLookup().lookupId("billy")); Assert.assertEquals(-1, NULL_SELECTOR.idLookup().lookupId("bob")); diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index f5196d062a52..2de02d6baa23 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -220,11 +220,11 @@ public void testPersistWithDifferentDims() throws Exception Assert.assertArrayEquals(new int[][]{{0}, {1}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{1}, {0}}, boatList.get(1).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim1", "3")); - checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim2", "")); + checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim2", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "2")); } @@ -876,17 +876,17 @@ public void testNonLexicographicDimOrderMerge() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {1}, {1}}, boatList.get(2).getDims()); Assert.assertArrayEquals(new Object[]{3L}, boatList.get(2).getMetrics()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d3", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d3", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d3", "30000")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d3", "40000")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d3", "50000")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d1", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d1", "100")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d1", "200")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d1", "300")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d2", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d2", null)); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d2", "2000")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d2", "3000")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d2", "4000")); @@ -985,13 +985,13 @@ public void testMergeWithDimensionsList() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}}, boatList.get(3).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList.get(3).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dimB", null)); - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", "")); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimC", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimC", "2")); } @@ -1083,11 +1083,11 @@ public void testDisjointDimMerge() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}}, boatList.get(4).getDims()); Assert.assertArrayEquals(new Object[]{1L}, boatList.get(4).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(4), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(3, 4), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(3, 4), adapter.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimB", "3")); @@ -1105,11 +1105,11 @@ public void testDisjointDimMerge() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}}, boatList2.get(4).getDims()); Assert.assertArrayEquals(new Object[]{1L}, boatList2.get(4).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter2.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter2.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(3), adapter2.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(4), adapter2.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(3, 4), adapter2.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(3, 4), adapter2.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(0), adapter2.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter2.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(2), adapter2.getBitmapIndex("dimB", "3")); @@ -1213,40 +1213,60 @@ public void testJointDimMerge() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); - Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals( + ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(0, 1, 3), adapter.getBitmapIndex("d5", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", null)); + } else { + Assert.assertEquals( + ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{1}, {1}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {2}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {1}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + checkBitmapIndex(Lists.newArrayList(1, 2, 3), adapter.getBitmapIndex("d1", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d5", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", null)); + } - checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d2", "210")); - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d3", "310")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d3", "311")); - checkBitmapIndex(Lists.newArrayList(0, 1, 3), adapter.getBitmapIndex("d5", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d5", "520")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d6", "620")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d7", "710")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d7", "720")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d8", "810")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d8", "820")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d9", "910")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d9", "911")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d9", "920")); @@ -1355,40 +1375,64 @@ public void testNoRollupMergeWithoutDuplicateRow() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals( + ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } else { + Assert.assertEquals( + ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + + checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + + } else { + // NULL and EMPTY Strings are considered different + Assert.assertArrayEquals(new int[][]{{1}, {1}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {2}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {1}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + + } + - checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d2", "210")); - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d3", "310")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d3", "311")); - checkBitmapIndex(Lists.newArrayList(0, 1, 3), adapter.getBitmapIndex("d5", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d5", "520")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d6", "620")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d7", "710")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d7", "720")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", "")); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", null)); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d8", "810")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d8", "820")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d9", "910")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d9", "911")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d9", "920")); @@ -1420,7 +1464,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910" + "d3", "310", "d7", "", "d9", "910" ) ) ); @@ -1429,7 +1473,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910" + "d3", "310", "d7", "", "d9", "910" ) ) ); @@ -1444,7 +1488,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910" + "d3", "310", "d7", "", "d9", "910" ) ) ); @@ -1453,7 +1497,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 4, Arrays.asList("d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d5", "", "d6", "621", "d7", "", "d8", "821", "d9", "921" + "d6", "621", "d7", "", "d8", "821", "d9", "921" ) ) ); @@ -1496,26 +1540,41 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d3", "d6", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); - Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(0).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(1).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(2).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {1}, {1}, {1}}, boatList.get(3).getDims()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals( + ImmutableList.of("d3", "d6", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {1}, {1}, {1}}, boatList.get(3).getDims()); + } else { + Assert.assertEquals( + ImmutableList.of("d3", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}, {0}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}, {0}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {1}, {1}}, boatList.get(3).getDims()); + checkBitmapIndex(Lists.newArrayList(), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3), adapter.getBitmapIndex("d7", "")); + } + - checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d3", "")); + checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d3", null)); checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d3", "310")); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d6", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d6", null)); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d8", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d8", null)); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d9", "910")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d9", "921")); } @@ -1547,7 +1606,7 @@ public void testMergeWithSupersetOrdering() throws Exception new MapBasedInputRow( 1, Arrays.asList("dimB", "dimA"), - ImmutableMap.of("dimB", "1", "dimA", "") + ImmutableMap.of("dimB", "1") ) ); @@ -1555,7 +1614,7 @@ public void testMergeWithSupersetOrdering() throws Exception new MapBasedInputRow( 1, Arrays.asList("dimB", "dimA"), - ImmutableMap.of("dimB", "", "dimA", "1") + ImmutableMap.of("dimA", "1") ) ); @@ -1664,11 +1723,11 @@ public void testMergeWithSupersetOrdering() throws Exception Assert.assertArrayEquals(new int[][]{{3}, {0}}, boatList.get(4).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList.get(4).getMetrics()); - checkBitmapIndex(Lists.newArrayList(2, 3, 4), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(2, 3, 4), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(4), adapter.getBitmapIndex("dimB", "3")); @@ -1704,16 +1763,16 @@ public void testMergeWithSupersetOrdering() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}, {0}}, boatList2.get(11).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList2.get(11).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(6), adapter2.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(7, 11), adapter2.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(3, 8), adapter2.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(4, 9), adapter2.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(5, 10), adapter2.getBitmapIndex("dimB", "3")); - checkBitmapIndex(Lists.newArrayList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", "")); + checkBitmapIndex(Lists.newArrayList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", null)); checkBitmapIndex(Lists.newArrayList(0), adapter2.getBitmapIndex("dimC", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter2.getBitmapIndex("dimC", "2")); checkBitmapIndex(Lists.newArrayList(2), adapter2.getBitmapIndex("dimC", "3")); @@ -2130,13 +2189,13 @@ public void testPersistNullColumnSkipping() throws Exception }); index1.add(new MapBasedInputRow( 1L, - Lists.newArrayList("d1", "d2"), + Lists.newArrayList("d1", "d2", "d3"), ImmutableMap.of("d1", "a", "d2", "", "A", 1) )); index1.add(new MapBasedInputRow( 1L, - Lists.newArrayList("d1", "d2"), + Lists.newArrayList("d1", "d2", "d3"), ImmutableMap.of("d1", "b", "d2", "", "A", 1) )); @@ -2150,14 +2209,30 @@ public void testPersistNullColumnSkipping() throws Exception ) ) ); - List expectedColumnNames = Arrays.asList("A", "d1"); + List expectedColumnNames = NullHandlingHelper.useDefaultValuesForNull() + ? Arrays.asList("A", "d1") + : Arrays.asList("A", "d1", "d2"); List actualColumnNames = Lists.newArrayList(index.getColumnNames()); Collections.sort(expectedColumnNames); Collections.sort(actualColumnNames); Assert.assertEquals(expectedColumnNames, actualColumnNames); SmooshedFileMapper sfm = closer.closeLater(SmooshedFileMapper.load(tempDir)); - List expectedFilenames = Arrays.asList("A", "__time", "d1", "index.drd", "metadata.drd"); + List expectedFilenames = NullHandlingHelper.useDefaultValuesForNull() ? Arrays.asList( + "A", + "__time", + "d1", + "index.drd", + "metadata.drd" + ) + : Arrays.asList( + "A", + "__time", + "d1", + "d2", + "index.drd", + "metadata.drd" + ); List actualFilenames = new ArrayList<>(sfm.getInternalFilenames()); Collections.sort(expectedFilenames); Collections.sort(actualFilenames); @@ -2341,7 +2416,7 @@ public void testMultiValueHandling() throws Exception Assert.assertArrayEquals(new int[][]{{0, 1, 2}, {0, 1, 2}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{0, 0, 1, 2}, {0, 1, 2, 2}}, boatList.get(1).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); @@ -2364,7 +2439,7 @@ public void testMultiValueHandling() throws Exception Assert.assertEquals(1, boatList.size()); Assert.assertArrayEquals(new int[][]{{0, 1, 2}, {0, 1, 2}}, boatList.get(0).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "a")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "b")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "x")); @@ -2388,7 +2463,7 @@ public void testMultiValueHandling() throws Exception Assert.assertArrayEquals(new int[][]{{0, 1, 2}, {2, 0, 1}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{2, 0, 0, 1}, {0, 2, 1, 2}}, boatList.get(1).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); diff --git a/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java b/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java index f116a2c022a8..1d47a354eda4 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java @@ -23,9 +23,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSource; import com.google.common.io.Files; -import io.druid.java.util.common.JodaUtils; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; +import io.druid.java.util.common.JodaUtils; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.segment.data.CompressedObjectStrategy; diff --git a/processing/src/test/java/io/druid/segment/TestDoubleColumnSelector.java b/processing/src/test/java/io/druid/segment/TestDoubleColumnSelector.java index 682d82d21e92..d44d0cc6cf43 100644 --- a/processing/src/test/java/io/druid/segment/TestDoubleColumnSelector.java +++ b/processing/src/test/java/io/druid/segment/TestDoubleColumnSelector.java @@ -29,4 +29,10 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { // Don't care about runtime shape in tests } + + @Override + public boolean isNull() + { + return false; + } } diff --git a/processing/src/test/java/io/druid/segment/TestHelper.java b/processing/src/test/java/io/druid/segment/TestHelper.java index a0f5e62d5d21..6a3db8e24bb7 100644 --- a/processing/src/test/java/io/druid/segment/TestHelper.java +++ b/processing/src/test/java/io/druid/segment/TestHelper.java @@ -319,12 +319,16 @@ private static void assertRow(String msg, Row expected, Row actual) final Object actualValue = actualMap.get(key); if (expectedValue instanceof Float || expectedValue instanceof Double) { - Assert.assertEquals( - StringUtils.format("%s: key[%s]", msg, key), - ((Number) expectedValue).doubleValue(), - ((Number) actualValue).doubleValue(), - ((Number) expectedValue).doubleValue() * 1e-6 - ); + if (expectedValue == null) { + Assert.assertNull(actualValue); + } else { + Assert.assertEquals( + StringUtils.format("%s: key[%s]", msg, key), + ((Number) expectedValue).doubleValue(), + ((Number) actualValue).doubleValue(), + ((Number) expectedValue).doubleValue() * 1e-6 + ); + } } else { Assert.assertEquals( StringUtils.format("%s: key[%s]", msg, key), diff --git a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java index f4bfd70bfffb..300b33ffc6cb 100644 --- a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java @@ -20,6 +20,7 @@ package io.druid.segment.data; import com.google.common.collect.Maps; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -99,7 +100,7 @@ private void checkBasicAPIs(String[] strings, Indexed index, boolean all { Assert.assertEquals(strings.length, index.size()); for (int i = 0; i < strings.length; i++) { - Assert.assertEquals(strings[i], index.get(i)); + Assert.assertEquals("Index " + i, strings[i], index.get(i)); } if (allowReverseLookup) { @@ -136,4 +137,37 @@ private GenericIndexed serializeAndDeserialize(GenericIndexed in Assert.assertEquals(0, byteBuffer.remaining()); return deserialized; } + + @Test + public void testNullSerializationWithoutCache() throws Exception + { + final String[] strings = {null, "", "c", "d", "e", "f", "g", "h", "i", "k", "j", "l"}; + + Indexed deserialized = serializeAndDeserialize( + GenericIndexed.fromArray( + strings, GenericIndexed.STRING_STRATEGY + ) + ); + Assert.assertNull(deserialized.get(0)); + Assert.assertEquals(deserialized.get(1), NullHandlingHelper.defaultToNull("")); + } + + @Test + public void testNullSerialization() throws Exception + { + final String[] strings = {null, "", "c", "d", "e", "f", "g", "h", "i", "k", "j", "l"}; + + GenericIndexed unCached = serializeAndDeserialize( + GenericIndexed.fromArray( + strings, GenericIndexed.STRING_STRATEGY + )); + + Indexed deserialized = new CachingIndexed( + unCached, CachingIndexed.INITIAL_CACHE_CAPACITY); + + Assert.assertNull(deserialized.get(0)); + Assert.assertEquals(deserialized.get(1), NullHandlingHelper.defaultToNull("")); + } + + } diff --git a/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java b/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java index bbcc3697ca3e..25eb0217e2df 100644 --- a/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java +++ b/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java @@ -388,22 +388,22 @@ public void testFilteredAggregators() throws Exception Assert.assertEquals(Arrays.asList("1"), row.getDimension("dim1")); Assert.assertEquals(Arrays.asList("2"), row.getDimension("dim2")); Assert.assertEquals(Arrays.asList("a", "b"), row.getDimension("dim3")); - Assert.assertEquals(1L, row.getLongMetric("count")); - Assert.assertEquals(1L, row.getLongMetric("count_selector_filtered")); - Assert.assertEquals(1L, row.getLongMetric("count_bound_filtered")); - Assert.assertEquals(1L, row.getLongMetric("count_multivaldim_filtered")); - Assert.assertEquals(0L, row.getLongMetric("count_numeric_filtered")); + Assert.assertEquals(1L, row.getLongMetric("count").longValue()); + Assert.assertEquals(1L, row.getLongMetric("count_selector_filtered").longValue()); + Assert.assertEquals(1L, row.getLongMetric("count_bound_filtered").longValue()); + Assert.assertEquals(1L, row.getLongMetric("count_multivaldim_filtered").longValue()); + Assert.assertEquals(0L, row.getLongMetric("count_numeric_filtered").longValue()); row = rows.next(); Assert.assertEquals(timestamp, row.getTimestampFromEpoch()); Assert.assertEquals(Arrays.asList("3"), row.getDimension("dim1")); Assert.assertEquals(Arrays.asList("4"), row.getDimension("dim2")); Assert.assertEquals(Arrays.asList("c", "d"), row.getDimension("dim3")); - Assert.assertEquals(1L, row.getLongMetric("count")); - Assert.assertEquals(0L, row.getLongMetric("count_selector_filtered")); - Assert.assertEquals(0L, row.getLongMetric("count_bound_filtered")); - Assert.assertEquals(0L, row.getLongMetric("count_multivaldim_filtered")); - Assert.assertEquals(1L, row.getLongMetric("count_numeric_filtered")); + Assert.assertEquals(1L, row.getLongMetric("count").longValue()); + Assert.assertEquals(0L, row.getLongMetric("count_selector_filtered").longValue()); + Assert.assertEquals(0L, row.getLongMetric("count_bound_filtered").longValue()); + Assert.assertEquals(0L, row.getLongMetric("count_multivaldim_filtered").longValue()); + Assert.assertEquals(1L, row.getLongMetric("count_numeric_filtered").longValue()); } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java index 58bc2bcec7cd..1452ba25d28c 100644 --- a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java @@ -27,8 +27,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import io.druid.common.guava.SettableSupplier; -import io.druid.java.util.common.Intervals; import io.druid.data.input.InputRow; +import io.druid.java.util.common.Intervals; import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.Granularities; diff --git a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java index f94e8769ba45..7b6eac561f14 100644 --- a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java @@ -36,6 +36,7 @@ import io.druid.query.filter.BoundDimFilter; import io.druid.query.ordering.StringComparators; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -88,6 +89,21 @@ public static void tearDown() throws Exception @Test public void testLexicographicMatchEverything() + { + final List filters = ImmutableList.of( + new BoundDimFilter("dim0", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter("dim1", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter("dim2", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter("dim3", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC) + ); + + for (BoundDimFilter filter : filters) { + assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + } + } + + @Test + public void testLexicographicMatchWithEmptyString() { final List filters = ImmutableList.of( new BoundDimFilter("dim0", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), @@ -95,9 +111,15 @@ public void testLexicographicMatchEverything() new BoundDimFilter("dim2", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), new BoundDimFilter("dim3", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC) ); - - for (BoundDimFilter filter : filters) { - assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + for (BoundDimFilter filter : filters) { + assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + } + } else { + assertFilterMatches(filters.get(0), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + assertFilterMatches(filters.get(1), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + assertFilterMatches(filters.get(2), ImmutableList.of("0", "2", "3", "4", "6", "7")); + assertFilterMatches(filters.get(3), ImmutableList.of()); } } @@ -112,19 +134,49 @@ public void testLexicographicMatchNull() new BoundDimFilter("dim1", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0") ); - assertFilterMatches( - new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("1", "2", "5") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("1", "2", "5") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("2") + ); + } } @Test public void testLexicographicMatchMissingColumn() { - assertFilterMatches( - new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", null, false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new BoundDimFilter("dim3", null, "", false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", null, false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + assertFilterMatches( + new BoundDimFilter("dim3", null, "", false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } assertFilterMatches( new BoundDimFilter("dim3", "", "", true, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of() @@ -133,18 +185,11 @@ public void testLexicographicMatchMissingColumn() new BoundDimFilter("dim3", "", "", false, true, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of() ); - assertFilterMatches( - new BoundDimFilter("dim3", "", null, false, true, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + assertFilterMatches( new BoundDimFilter("dim3", null, "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - assertFilterMatches( - new BoundDimFilter("dim3", null, "", false, true, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of() - ); } @@ -229,14 +274,25 @@ public void testAlphaNumericMatchNull() new BoundDimFilter("dim1", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), ImmutableList.of("0") ); - assertFilterMatches( - new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( - new BoundDimFilter("dim3", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of("1", "2", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of("2") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of() + ); + } } @Test @@ -327,14 +383,26 @@ public void testNumericMatchNull() new BoundDimFilter("dim1", "", "", false, false, false, null, StringComparators.NUMERIC), ImmutableList.of("0") ); - assertFilterMatches( - new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( - new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.NUMERIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("1", "2", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("2") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of() + ); + } + } @Test @@ -432,10 +500,17 @@ public void testMatchWithExtractionFn() String nullJsFn = "function(str) { return null; }"; ExtractionFn makeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - assertFilterMatches( - new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + } assertFilterMatches( new BoundDimFilter("dim1", "super-ab", "super-abd", true, true, false, superFn, StringComparators.LEXICOGRAPHIC), @@ -452,10 +527,69 @@ public void testMatchWithExtractionFn() ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - assertFilterMatches( - new BoundDimFilter("dim2", "super-null", "super-null", false, false, false, superFn, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("1", "2", "5") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.LEXICOGRAPHIC + ), + ImmutableList.of("1", "2", "5") + ); + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.NUMERIC + ), + ImmutableList.of("1", "2", "5") + ); + } else { + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.LEXICOGRAPHIC + ), + ImmutableList.of("1", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim2", "super-", "super-", false, false, false, superFn, StringComparators.NUMERIC), + ImmutableList.of("2") + ); + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.LEXICOGRAPHIC + ), + ImmutableList.of("1", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim2", "super-", "super-", false, false, false, superFn, StringComparators.NUMERIC), + ImmutableList.of("2") + ); + } assertFilterMatches( new BoundDimFilter("dim3", "super-null", "super-null", false, false, false, superFn, StringComparators.LEXICOGRAPHIC), @@ -467,11 +601,6 @@ public void testMatchWithExtractionFn() ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - assertFilterMatches( - new BoundDimFilter("dim2", "super-null", "super-null", false, false, false, superFn, StringComparators.NUMERIC), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( new BoundDimFilter("dim4", "super-null", "super-null", false, false, false, superFn, StringComparators.NUMERIC), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") diff --git a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java index 8aa0e5882558..13abd491f2a6 100644 --- a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java @@ -38,6 +38,7 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -66,7 +67,7 @@ public class ColumnComparisonFilterTest extends BaseFilterTest private static final List ROWS = ImmutableList.of( PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("1", "2"))), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())), + PARSER.parse(ImmutableMap.of("dim0", "1", "dim2", ImmutableList.of())), PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))), PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("3"))), PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "1", "dim2", ImmutableList.of("4", "5"))), @@ -108,7 +109,7 @@ public void testColumnsWithoutNulls() assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( DefaultDimensionSpec.of("dim1"), DefaultDimensionSpec.of("dim2") - )), ImmutableList.of("5", "9")); + )), ImmutableList.of("1", "5", "9")); assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( DefaultDimensionSpec.of("dim0"), DefaultDimensionSpec.of("dim1"), @@ -123,14 +124,28 @@ public void testMissingColumnNotSpecifiedInDimensionList() DefaultDimensionSpec.of("dim6"), DefaultDimensionSpec.of("dim7") )), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( - DefaultDimensionSpec.of("dim1"), - DefaultDimensionSpec.of("dim6") - )), ImmutableList.of("0")); - assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( - DefaultDimensionSpec.of("dim2"), - DefaultDimensionSpec.of("dim6") - )), ImmutableList.of("1", "2", "6", "7", "8")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim1"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("0", "1")); + + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim2"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("1", "2", "6", "7", "8")); + } else { + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim1"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("1")); + + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim2"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("1", "6", "7", "8")); + } + } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java index 8e3837e5c7eb..10e8892732ca 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java @@ -36,6 +36,7 @@ import io.druid.query.expression.TestExprMacroTable; import io.druid.query.filter.ExpressionDimFilter; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import io.druid.segment.incremental.IncrementalIndexSchema; import org.junit.AfterClass; @@ -130,8 +131,12 @@ public void testOneMultiValuedStringColumn() { // Expressions currently treat multi-valued arrays as nulls. // This test is just documenting the current behavior, not necessarily saying it makes sense. - - assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("0", "1", "2", "4", "5", "6", "7", "8")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("0", "1", "2", "4", "5", "6", "7", "8")); + } else { + assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("2")); + assertFilterMatches(EDF("dim4 == null"), ImmutableList.of("0", "1", "4", "5", "6", "7", "8")); + } assertFilterMatches(EDF("dim4 == '1'"), ImmutableList.of()); assertFilterMatches(EDF("dim4 == '3'"), ImmutableList.of("3")); } @@ -188,7 +193,7 @@ public void testCompareColumns() @Test public void testMissingColumn() { - assertFilterMatches(EDF("missing == ''"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + assertFilterMatches(EDF("missing == null"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); assertFilterMatches(EDF("missing == '1'"), ImmutableList.of()); assertFilterMatches(EDF("missing == 2"), ImmutableList.of()); assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); diff --git a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java index 309a9b908458..221e8fa7a8b4 100644 --- a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java @@ -46,6 +46,7 @@ import io.druid.query.filter.OrDimFilter; import io.druid.query.filter.SelectorDimFilter; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Assert; @@ -199,7 +200,11 @@ public static void tearDown() throws Exception @Test public void testSinglePreFilterWithNulls() { - assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + } else { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of()); + } assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); assertFilterMatches(new SelectorDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new SelectorDimFilter("dim1", "2", null), ImmutableList.of("2")); @@ -212,7 +217,11 @@ public void testSinglePreFilterWithNulls() @Test public void testSinglePostFilterWithNulls() { - assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + } else { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of()); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "", null), ImmutableList.of("0")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "2", null), ImmutableList.of("2")); @@ -221,8 +230,11 @@ public void testSinglePostFilterWithNulls() assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "abc", null), ImmutableList.of("5", "8")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "ab", null), ImmutableList.of()); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); + } else { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-", JS_EXTRACTION_FN), ImmutableList.of("0")); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-10", JS_EXTRACTION_FN), ImmutableList.of("1")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), ImmutableList.of("2")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-1", JS_EXTRACTION_FN), ImmutableList.of("3", "9")); @@ -234,13 +246,23 @@ public void testSinglePostFilterWithNulls() @Test public void testBasicPreAndPostFilterWithNulls() { - assertFilterMatches( - new AndDimFilter(Arrays.asList( - new SelectorDimFilter("dim2", "a", null), - new NoBitmapSelectorDimFilter("dim1", null, null) - )), - ImmutableList.of("0") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "a", null), + new NoBitmapSelectorDimFilter("dim1", null, null) + )), + ImmutableList.of("0") + ); + } else { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "a", null), + new NoBitmapSelectorDimFilter("dim1", null, null) + )), + ImmutableList.of() + ); + } assertFilterMatches( new AndDimFilter(Arrays.asList( @@ -274,14 +296,51 @@ public void testBasicPreAndPostFilterWithNulls() ImmutableList.of() ); - - assertFilterMatches( - new AndDimFilter(Arrays.asList( - new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), - new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN) - )), - ImmutableList.of("0") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("0") + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("2") + ); + } else { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim1", "super-", JS_EXTRACTION_FN) + )), + ImmutableList.of("0") + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of() + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-", JS_EXTRACTION_FN) + )), + ImmutableList.of("2") + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of() + ); + } assertFilterMatches( new AndDimFilter(Arrays.asList( @@ -291,14 +350,6 @@ public void testBasicPreAndPostFilterWithNulls() ImmutableList.of("1") ); - assertFilterMatches( - new AndDimFilter(Arrays.asList( - new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), - new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) - )), - ImmutableList.of("2") - ); - assertFilterMatches( new AndDimFilter(Arrays.asList( new SelectorDimFilter("dim1", "super-1", JS_EXTRACTION_FN), @@ -327,13 +378,23 @@ public void testOrPostFilterWithNulls() ImmutableList.of("0", "3") ); - assertFilterMatches( - new OrDimFilter(Arrays.asList( - new SelectorDimFilter("dim1", "abc", null), - new NoBitmapSelectorDimFilter("dim2", null, null) - )), - ImmutableList.of("1", "2", "5", "8") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "abc", null), + new NoBitmapSelectorDimFilter("dim2", null, null) + )), + ImmutableList.of("1", "2", "5", "8") + ); + } else { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "abc", null), + new NoBitmapSelectorDimFilter("dim2", null, null) + )), + ImmutableList.of("1", "5", "8") + ); + } assertFilterMatches( new OrDimFilter(Arrays.asList( @@ -382,13 +443,32 @@ public void testOrPostFilterWithNulls() )), ImmutableList.of("0", "3") ); - assertFilterMatches( - new OrDimFilter(Arrays.asList( - new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), - new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) - )), - ImmutableList.of("1", "2", "5", "8") - ); + + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("1", "2", "5", "8") + ); + } else { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("1", "5", "8") + ); + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-", JS_EXTRACTION_FN) + )), + ImmutableList.of("2", "5", "8") + ); + } + assertFilterMatches( new OrDimFilter(Arrays.asList( new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), @@ -432,7 +512,14 @@ public void testOrPostFilterWithNulls() public void testMissingColumnSpecifiedInDimensionList() { assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new NoBitmapSelectorDimFilter("dim3", "", null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") + ); + } else { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "", null), ImmutableList.of()); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "a", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "b", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "c", null), ImmutableList.of()); @@ -482,7 +569,17 @@ public void testMissingColumnSpecifiedInDimensionList() public void testMissingColumnNotSpecifiedInDimensionList() { assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new NoBitmapSelectorDimFilter("dim4", "", null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") + ); + } else { + assertFilterMatches( + new NoBitmapSelectorDimFilter("dim4", "", null), + ImmutableList.of() + ); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "a", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "b", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "c", null), ImmutableList.of()); diff --git a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java index 74d5c58adfb7..12122307d341 100644 --- a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java @@ -40,6 +40,7 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -125,10 +126,17 @@ public void testSingleValueStringColumnWithNulls() ImmutableList.of("a") ); - assertFilterMatches( - toInFilter("dim1", null, "10", "abc"), - ImmutableList.of("a", "b", "f") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + toInFilter("dim1", null, "10", "abc"), + ImmutableList.of("a", "b", "f") + ); + } else { + assertFilterMatches( + toInFilter("dim1", null, "10", "abc"), + ImmutableList.of("b", "f") + ); + } assertFilterMatches( toInFilter("dim1", "-1", "ab", "de"), @@ -139,28 +147,47 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches( - toInFilter("dim2", null), - ImmutableList.of("b", "c", "f") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + toInFilter("dim2", null), + ImmutableList.of("b", "c", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "a"), + ImmutableList.of("a", "b", "c", "d", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "b"), + ImmutableList.of("a", "b", "c", "f") + ); + assertFilterMatches( + toInFilter("dim2", ""), + ImmutableList.of("b", "c", "f") + ); + } else { + assertFilterMatches( + toInFilter("dim2", null), + ImmutableList.of("b", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "a"), + ImmutableList.of("a", "b", "d", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "b"), + ImmutableList.of("a", "b", "f") + ); + assertFilterMatches( + toInFilter("dim2", ""), + ImmutableList.of("c") + ); + } assertFilterMatches( toInFilter("dim2", "", (String) null), ImmutableList.of("b", "c", "f") ); - assertFilterMatches( - toInFilter("dim2", null, "a"), - ImmutableList.of("a", "b", "c", "d", "f") - - ); - - assertFilterMatches( - toInFilter("dim2", null, "b"), - ImmutableList.of("a", "b", "c", "f") - - ); - assertFilterMatches( toInFilter("dim2", "c"), ImmutableList.of("e") @@ -180,10 +207,17 @@ public void testMissingColumn() ImmutableList.of("a", "b", "c", "d", "e", "f") ); - assertFilterMatches( - toInFilter("dim3", ""), - ImmutableList.of("a", "b", "c", "d", "e", "f") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + toInFilter("dim3", ""), + ImmutableList.of("a", "b", "c", "d", "e", "f") + ); + } else { + assertFilterMatches( + toInFilter("dim3", ""), + ImmutableList.of() + ); + } assertFilterMatches( toInFilter("dim3", null, "a"), @@ -215,20 +249,43 @@ public void testMatchWithExtractionFn() String nullJsFn = "function(str) { if (str === null) { return 'YES'; } else { return 'NO';} }"; ExtractionFn yesNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - assertFilterMatches( - toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), - ImmutableList.of("a", "b", "c", "d", "f") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), + ImmutableList.of("a", "b", "c", "d", "f") + ); + assertFilterMatches( + toInFilterWithFn("dim1", superFn, "super-null", "super-10", "super-def"), + ImmutableList.of("a", "b", "e") + ); + assertFilterMatches( + toInFilterWithFn("dim2", yesNullFn, "YES"), + ImmutableList.of("b", "c", "f") + ); + assertFilterMatches( + toInFilterWithFn("dim1", yesNullFn, "NO"), + ImmutableList.of("b", "c", "d", "e", "f") + ); + } else { + assertFilterMatches( + toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), + ImmutableList.of("a", "b", "d", "f") + ); + assertFilterMatches( + toInFilterWithFn("dim1", superFn, "super-null", "super-10", "super-def"), + ImmutableList.of("b", "e") + ); + assertFilterMatches( + toInFilterWithFn("dim2", yesNullFn, "YES"), + ImmutableList.of("b", "f") + ); + + assertFilterMatches( + toInFilterWithFn("dim1", yesNullFn, "NO"), + ImmutableList.of("a", "b", "c", "d", "e", "f") + ); + } - assertFilterMatches( - toInFilterWithFn("dim2", yesNullFn, "YES"), - ImmutableList.of("b", "c", "f") - ); - - assertFilterMatches( - toInFilterWithFn("dim1", superFn, "super-null", "super-10", "super-def"), - ImmutableList.of("a", "b", "e") - ); assertFilterMatches( toInFilterWithFn("dim3", yesNullFn, "NO"), @@ -240,10 +297,6 @@ public void testMatchWithExtractionFn() ImmutableList.of("a", "b", "c", "d", "e", "f") ); - assertFilterMatches( - toInFilterWithFn("dim1", yesNullFn, "NO"), - ImmutableList.of("b", "c", "d", "e", "f") - ); } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java index fd140811a429..8182429dd1d5 100644 --- a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java @@ -37,6 +37,7 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -109,7 +110,12 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of("0")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of("0")); + } else { + assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of()); + assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter(""), null), ImmutableList.of("0")); + } assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("10"), null), ImmutableList.of("1")); assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("2"), null), ImmutableList.of("2")); assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("1"), null), ImmutableList.of("3")); @@ -122,7 +128,12 @@ public void testSingleValueStringColumnWithNulls() public void testMultiValueStringColumn() { // multi-val null...... - assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "2", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "2", "5")); + } else { + assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "5")); + assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter(""), null), ImmutableList.of("2")); + } assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("a"), null), ImmutableList.of("0", "3")); assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("b"), null), ImmutableList.of("0")); assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("c"), null), ImmutableList.of("4")); diff --git a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java index 728bdc9e7387..9b9fb6a563f4 100644 --- a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java @@ -33,6 +33,7 @@ import io.druid.query.extraction.SubstringDimExtractionFn; import io.druid.query.filter.LikeDimFilter; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -156,10 +157,17 @@ public void testMatchEmptyString() @Test public void testMatchEmptyStringWithExtractionFn() { - assertFilterMatches( - new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), + ImmutableList.of() + ); + } } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java index cb62975b846f..7dbff91f50fa 100644 --- a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java @@ -35,6 +35,7 @@ import io.druid.query.extraction.JavaScriptExtractionFn; import io.druid.query.filter.RegexDimFilter; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -99,7 +100,11 @@ public void testSingleValueStringColumnWithoutNulls() public void testSingleValueStringColumnWithNulls() { // RegexFilter always returns false for null row values. - assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("1", "2", "3", "4", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("1", "2", "3", "4", "5")); + } else { + assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + } assertFilterMatches(new RegexDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new RegexDimFilter("dim1", "2", null), ImmutableList.of("2")); assertFilterMatches(new RegexDimFilter("dim1", "1", null), ImmutableList.of("1", "3")); @@ -111,7 +116,11 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "3", "4")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "3", "4")); + } else { + assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "2", "3", "4")); + } assertFilterMatches(new RegexDimFilter("dim2", "a", null), ImmutableList.of("0", "3")); assertFilterMatches(new RegexDimFilter("dim2", "b", null), ImmutableList.of("0")); assertFilterMatches(new RegexDimFilter("dim2", "c", null), ImmutableList.of("4")); @@ -141,11 +150,16 @@ public void testRegexWithExtractionFn() { String nullJsFn = "function(str) { if (str === null) { return 'NOT_NULL_ANYMORE'; } else { return str;} }"; ExtractionFn changeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - - assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of("0")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of("0")); + assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "2", "5")); + } else { + assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of()); + assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "5")); + } assertFilterMatches(new RegexDimFilter("dim1", "ab.*", changeNullFn), ImmutableList.of("4", "5")); - assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "2", "5")); + assertFilterMatches(new RegexDimFilter("dim2", "a.*", changeNullFn), ImmutableList.of("0", "3")); assertFilterMatches(new RegexDimFilter("dim3", ".*ANYMORE", changeNullFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); diff --git a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java index e3cf67ef6c13..140b8b78dcf9 100644 --- a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java @@ -37,6 +37,7 @@ import io.druid.query.search.ContainsSearchQuerySpec; import io.druid.query.search.SearchQuerySpec; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -105,8 +106,18 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - // SearchQueryFilter always returns false for null row values. - assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue(""), null), ImmutableList.of("1", "2", "3", "4", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + // SearchQueryFilter always returns false for null row values. + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue(""), null), + ImmutableList.of("1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue(""), null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("10"), null), ImmutableList.of("1")); assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("2"), null), ImmutableList.of("2")); assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("1"), null), ImmutableList.of("1", "3")); @@ -118,7 +129,14 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue(""), null), ImmutableList.of("0", "3", "4")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue(""), null), ImmutableList.of("0", "3", "4")); + } else { + assertFilterMatches( + new SearchQueryDimFilter("dim2", specForValue(""), null), + ImmutableList.of("0", "2", "3", "4") + ); + } assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("a"), null), ImmutableList.of("0", "3")); assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("b"), null), ImmutableList.of("0")); assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("c"), null), ImmutableList.of("4")); @@ -151,10 +169,31 @@ public void testSearchQueryWithExtractionFn() String nullJsFn = "function(str) { if (str === null) { return 'NOT_NULL_ANYMORE'; } else { return str;} }"; ExtractionFn changeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("0")); - assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("ab"), changeNullFn), ImmutableList.of("4", "5")); - - assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("1", "2", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of("0") + ); + assertFilterMatches( + new SearchQueryDimFilter("dim2", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of("1", "2", "5") + ); + + } else { + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of() + ); + assertFilterMatches( + new SearchQueryDimFilter("dim2", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of("1", "5") + ); + } + + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue("ab"), changeNullFn), + ImmutableList.of("4", "5") + ); assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("a"), changeNullFn), ImmutableList.of("0", "3")); assertFilterMatches(new SearchQueryDimFilter("dim3", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); diff --git a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java index 2f41178aee52..d45e45dd364f 100644 --- a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java @@ -38,6 +38,7 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Assert; @@ -113,8 +114,13 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); - assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); + } else { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of()); + assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); + } assertFilterMatches(new SelectorDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new SelectorDimFilter("dim1", "2", null), ImmutableList.of("2")); assertFilterMatches(new SelectorDimFilter("dim1", "1", null), ImmutableList.of("3")); @@ -126,8 +132,13 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "2", "5")); - assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("1", "2", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "2", "5")); + assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("1", "2", "5")); + } else { + assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "5")); + assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("2")); + } assertFilterMatches(new SelectorDimFilter("dim2", "a", null), ImmutableList.of("0", "3")); assertFilterMatches(new SelectorDimFilter("dim2", "b", null), ImmutableList.of("0")); assertFilterMatches(new SelectorDimFilter("dim2", "c", null), ImmutableList.of("4")); @@ -138,7 +149,11 @@ public void testMultiValueStringColumn() public void testMissingColumnSpecifiedInDimensionList() { assertFilterMatches(new SelectorDimFilter("dim3", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + } else { + assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of()); + } assertFilterMatches(new SelectorDimFilter("dim3", "a", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim3", "b", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim3", "c", null), ImmutableList.of()); @@ -148,7 +163,11 @@ public void testMissingColumnSpecifiedInDimensionList() public void testMissingColumnNotSpecifiedInDimensionList() { assertFilterMatches(new SelectorDimFilter("dim4", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + } else { + assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of()); + } assertFilterMatches(new SelectorDimFilter("dim4", "a", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim4", "b", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim4", "c", null), ImmutableList.of()); @@ -200,7 +219,23 @@ public void testSelectorWithLookupExtractionFn() ); LookupExtractor mapExtractor3 = new MapLookupExtractor(stringMap3, false); LookupExtractionFn lookupFn3 = new LookupExtractionFn(mapExtractor3, false, null, false, true); - assertFilterMatches(new SelectorDimFilter("dim0", null, lookupFn3), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + // Nulls and empty strings are considered equivalent + assertFilterMatches( + new SelectorDimFilter("dim0", null, lookupFn3), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new SelectorDimFilter("dim0", null, lookupFn3), + ImmutableList.of("0", "2", "3", "4", "5") + ); + assertFilterMatches( + new SelectorDimFilter("dim0", "", lookupFn3), + ImmutableList.of("1") + ); + } + final Map stringMap4 = ImmutableMap.of( "9", "4" @@ -228,6 +263,7 @@ public void testSelectorWithLookupExtractionFn() SelectorDimFilter optFilter5 = new SelectorDimFilter("dim0", "5", lookupFn5); SelectorDimFilter optFilter6 = new SelectorDimFilter("dim0", "5", lookupFn6); + InDimFilter optFilter2Optimized = new InDimFilter("dim0", Arrays.asList("2", "5"), null); SelectorDimFilter optFilter4Optimized = new SelectorDimFilter("dim0", "5", null); SelectorDimFilter optFilter6Optimized = new SelectorDimFilter("dim0", "5", null); @@ -238,10 +274,15 @@ public void testSelectorWithLookupExtractionFn() Assert.assertTrue(optFilter4Optimized.equals(optFilter4.optimize())); Assert.assertTrue(optFilter5.equals(optFilter5.optimize())); Assert.assertTrue(optFilter6Optimized.equals(optFilter6.optimize())); - assertFilterMatches(optFilter1, ImmutableList.of("0", "1", "2", "5")); + assertFilterMatches(optFilter2, ImmutableList.of("2", "5")); - assertFilterMatches(optFilter3, ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + // Null and Empty strings are same + assertFilterMatches(optFilter3, ImmutableList.of("0", "1", "2", "3", "4", "5")); + } else { + assertFilterMatches(optFilter3, ImmutableList.of("0", "2", "3", "4", "5")); + } assertFilterMatches(optFilter4, ImmutableList.of("5")); assertFilterMatches(optFilter5, ImmutableList.of()); assertFilterMatches(optFilter6, ImmutableList.of("5")); @@ -250,6 +291,20 @@ public void testSelectorWithLookupExtractionFn() // remove these when ExtractionDimFilter is removed. assertFilterMatches(new ExtractionDimFilter("dim1", "UNKNOWN", lookupFn, null), ImmutableList.of("0", "1", "2", "5")); assertFilterMatches(new ExtractionDimFilter("dim0", "5", lookupFn2, null), ImmutableList.of("2", "5")); - assertFilterMatches(new ExtractionDimFilter("dim0", null, lookupFn3, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches( + new ExtractionDimFilter("dim0", null, lookupFn3, null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new ExtractionDimFilter("dim0", null, lookupFn3, null), + ImmutableList.of("0", "2", "3", "4", "5") + ); + assertFilterMatches( + new ExtractionDimFilter("dim0", "", lookupFn3, null), + ImmutableList.of("1") + ); + } } } diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index d3cf377e4421..b67cde6efe04 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -50,6 +50,7 @@ import io.druid.query.topn.TopNResultValue; import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumns; import io.druid.segment.data.IndexedInts; @@ -608,9 +609,13 @@ public Object apply(Cursor cursor) // no null id, so should get empty dims array Assert.assertEquals(0, rowD.size()); IndexedInts rowE = dimSelector3E.getRow(); - Assert.assertEquals(1, rowE.size()); - // the null id - Assert.assertEquals(0, rowE.get(0)); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(1, rowE.size()); + // the null id + Assert.assertEquals(0, rowE.get(0)); + } else { + Assert.assertEquals(0, rowE.size()); + } cursor.advance(); rowNumInCursor++; } diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java index e283efed0905..c542f40e99ac 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java @@ -37,6 +37,7 @@ import io.druid.query.aggregation.FilteredAggregatorFactory; import io.druid.query.filter.SelectorDimFilter; import io.druid.segment.CloserRule; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -224,10 +225,24 @@ public void testNullDimensionTransform() throws IndexSizeExceededException Row row = index.iterator().next(); - Assert.assertEquals(Arrays.asList(new String[]{"", "", "A"}), row.getRaw("string")); - Assert.assertEquals(Arrays.asList(new String[]{"", "", String.valueOf(Float.POSITIVE_INFINITY)}), row.getRaw("float")); - Assert.assertEquals(Arrays.asList(new String[]{"", "", String.valueOf(Long.MIN_VALUE)}), row.getRaw("long")); - Assert.assertEquals(0.0, row.getRaw("double")); + + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(Arrays.asList(null, null, "A"), row.getRaw("string")); + Assert.assertEquals( + Arrays.asList(null, null, String.valueOf(Float.POSITIVE_INFINITY)), + row.getRaw("float") + ); + Assert.assertEquals(Arrays.asList(null, null, String.valueOf(Long.MIN_VALUE)), row.getRaw("long")); + Assert.assertEquals(NullHandlingHelper.nullToDefault((Double) null), row.getRaw("double")); + } else { + Assert.assertEquals(Arrays.asList(null, "", "A"), row.getRaw("string")); + Assert.assertEquals( + Arrays.asList(null, "", String.valueOf(Float.POSITIVE_INFINITY)), + row.getRaw("float") + ); + Assert.assertEquals(Arrays.asList(null, "", String.valueOf(Long.MIN_VALUE)), row.getRaw("long")); + Assert.assertEquals(NullHandlingHelper.nullToDefault((Double) null), row.getRaw("double")); + } } @Test diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionObjectSelectorTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionObjectSelectorTest.java index 82c54d5544b7..6daec09eff48 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionObjectSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionObjectSelectorTest.java @@ -23,6 +23,7 @@ import io.druid.common.guava.SettableSupplier; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import org.junit.Assert; import org.junit.Test; @@ -40,10 +41,10 @@ public void testSupplierFromDimensionSelector() ); Assert.assertNotNull(supplier); - Assert.assertEquals(null, supplier.get()); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? "" : null, supplier.get()); settableSupplier.set(null); - Assert.assertEquals(null, supplier.get()); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? "" : null, supplier.get()); settableSupplier.set("1234"); Assert.assertEquals("1234", supplier.get()); @@ -58,7 +59,7 @@ public void testSupplierFromObjectSelectorObject() ); Assert.assertNotNull(supplier); - Assert.assertEquals(null, supplier.get()); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? "" : null, supplier.get()); settableSupplier.set(1.1f); Assert.assertEquals(1.1f, supplier.get()); @@ -81,7 +82,6 @@ public void testSupplierFromObjectSelectorNumber() objectSelectorFromSupplier(settableSupplier, Number.class) ); - Assert.assertNotNull(supplier); Assert.assertEquals(null, supplier.get()); diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java index 92b718bfab4a..7fc853797dbb 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -33,6 +33,7 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.ValueType; import org.junit.Assert; @@ -95,7 +96,7 @@ public void testObjectSelector() final ObjectColumnSelector selector = XPLUSY.makeObjectColumnSelector("expr", COLUMN_SELECTOR_FACTORY); COLUMN_SELECTOR_FACTORY.setRow(ROW0); - Assert.assertEquals(null, selector.get()); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? "" : null, selector.get()); COLUMN_SELECTOR_FACTORY.setRow(ROW1); Assert.assertEquals(4.0d, selector.get()); @@ -134,7 +135,7 @@ public void testLongSelectorUsingStringFunction() Assert.assertEquals(0L, selector.getLong()); COLUMN_SELECTOR_FACTORY.setRow(ROW1); - Assert.assertEquals(4L, selector.getLong()); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 4L : 0L, selector.getLong()); COLUMN_SELECTOR_FACTORY.setRow(ROW2); Assert.assertEquals(0L, selector.getLong()); @@ -214,7 +215,7 @@ public void testDimensionSelectorUsingStringFunction() COLUMN_SELECTOR_FACTORY.setRow(ROW1); Assert.assertEquals(1, selector.getRow().size()); - Assert.assertEquals("4", selector.lookupName(selector.getRow().get(0))); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? "4" : null, selector.lookupName(selector.getRow().get(0))); COLUMN_SELECTOR_FACTORY.setRow(ROW2); Assert.assertEquals(1, selector.getRow().size()); diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java index d2a27b3c8331..45f72e072eda 100644 --- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -405,6 +405,12 @@ public float getFloat() { return selector.getFloat(); } + + @Override + public boolean isNull() + { + return selector.isNull(); + } }; } @@ -421,6 +427,12 @@ public long getLong() { return theLong; } + + @Override + public boolean isNull() + { + return false; + } }; } diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java b/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java index 132f669725c4..5141cb4c7d55 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java @@ -285,19 +285,19 @@ public Object getRaw(String dimension) @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { return new Double(metrics.get(metric)).doubleValue(); } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { return metrics.get(metric); } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return new Float(metrics.get(metric)).longValue(); } diff --git a/server/src/test/java/io/druid/realtime/firehose/CombiningFirehoseFactoryTest.java b/server/src/test/java/io/druid/realtime/firehose/CombiningFirehoseFactoryTest.java index c44f7b80efe9..f8dadfa86001 100644 --- a/server/src/test/java/io/druid/realtime/firehose/CombiningFirehoseFactoryTest.java +++ b/server/src/test/java/io/druid/realtime/firehose/CombiningFirehoseFactoryTest.java @@ -92,19 +92,19 @@ public List getDimension(String dimension) } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { return metricValue; } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return new Float(metricValue).longValue(); } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { return new Float(metricValue).doubleValue(); } diff --git a/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java b/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java index 90b1e66f4b24..f57133f05960 100644 --- a/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java +++ b/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java @@ -811,19 +811,19 @@ public List getDimension(String dimension) } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { - return 0; + return 0F; } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return 0L; } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { return 0.0d; } diff --git a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java index a9a9c1ab48ea..0d599a473110 100644 --- a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java +++ b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java @@ -605,21 +605,21 @@ public List getDimension(String dimension) } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { - return 0; + return 0F; } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return 0L; } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { - return 0; + return 0D; } @Override @@ -665,13 +665,13 @@ public List getDimension(String dimension) } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { - return 0; + return 0F; } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return 0L; } @@ -683,9 +683,9 @@ public Object getRaw(String dimension) } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { - return 0; + return 0D; } @Override diff --git a/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java b/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java index b84d154b2f07..402fe62f1f9e 100644 --- a/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java +++ b/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java @@ -112,19 +112,19 @@ public List getDimension(String dimension) } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { - return 0; + return 0F; } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return 0L; } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { return 0.0d; } @@ -177,19 +177,19 @@ public List getDimension(String dimension) } @Override - public float getFloatMetric(String metric) + public Float getFloatMetric(String metric) { - return 0; + return 0F; } @Override - public long getLongMetric(String metric) + public Long getLongMetric(String metric) { return 0L; } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { return 0.0d; } diff --git a/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java b/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java index e7688a52de99..d4a4781460e2 100644 --- a/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java +++ b/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java @@ -237,19 +237,19 @@ public Object getRaw(String s) } @Override - public float getFloatMetric(String s) + public Float getFloatMetric(String s) { - return 0; + return 0F; } @Override - public long getLongMetric(String s) + public Long getLongMetric(String s) { return 0L; } @Override - public double getDoubleMetric(String metric) + public Double getDoubleMetric(String metric) { return 0.0d; } diff --git a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java index 88e9fbd2415a..b2b9cae6acb0 100644 --- a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import io.druid.java.util.common.DateTimes; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.NullHandlingHelper; import io.druid.sql.calcite.planner.Calcites; import io.druid.sql.calcite.planner.DruidOperatorTable; import io.druid.sql.calcite.planner.PlannerConfig; @@ -133,6 +134,7 @@ public void testSelectAllInFirstFrame() throws Exception Meta.Frame.create( 0, true, + NullHandlingHelper.useDefaultValuesForNull() ? Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", "", 2.0f}, @@ -140,6 +142,14 @@ public void testSelectAllInFirstFrame() throws Exception new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", "", 6.0f} + ) : + Lists.newArrayList( + new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, + new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", null, 2.0f}, + new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, + new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, + new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, + new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", null, 6.0f} ) ), frame @@ -159,9 +169,14 @@ public void testSelectSplitOverTwoFrames() throws Exception Meta.Frame.create( 0, false, + NullHandlingHelper.useDefaultValuesForNull() ? Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", "", 2.0f} + ) : + Lists.newArrayList( + new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, + new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", null, 2.0f} ) ), frame @@ -174,11 +189,18 @@ public void testSelectSplitOverTwoFrames() throws Exception Meta.Frame.create( 2, true, + NullHandlingHelper.useDefaultValuesForNull() ? Lists.newArrayList( new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", "", 6.0f} + ) : + Lists.newArrayList( + new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, + new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, + new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, + new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", null, 6.0f} ) ), frame diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 52975b80cf67..7f50eb185add 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -83,9 +83,11 @@ import io.druid.query.topn.InvertedTopNMetricSpec; import io.druid.query.topn.NumericTopNMetricSpec; import io.druid.query.topn.TopNQueryBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; +import io.druid.sql.calcite.expression.DruidExpression; import io.druid.sql.calcite.filtration.Filtration; import io.druid.sql.calcite.planner.Calcites; import io.druid.sql.calcite.planner.DruidOperatorTable; @@ -406,6 +408,7 @@ public void testSelectStar() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, @@ -413,6 +416,14 @@ public void testSelectStar() throws Exception new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} + ) : + ImmutableList.of( + new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", null, 2f, 2.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, + new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, + new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, + new Object[]{T("2001-01-03"), 1L, "abc", null, 6f, 6.0, HLLCV1.class.getName()} ) ); } @@ -466,9 +477,14 @@ public void testSelectStarWithLimit() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0, HLLCV1.class.getName()}, new Object[]{T("2000-01-02"), 1L, "10.1", "", 2.0f, 2.0, HLLCV1.class.getName()} + ) : + ImmutableList.of( + new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", null, 2.0f, 2.0, HLLCV1.class.getName()} ) ); } @@ -491,9 +507,14 @@ public void testSelectWithProjection() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"a"}, new Object[]{""} + ) : + ImmutableList.of( + new Object[]{"a"}, + new Object[]{null} ) ); } @@ -515,9 +536,14 @@ public void testSelectStarWithLimitTimeDescending() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6d, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5d, HLLCV1.class.getName()} + ) : + ImmutableList.of( + new Object[]{T("2001-01-03"), 1L, "abc", null, 6f, 6d, HLLCV1.class.getName()}, + new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5d, HLLCV1.class.getName()} ) ); } @@ -555,6 +581,7 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, @@ -562,6 +589,14 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} + ) : + ImmutableList.of( + new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", null, 2f, 2.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, + new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, + new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, + new Object[]{T("2001-01-03"), 1L, "abc", null, 6f, 6.0, HLLCV1.class.getName()} ) ); } @@ -581,9 +616,14 @@ public void testSelectSingleColumnTwice() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"a", "a"}, new Object[]{"", ""} + ) : + ImmutableList.of( + new Object[]{"a", "a"}, + new Object[]{null, null} ) ); } @@ -672,7 +712,7 @@ public void testSelfJoinWithFallback() throws Exception newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) - .filters(NOT(SELECTOR("dim1", "", null))) + .filters(NOT(SELECTOR("dim1", NullHandlingHelper.useDefaultValuesForNull() ? null : "", null))) .columns("__time", "cnt", "dim1", "dim2", "m1", "m2", "unique_dim1") .resultFormat(ScanQuery.RESULT_FORMAT_COMPACTED_LIST) .context(QUERY_CONTEXT_DEFAULT) @@ -687,6 +727,7 @@ public void testSelfJoinWithFallback() throws Exception @Test public void testExplainSelfJoinWithFallback() throws Exception { + String emptyStringEq = NullHandlingHelper.useDefaultValuesForNull() ? null : "\"\""; testQuery( PLANNER_CONFIG_FALLBACK, "EXPLAIN PLAN FOR\n" @@ -701,7 +742,9 @@ public void testExplainSelfJoinWithFallback() throws Exception "BindableProject(dim1=[$9], dim10=[$2], dim2=[$3])\n" + " BindableJoin(condition=[=($9, $3)], joinType=[inner])\n" + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807},\"descending\":false}])\n" - + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"\",\"extractionFn\":null}},\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807},\"descending\":false}])\n" + + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":" + + emptyStringEq + + ",\"extractionFn\":null}},\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807},\"descending\":false}])\n" } ) ); @@ -1153,7 +1196,7 @@ public void testGroupByCaseWhen() throws Exception + "'match-cnt'," + "(timestamp_extract(\"__time\",'DAY','UTC') == 0)," + "'zero '," - + "'')", + + DruidExpression.nullLiteral() + ")", ValueType.STRING ) ) @@ -1162,10 +1205,16 @@ public void testGroupByCaseWhen() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 2L}, new Object[]{"match-cnt", 1L}, new Object[]{"match-m1 ", 3L} + ) : + ImmutableList.of( + new Object[]{null, 2L}, + new Object[]{"match-cnt", 1L}, + new Object[]{"match-m1 ", 3L} ) ); } @@ -1181,27 +1230,60 @@ public void testNullEmptyStringEquality() throws Exception "NULLIF(dim2, 'a') IS NULL" ); - for (String where : wheres) { - testQuery( - "SELECT COUNT(*)\n" - + "FROM druid.foo\n" - + "WHERE " + where, - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,(\"dim2\" == ''))")) - .aggregators(AGGS(new CountAggregatorFactory("a0"))) - .context(TIMESERIES_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - // Matches everything but "abc" - new Object[]{5L} - ) - ); - } + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE " + wheres.get(0), + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," + + (NullHandlingHelper.useDefaultValuesForNull() ? "1" : "0") + + ",(\"dim2\" == ''))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + NullHandlingHelper.useDefaultValuesForNull() ? + ImmutableList.of( + // Matches everything but "abc" + new Object[]{5L} + ) : + ImmutableList.of( + // Matches only null value + new Object[]{1L} + ) + ); + + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE " + wheres.get(1), + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,(\"dim2\" == " + + DruidExpression.nullLiteral() + + "))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + NullHandlingHelper.useDefaultValuesForNull() ? + ImmutableList.of( + // Matches everything but "abc" + new Object[]{5L} + ) : + ImmutableList.of( + // Matches everything but "abc" and ""(Empty String) + new Object[]{4L} + ) + ); + } @Test @@ -1220,7 +1302,7 @@ public void testCoalesceColumns() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "case_searched((\"dim2\" != ''),\"dim2\",\"dim1\")", + "case_searched((\"dim2\" != " + DruidExpression.nullLiteral() + "),\"dim2\",\"dim1\")", ValueType.STRING ) ) @@ -1229,11 +1311,18 @@ public void testCoalesceColumns() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"10.1", 1L}, new Object[]{"2", 1L}, new Object[]{"a", 2L}, new Object[]{"abc", 2L} + ) : + ImmutableList.of( + new Object[]{"", 1L}, + new Object[]{"10.1", 1L}, + new Object[]{"a", 2L}, + new Object[]{"abc", 2L} ) ); } @@ -1257,7 +1346,7 @@ public void testColumnIsNull() throws Exception .build() ), ImmutableList.of( - new Object[]{3L} + new Object[]{NullHandlingHelper.useDefaultValuesForNull() ? 3L : 2L} ) ); } @@ -1447,14 +1536,18 @@ public void testCountNullableColumn() throws Exception .aggregators(AGGS( new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), - NOT(SELECTOR("dim2", "", null)) + NOT(SELECTOR("dim2", null, null)) ) )) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L} + ) : + ImmutableList.of( + new Object[]{4L} ) ); } @@ -1473,7 +1566,11 @@ public void testCountNullableExpression() throws Exception new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), EXPRESSION_FILTER( - "(case_searched((\"dim2\" == 'abc'),'yes',(\"dim2\" == 'def'),'yes','') != '')" + "(case_searched((\"dim2\" == 'abc'),'yes',(\"dim2\" == 'def'),'yes'," + + DruidExpression.nullLiteral() + + ") != " + + DruidExpression.nullLiteral() + + ")" ) ) )) @@ -1763,7 +1860,7 @@ public void testFilterOnStringAsNumber() throws Exception public void testSimpleAggregations() throws Exception { testQuery( - "SELECT COUNT(*), COUNT(cnt), COUNT(dim1), AVG(cnt), SUM(cnt), SUM(cnt) + MIN(cnt) + MAX(cnt) FROM druid.foo", + "SELECT COUNT(*), COUNT(cnt), COUNT(dim1), AVG(cnt), SUM(cnt), SUM(cnt) + MIN(cnt) + MAX(cnt), COUNT(dim2) FROM druid.foo", ImmutableList.of( Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -1774,13 +1871,17 @@ public void testSimpleAggregations() throws Exception new CountAggregatorFactory("a0"), new FilteredAggregatorFactory( new CountAggregatorFactory("a1"), - NOT(SELECTOR("dim1", "", null)) + NOT(SELECTOR("dim1", null, null)) ), new LongSumAggregatorFactory("a2:sum", "cnt"), new CountAggregatorFactory("a2:count"), new LongSumAggregatorFactory("a3", "cnt"), new LongMinAggregatorFactory("a4", "cnt"), - new LongMaxAggregatorFactory("a5", "cnt") + new LongMaxAggregatorFactory("a5", "cnt"), + new FilteredAggregatorFactory( + new CountAggregatorFactory("a6"), + NOT(SELECTOR("dim2", null, null)) + ) ) ) .postAggregators( @@ -1799,8 +1900,12 @@ public void testSimpleAggregations() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( - new Object[]{6L, 6L, 5L, 1L, 6L, 8L} + new Object[]{6L, 6L, 5L, 1L, 6L, 8L, 3L} + ) : + ImmutableList.of( + new Object[]{6L, 6L, 6L, 1L, 6L, 8L, 4L} ) ); } @@ -1971,7 +2076,7 @@ public void testFilteredAggregations() throws Exception new CountAggregatorFactory("a3"), AND( NOT(SELECTOR("dim1", "1", null)), - NOT(SELECTOR("dim2", "", null)) + NOT(SELECTOR("dim2", null, null)) ) ), new FilteredAggregatorFactory( @@ -2005,8 +2110,12 @@ public void testFilteredAggregations() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{1L, 5L, 1L, 2L, 5L, 5L, 2L, 1L, 5L, 1L} + ) : + ImmutableList.of( + new Object[]{1L, 5L, 1L, 3L, 5L, 5L, 2L, 1L, 5L, 1L} ) ); } @@ -2837,7 +2946,14 @@ public void testSelectDistinctWithLimit() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{""}, + new Object[]{"a"}, + new Object[]{"abc"} + ) : ImmutableList.of( + new Object[]{null}, new Object[]{""}, new Object[]{"a"}, new Object[]{"abc"} @@ -2861,7 +2977,14 @@ public void testSelectDistinctWithSortAsOuterQuery() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{""}, + new Object[]{"a"}, + new Object[]{"abc"} + ) : ImmutableList.of( + new Object[]{null}, new Object[]{""}, new Object[]{"a"}, new Object[]{"abc"} @@ -2885,7 +3008,14 @@ public void testSelectDistinctWithSortAsOuterQuery2() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{""}, + new Object[]{"a"}, + new Object[]{"abc"} + ) : ImmutableList.of( + new Object[]{null}, new Object[]{""}, new Object[]{"a"}, new Object[]{"abc"} @@ -2921,10 +3051,17 @@ public void testSelectDistinctWithSortAsOuterQuery4() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{""}, new Object[]{"abc"}, new Object[]{"a"} + ) : + ImmutableList.of( + new Object[]{null}, + new Object[]{"abc"}, + new Object[]{"a"}, + new Object[]{""} ) ); } @@ -2987,14 +3124,14 @@ public void testExactCountDistinct() throws Exception .setAggregatorSpecs(AGGS( new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), - NOT(SELECTOR("d0", "", null)) + NOT(SELECTOR("d0", null, null)) ) )) .setContext(QUERY_CONTEXT_DEFAULT) .build() ), ImmutableList.of( - new Object[]{2L} + new Object[]{NullHandlingHelper.useDefaultValuesForNull() ? 2L : 3L} ) ); } @@ -3064,16 +3201,23 @@ public void testExactCountDistinctWithGroupingAndOtherAggregators() throws Excep new LongSumAggregatorFactory("a0", "a0"), new FilteredAggregatorFactory( new CountAggregatorFactory("a1"), - NOT(SELECTOR("d0", "", null)) + NOT(SELECTOR("d0", null, null)) ) )) .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 3L, 3L}, new Object[]{"a", 2L, 1L}, new Object[]{"abc", 1L, 1L} + ) : + ImmutableList.of( + new Object[]{null, 2L, 2L}, + new Object[]{"", 1L, 1L}, + new Object[]{"a", 2L, 2L}, + new Object[]{"abc", 1L, 1L} ) ); } @@ -3145,8 +3289,12 @@ public void testApproxCountDistinct() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 3L, 2L, 2L, 2L, 6L} + ) : + ImmutableList.of( + new Object[]{6L, 3L, 2L, 1L, 2L, 6L} ) ); } @@ -3199,8 +3347,12 @@ public void testDoubleNestedGroupBy() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 3L} + ) : + ImmutableList.of( + new Object[]{6L, 4L} ) ); } @@ -3263,8 +3415,12 @@ public void testExactCountDistinctUsingSubquery() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 3L} + ) : + ImmutableList.of( + new Object[]{6L, 4L} ) ); } @@ -3302,7 +3458,7 @@ public void testTopNFilterJoin() throws Exception .setDataSource(CalciteTests.DATASOURCE1) .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) - .setDimFilter(IN("dim2", ImmutableList.of("", "a"), null)) + .setDimFilter(IN("dim2", Arrays.asList(null, "a"), null)) .setDimensions(DIMS(new DefaultDimensionSpec("dim1", "d0"))) .setAggregatorSpecs(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec( @@ -3320,12 +3476,19 @@ public void testTopNFilterJoin() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 1L}, new Object[]{"1", 1L}, new Object[]{"10.1", 1L}, new Object[]{"2", 1L}, new Object[]{"abc", 1L} + ) : + ImmutableList.of( + new Object[]{"", 1L}, + new Object[]{"1", 1L}, + new Object[]{"10.1", 1L}, + new Object[]{"abc", 1L} ) ); } @@ -3448,7 +3611,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception + " SELECT DISTINCT dim2\n" + " FROM druid.foo\n" + " WHERE SUBSTRING(dim2, 1, 1) IN (\n" - + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 <> ''\n" + + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 IS NOT NULL\n" + " )\n" + ")", ImmutableList.of(), @@ -3456,7 +3619,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception new Object[]{ "DruidOuterQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[],\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807},\"descending\":false}])\n" + " DruidSemiJoin(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807},\"descending\":false}], leftExpressions=[[DruidExpression{simpleExtraction=null, expression='substring(\"dim2\", 0, 1)'}]], rightKeys=[[0]])\n" - + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"\",\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807},\"descending\":false}])\n" + + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807},\"descending\":false}])\n" } ) ); @@ -3495,8 +3658,51 @@ public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws Except .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L, 2L} + ) : + ImmutableList.of( + new Object[]{5L, 3L} + ) + ); + + testQuery( + "SELECT\n" + + " SUM(cnt),\n" + + " COUNT(*)\n" + + "FROM (SELECT dim2, SUM(cnt) AS cnt FROM druid.foo GROUP BY dim2)\n" + + "WHERE dim2 IS NOT NULL", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource( + new QueryDataSource( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(QSS(Filtration.eternity())) + .setDimFilter(NOT(SELECTOR("dim2", null, null))) + .setGranularity(Granularities.ALL) + .setDimensions(DIMS(new DefaultDimensionSpec("dim2", "d0"))) + .setAggregatorSpecs(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ) + ) + .setInterval(QSS(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setAggregatorSpecs(AGGS( + new LongSumAggregatorFactory("a0", "a0"), + new CountAggregatorFactory("a1") + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + NullHandlingHelper.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{3L, 2L} + ) : + ImmutableList.of( + new Object[]{4L, 3L} ) ); } @@ -3535,8 +3741,12 @@ public void testExactCountDistinctUsingSubqueryWithWhereToOuterFilter() throws E .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L, 1L} + ) : + ImmutableList.of( + new Object[]{2L, 1L} ) ); } @@ -3622,10 +3832,15 @@ public void testHistogramUsingSubquery() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"1", 1L}, new Object[]{"2", 1L}, new Object[]{"3", 1L} + ) : + ImmutableList.of( + new Object[]{"1", 2L}, + new Object[]{"2", 2L} ) ); } @@ -3672,9 +3887,14 @@ public void testHistogramUsingSubqueryWithSort() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"1", 1L}, new Object[]{"2", 1L} + ) : + ImmutableList.of( + new Object[]{"1", 2L}, + new Object[]{"2", 2L} ) ); } @@ -3863,12 +4083,20 @@ public void testRegexpExtract() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", ""}, new Object[]{"1", "1"}, new Object[]{"2", "2"}, new Object[]{"a", "a"}, new Object[]{"d", "d"} + ) : + ImmutableList.of( + new Object[]{null, null}, + new Object[]{"1", "1"}, + new Object[]{"2", "2"}, + new Object[]{"a", "a"}, + new Object[]{"d", "d"} ) ); } @@ -3905,11 +4133,18 @@ public void testGroupBySortPushDown() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"10.1", "", 1L}, new Object[]{"2", "", 1L}, new Object[]{"abc", "", 1L}, new Object[]{"", "a", 1L} + ) : + ImmutableList.of( + new Object[]{"10.1", null, 1L}, + new Object[]{"abc", null, 1L}, + new Object[]{"2", "", 1L}, + new Object[]{"", "a", 1L} ) ); } @@ -3952,11 +4187,18 @@ public void testGroupByLimitPushDownWithHavingOnLong() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"10.1", "", 1L}, new Object[]{"2", "", 1L}, new Object[]{"abc", "", 1L}, new Object[]{"", "a", 1L} + ) : + ImmutableList.of( + new Object[]{"10.1", null, 1L}, + new Object[]{"abc", null, 1L}, + new Object[]{"2", "", 1L}, + new Object[]{"", "a", 1L} ) ); } @@ -4356,12 +4598,21 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Excepti .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{T("2000"), "", 2L}, new Object[]{T("2000"), "a", 1L}, new Object[]{T("2001"), "", 1L}, new Object[]{T("2001"), "a", 1L}, new Object[]{T("2001"), "abc", 1L} + ) : + ImmutableList.of( + new Object[]{T("2000"), null, 1L}, + new Object[]{T("2000"), "", 1L}, + new Object[]{T("2000"), "a", 1L}, + new Object[]{T("2001"), null, 1L}, + new Object[]{T("2001"), "a", 1L}, + new Object[]{T("2001"), "abc", 1L} ) ); } @@ -4437,9 +4688,14 @@ public void testFilterAndGroupByLookup() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 5L}, new Object[]{"xabc", 1L} + ) : + ImmutableList.of( + new Object[]{null, 5L}, + new Object[]{"xabc", 1L} ) ); } @@ -4476,7 +4732,7 @@ public void testCountDistinctOfLookup() throws Exception .build() ), ImmutableList.of( - new Object[]{2L} + new Object[]{NullHandlingHelper.useDefaultValuesForNull() ? 2L : 1L} ) ); } @@ -4634,7 +4890,9 @@ public void testTimeseriesUsingTimeFloorWithTimeShift() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_floor(timestamp_shift(\"__time\",'P1D',-1),'P1M','','UTC')", + "timestamp_floor(timestamp_shift(\"__time\",'P1D',-1),'P1M'," + + DruidExpression.nullLiteral() + + ",'UTC')", ValueType.LONG ) ) @@ -5220,12 +5478,21 @@ public void testGroupByTimeAndOtherDimension() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", T("2000-01-01"), 2L}, new Object[]{"", T("2001-01-01"), 1L}, new Object[]{"a", T("2000-01-01"), 1L}, new Object[]{"a", T("2001-01-01"), 1L}, new Object[]{"abc", T("2001-01-01"), 1L} + ) : + ImmutableList.of( + new Object[]{null, T("2000-01-01"), 1L}, + new Object[]{null, T("2001-01-01"), 1L}, + new Object[]{"", T("2000-01-01"), 1L}, + new Object[]{"a", T("2000-01-01"), 1L}, + new Object[]{"a", T("2001-01-01"), 1L}, + new Object[]{"abc", T("2001-01-01"), 1L} ) ); } diff --git a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java index fcef10cd7306..eb573e2aede6 100644 --- a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java @@ -199,7 +199,9 @@ public void testTimeFloor() timestampLiteral(DateTimes.of("2000-02-03T04:05:06Z")), rexBuilder.makeLiteral("PT1H") ), - DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H'," + + DruidExpression.nullLiteral() + + ",'UTC')"), DateTimes.of("2000-02-03T04:00:00").getMillis() ); @@ -222,7 +224,7 @@ public void testTimeFloor() true ) ), - "timestamp_floor(\"t\",'P1D','','America/Los_Angeles')" + "timestamp_floor(\"t\",'P1D'," + DruidExpression.nullLiteral() + ",'America/Los_Angeles')" ), DateTimes.of("2000-02-02T08:00:00").getMillis() ); @@ -244,7 +246,7 @@ public void testOtherTimeFloor() "t", new TimeFormatExtractionFn(null, null, null, Granularities.YEAR, true) ), - "timestamp_floor(\"t\",'P1Y','','UTC')" + "timestamp_floor(\"t\",'P1Y'," + DruidExpression.nullLiteral() + ",'UTC')" ), DateTimes.of("2000").getMillis() ); @@ -261,7 +263,7 @@ public void testOtherTimeCeil() inputRef("t"), rexBuilder.makeFlag(TimeUnitRange.YEAR) ), - DruidExpression.fromExpression("timestamp_ceil(\"t\",'P1Y','','UTC')"), + DruidExpression.fromExpression("timestamp_ceil(\"t\",'P1Y'," + DruidExpression.nullLiteral() + ",'UTC')"), DateTimes.of("2001").getMillis() ); } @@ -581,7 +583,7 @@ public void testCastAsDate() ), DruidExpression.of( SimpleExtraction.of("t", new TimeFormatExtractionFn(null, null, null, Granularities.DAY, true)), - "timestamp_floor(\"t\",'P1D','','UTC')" + "timestamp_floor(\"t\",'P1D'," + DruidExpression.nullLiteral() + ",'UTC')" ), DateTimes.of("2000-02-03").getMillis() ); @@ -592,7 +594,7 @@ public void testCastAsDate() inputRef("dstr") ), DruidExpression.fromExpression( - "timestamp_floor(timestamp_parse(\"dstr\",'yyyy-MM-dd'),'P1D','','UTC')" + "timestamp_floor(timestamp_parse(\"dstr\",'yyyy-MM-dd'),'P1D'," + DruidExpression.nullLiteral() + ",'UTC')" ), DateTimes.of("2000-02-03").getMillis() ); @@ -610,7 +612,7 @@ public void testCastFromDate() ) ), DruidExpression.fromExpression( - "timestamp_format(timestamp_floor(\"t\",'P1D','','UTC'),'yyyy-MM-dd')" + "timestamp_format(timestamp_floor(\"t\",'P1D'," + DruidExpression.nullLiteral() + ",'UTC'),'yyyy-MM-dd')" ), "2000-02-03" ); @@ -625,7 +627,7 @@ public void testCastFromDate() ), DruidExpression.of( SimpleExtraction.of("t", new TimeFormatExtractionFn(null, null, null, Granularities.DAY, true)), - "timestamp_floor(\"t\",'P1D','','UTC')" + "timestamp_floor(\"t\",'P1D'," + DruidExpression.nullLiteral() + ",'UTC')" ), DateTimes.of("2000-02-03").getMillis() ); diff --git a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java index 05684b41bf34..1f8c202ff593 100644 --- a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java @@ -23,12 +23,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.ISE; import io.druid.java.util.common.Pair; import io.druid.math.expr.ExprMacroTable; import io.druid.query.QueryInterruptedException; import io.druid.query.ResourceLimitExceededException; +import io.druid.segment.NullHandlingHelper; import io.druid.sql.calcite.planner.Calcites; import io.druid.sql.calcite.planner.DruidOperatorTable; import io.druid.sql.calcite.planner.PlannerConfig; @@ -168,10 +170,18 @@ public void testFieldAliasingGroupBy() throws Exception ).rhs; Assert.assertEquals( + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( ImmutableMap.of("x", "", "y", ""), ImmutableMap.of("x", "a", "y", "a"), ImmutableMap.of("x", "abc", "y", "abc") + ) : + ImmutableList.of( + // x and y both should be null instead of empty string + Maps.transformValues(ImmutableMap.of("x", "", "y", ""), (val) -> null), + ImmutableMap.of("x", "", "y", ""), + ImmutableMap.of("x", "a", "y", "a"), + ImmutableMap.of("x", "abc", "y", "abc") ), rows ); diff --git a/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java b/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java index 6afc5ea27d7e..96eaa4994f97 100644 --- a/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java @@ -19,6 +19,7 @@ package io.druid.sql.calcite.planner; +import io.druid.sql.calcite.expression.DruidExpression; import org.junit.Assert; import org.junit.Test; @@ -27,7 +28,7 @@ public class CalcitesTest @Test public void testEscapeStringLiteral() { - Assert.assertEquals("''", Calcites.escapeStringLiteral(null)); + Assert.assertEquals(DruidExpression.nullLiteral(), Calcites.escapeStringLiteral(null)); Assert.assertEquals("''", Calcites.escapeStringLiteral("")); Assert.assertEquals("'foo'", Calcites.escapeStringLiteral("foo")); Assert.assertEquals("'foo bar'", Calcites.escapeStringLiteral("foo bar")); From 51a9db017d1ce7008576a38fdfb555431a219eec Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 02:26:21 +0530 Subject: [PATCH 09/50] handle review comment --- .../java/io/druid/query/aggregation/NullableAggregator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index 3a09ee46f54d..cf2b36a7c9ae 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -25,7 +25,7 @@ public class NullableAggregator implements Aggregator { private final Aggregator delegate; private final ColumnValueSelector selector; - private boolean isNull; + private boolean isNull = true; public NullableAggregator(Aggregator delegate, ColumnValueSelector selector) { From 2d81e72cf79e63481bf16f4dbe84e62d7705214e Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 16 Sep 2017 02:27:57 +0530 Subject: [PATCH 10/50] use default value for isNull in NullableAgg/combiners --- .../java/io/druid/query/aggregation/NullableAggregator.java | 1 - .../query/aggregation/NullableDoubleAggregateCombiner.java | 2 +- .../druid/query/aggregation/NullableLongAggregateCombiner.java | 2 +- .../query/aggregation/NullableObjectAggregateCombiner.java | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index cf2b36a7c9ae..044183f97730 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -31,7 +31,6 @@ public NullableAggregator(Aggregator delegate, ColumnValueSelector selector) { this.delegate = delegate; this.selector = selector; - reset(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java index 3d16cacfb649..f5bf33e71eff 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java @@ -23,7 +23,7 @@ public class NullableDoubleAggregateCombiner extends DoubleAggregateCombiner { - private boolean isNull; + private boolean isNull = true; private DoubleAggregateCombiner delegate; diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java index 61ca61ea9a10..3c5664cad245 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java @@ -23,7 +23,7 @@ public class NullableLongAggregateCombiner extends LongAggregateCombiner { - private boolean isNull; + private boolean isNull = true; private LongAggregateCombiner delegate; diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java index e644c9db5c13..dd2cc0ba179d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java @@ -23,7 +23,7 @@ public class NullableObjectAggregateCombiner extends ObjectAggregateCombiner { - private boolean isNull; + private boolean isNull = true; private ObjectAggregateCombiner delegate; From 65c338040fa2dd3350ce46c09b44cbccc08a19ed Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 18 Sep 2017 20:56:26 +0530 Subject: [PATCH 11/50] Handle review comments --- .../java/io/druid/data/input/MapBasedRow.java | 5 ++ .../main/java/io/druid/data/input/Row.java | 8 +-- .../java/io/druid/math/expr/ExprEval.java | 3 + .../io/druid/indexer/IndexGeneratorJob.java | 5 ++ .../java/io/druid/indexer/InputRowSerde.java | 9 ++- .../druid/indexer/hadoop/SegmentInputRow.java | 5 ++ .../druid/java/util/common/StringUtils.java | 34 +++++++--- .../java/util/common/StringUtilsTest.java | 6 +- .../druid/query/aggregation/Aggregator.java | 4 +- .../query/aggregation/BufferAggregator.java | 1 - .../FloatMaxAggregatorFactory.java | 8 +-- .../aggregation/LongMaxAggregatorFactory.java | 7 +- .../aggregation/LongMinAggregatorFactory.java | 7 +- .../aggregation/LongSumAggregatorFactory.java | 2 +- ...er.java => NullableAggregateCombiner.java} | 40 ++++++++--- .../query/aggregation/NullableAggregator.java | 24 +++++-- .../aggregation/NullableBufferAggregator.java | 23 +++++-- .../NullableLongAggregateCombiner.java | 61 ----------------- .../NullableObjectAggregateCombiner.java | 67 ------------------- .../SimpleDoubleAggregatorFactory.java | 2 +- .../SimpleFloatAggregatorFactory.java | 2 +- .../first/DoubleFirstAggregatorFactory.java | 2 +- .../first/FloatFirstAggregatorFactory.java | 2 +- .../first/LongFirstAggregatorFactory.java | 2 +- .../last/DoubleLastAggregatorFactory.java | 2 +- .../last/FloatLastAggregatorFactory.java | 2 +- .../last/LongLastAggregatorFactory.java | 2 +- .../io/druid/segment/ColumnValueSelector.java | 17 ++++- .../io/druid/segment/NullHandlingHelper.java | 21 ++---- .../segment/NullValueHandlingConfig.java | 6 +- .../druid/segment/column/ColumnBuilder.java | 15 +---- .../segment/column/ColumnCapabilities.java | 2 - .../column/ColumnCapabilitiesImpl.java | 15 ----- .../segment/column/ColumnDescriptor.java | 11 --- .../column/DictionaryEncodedColumn.java | 3 + .../io/druid/segment/column/DoubleColumn.java | 5 +- .../io/druid/segment/column/FloatColumn.java | 4 +- .../column/SimpleDictionaryEncodedColumn.java | 1 + .../io/druid/segment/data/GenericIndexed.java | 4 +- .../segment/incremental/IncrementalIndex.java | 5 -- .../firehose/WikipediaIrcDecoder.java | 2 +- .../CombiningFirehoseFactoryTest.java | 1 - .../HashBasedNumberedShardSpecTest.java | 5 ++ 43 files changed, 186 insertions(+), 266 deletions(-) rename processing/src/main/java/io/druid/query/aggregation/{NullableDoubleAggregateCombiner.java => NullableAggregateCombiner.java} (55%) delete mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java delete mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java diff --git a/api/src/main/java/io/druid/data/input/MapBasedRow.java b/api/src/main/java/io/druid/data/input/MapBasedRow.java index 2be503918e76..462e9c8e8ee5 100644 --- a/api/src/main/java/io/druid/data/input/MapBasedRow.java +++ b/api/src/main/java/io/druid/data/input/MapBasedRow.java @@ -27,6 +27,7 @@ import io.druid.java.util.common.parsers.ParseException; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.Collections; import java.util.List; import java.util.Map; @@ -95,12 +96,14 @@ public List getDimension(String dimension) } @Override + @Nullable public Object getRaw(String dimension) { return event.get(dimension); } @Override + @Nullable public Float getFloatMetric(String metric) { Object metricValue = event.get(metric); @@ -124,6 +127,7 @@ public Float getFloatMetric(String metric) } @Override + @Nullable public Long getLongMetric(String metric) { Object metricValue = event.get(metric); @@ -148,6 +152,7 @@ public Long getLongMetric(String metric) } @Override + @Nullable public Double getDoubleMetric(String metric) { Object metricValue = event.get(metric); diff --git a/api/src/main/java/io/druid/data/input/Row.java b/api/src/main/java/io/druid/data/input/Row.java index 94b8eb78a0d9..29de817585d7 100644 --- a/api/src/main/java/io/druid/data/input/Row.java +++ b/api/src/main/java/io/druid/data/input/Row.java @@ -72,7 +72,7 @@ public interface Row extends Comparable * * @return the value of the provided column name */ - public @Nullable Object getRaw(String dimension); + @Nullable public Object getRaw(String dimension); /** * Returns the float value of the given metric column. @@ -82,7 +82,7 @@ public interface Row extends Comparable * * @return the float value for the provided column name. */ - public @Nullable Float getFloatMetric(String metric); + @Nullable public Float getFloatMetric(String metric); /** * Returns the long value of the given metric column. @@ -92,7 +92,7 @@ public interface Row extends Comparable * * @return the long value for the provided column name. */ - public @Nullable Long getLongMetric(String metric); + @Nullable public Long getLongMetric(String metric); /** * Returns the double value of the given metric column. @@ -102,5 +102,5 @@ public interface Row extends Comparable * * @return the double value for the provided column name. */ - public @Nullable Double getDoubleMetric(String metric); + @Nullable public Double getDoubleMetric(String metric); } diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 676a616e0f49..ed8b5bfffc00 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -25,6 +25,8 @@ import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.IAE; +import javax.annotation.Nullable; + /** */ public abstract class ExprEval @@ -107,6 +109,7 @@ public boolean isNull() public abstract double asDouble(); + @Nullable public String asString() { return value == null ? null : String.valueOf(value); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java index 12e4ae4a11a6..64ee3506f5f3 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java @@ -73,6 +73,7 @@ import org.joda.time.DateTime; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -420,24 +421,28 @@ public List getDimension(String dimension) } @Override + @Nullable public Object getRaw(String dimension) { return row.getRaw(dimension); } @Override + @Nullable public Float getFloatMetric(String metric) { return row.getFloatMetric(metric); } @Override + @Nullable public Long getLongMetric(String metric) { return row.getLongMetric(metric); } @Override + @Nullable public Double getDoubleMetric(String metric) { return row.getDoubleMetric(metric); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index 0a4c1b7ec6b2..98a6f691eea4 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -40,6 +40,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; +import javax.annotation.Nullable; import java.io.DataInput; import java.io.IOException; import java.util.List; @@ -132,7 +133,7 @@ public InputRow get() } } - private static void writeBytes(byte[] value, ByteArrayDataOutput out) throws IOException + private static void writeBytes(@Nullable byte[] value, ByteArrayDataOutput out) throws IOException { int length = value == null ? -1 : value.length; WritableUtils.writeVInt(out, length); @@ -141,9 +142,9 @@ private static void writeBytes(byte[] value, ByteArrayDataOutput out) throws IOE } } - private static void writeString(String value, ByteArrayDataOutput out) throws IOException + private static void writeString(@Nullable String value, ByteArrayDataOutput out) throws IOException { - writeBytes(StringUtils.toUtf8(value), out); + writeBytes(StringUtils.toUtf8Nullable(value), out); } private static void writeStringArray(List values, ByteArrayDataOutput out) throws IOException @@ -158,12 +159,14 @@ private static void writeStringArray(List values, ByteArrayDataOutput ou } } + @Nullable private static String readString(DataInput in) throws IOException { byte[] result = readBytes(in); return StringUtils.fromUtf8(result); } + @Nullable private static byte[] readBytes(DataInput in) throws IOException { int size = WritableUtils.readVInt(in); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java index 442d0bd75325..74ff7088a0e1 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java @@ -23,6 +23,7 @@ import io.druid.data.input.Row; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.List; /** @@ -65,24 +66,28 @@ public List getDimension(String dimension) } @Override + @Nullable public Object getRaw(String dimension) { return delegate.getRaw(dimension); } @Override + @Nullable public Float getFloatMetric(String metric) { return delegate.getFloatMetric(metric); } @Override + @Nullable public Long getLongMetric(String metric) { return delegate.getLongMetric(metric); } @Override + @Nullable public Double getDoubleMetric(String metric) { return delegate.getDoubleMetric(metric); diff --git a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java index d4fc2a2f818e..22ab8bab3cd3 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java @@ -67,12 +67,8 @@ public static byte[] toUtf8WithNullToEmpty(final String string) return string == null ? EMPTY_BYTES : toUtf8(string); } - @Nullable public static String fromUtf8(final byte[] bytes) { - if (bytes == null) { - return null; - } try { return new String(bytes, UTF8_STRING); } @@ -82,6 +78,15 @@ public static String fromUtf8(final byte[] bytes) } } + @Nullable + public static String fromUtf8Nullable(final byte[] bytes) + { + if (bytes == null) { + return null; + } + return fromUtf8(bytes); + } + public static String fromUtf8(final ByteBuffer buffer, final int numBytes) { final byte[] bytes = new byte[numBytes]; @@ -89,17 +94,21 @@ public static String fromUtf8(final ByteBuffer buffer, final int numBytes) return StringUtils.fromUtf8(bytes); } + @Nullable + public static String fromUtf8Nullable(final ByteBuffer buffer, final int numBytes) + { + final byte[] bytes = new byte[numBytes]; + buffer.get(bytes); + return StringUtils.fromUtf8Nullable(bytes); + } + public static String fromUtf8(final ByteBuffer buffer) { return StringUtils.fromUtf8(buffer, buffer.remaining()); } - @Nullable public static byte[] toUtf8(final String string) { - if (string == null) { - return null; - } try { return string.getBytes(UTF8_STRING); } @@ -109,6 +118,15 @@ public static byte[] toUtf8(final String string) } } + @Nullable + public static byte[] toUtf8Nullable(@Nullable final String string) + { + if (string == null) { + return null; + } + return toUtf8(string); + } + /** * Equivalent of String.format(Locale.ENGLISH, message, formatArgs). */ diff --git a/java-util/src/test/java/io/druid/java/util/common/StringUtilsTest.java b/java-util/src/test/java/io/druid/java/util/common/StringUtilsTest.java index e3010e8ba8cf..30cea273b722 100644 --- a/java-util/src/test/java/io/druid/java/util/common/StringUtilsTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/StringUtilsTest.java @@ -85,10 +85,10 @@ public void testNullPointerByteBuffer() StringUtils.fromUtf8((ByteBuffer) null); } - @Test - public void testNullByteArray() + @Test(expected = NullPointerException.class) + public void testNullPointerByteArray() { - Assert.assertNull(StringUtils.fromUtf8((byte[]) null)); + StringUtils.fromUtf8((byte[]) null); } @Test diff --git a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java index 397aeefa09cf..cac559359fb9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java @@ -55,8 +55,8 @@ default double getDouble() } /** - * returns true if the Aggregator support returning null values and the aggregated value is Null. - * The default implementation always return false to enable smooth backward compatibility. re-implement if your aggregator is nullable. + * returns true if the Aggregator supports returning null values and the aggregated value is Null. + * The default implementation always return false to enable smooth backward compatibility, re-implement if your aggregator is nullable. */ default boolean isNull() { diff --git a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java index 9a9d3ce75d71..f85ec81f6039 100644 --- a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java @@ -181,7 +181,6 @@ default void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, By /** * Returns true if the aggregator is nullable and the aggregated value is null *

- *

* Implementations must not change the position, limit or mark of the given buffer *

* Implementations are only required to support this method if they the aggregator supports null values. diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java index f44e0b0c40fc..8ef29d6d0357 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -63,10 +63,10 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { FloatColumnSelector floatColumnSelector = getFloatColumnSelector(metricFactory, Float.NEGATIVE_INFINITY); - return NullHandlingHelper.getNullableAggregator(new FloatMaxBufferAggregator(getFloatColumnSelector( - metricFactory, - Float.NEGATIVE_INFINITY - )), floatColumnSelector); + return NullHandlingHelper.getNullableAggregator( + new FloatMaxBufferAggregator(floatColumnSelector), + floatColumnSelector + ); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index 89ddd8d1b4f7..fa82619c6d95 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -110,7 +110,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return NullHandlingHelper.getNullableCombiner(new LongAggregateCombiner() + LongAggregateCombiner combiner = new LongAggregateCombiner() { private long max; @@ -131,7 +131,8 @@ public long getLong() { return max; } - }); + }; + return NullHandlingHelper.getNullableCombiner(combiner); } @Override @@ -218,7 +219,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Longs.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Longs.BYTES + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 3524d83b4a54..1c735304c233 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -113,7 +113,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return NullHandlingHelper.getNullableCombiner(new LongAggregateCombiner() + LongAggregateCombiner combiner = new LongAggregateCombiner() { private long min; @@ -134,7 +134,8 @@ public long getLong() { return min; } - }); + }; + return NullHandlingHelper.getNullableCombiner(combiner); } @Override @@ -221,7 +222,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Longs.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Longs.BYTES + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 9942eb632f99..445d46e7b9e6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -196,7 +196,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Longs.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Longs.BYTES + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java similarity index 55% rename from processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java rename to processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java index f5bf33e71eff..1316b2e65141 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableDoubleAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -21,13 +21,20 @@ import io.druid.segment.ColumnValueSelector; -public class NullableDoubleAggregateCombiner extends DoubleAggregateCombiner +/** + * The result of a NullableAggregateCombiner will be null if all the values to be combined are null values or no values are combined at all. + * If any of the value is non-null, the result would be the value of the delegate combiner. + * Note that the delegate combiner is not required to perform check for isNull on the columnValueSelector as only non-null values + * will be passed to the delegate combiner. + */ + +public class NullableAggregateCombiner implements AggregateCombiner { - private boolean isNull = true; + private boolean isNullResult = true; - private DoubleAggregateCombiner delegate; + private final AggregateCombiner delegate; - public NullableDoubleAggregateCombiner(DoubleAggregateCombiner delegate) + public NullableAggregateCombiner(AggregateCombiner delegate) { this.delegate = delegate; } @@ -35,17 +42,26 @@ public NullableDoubleAggregateCombiner(DoubleAggregateCombiner delegate) @Override public void reset(ColumnValueSelector selector) { - isNull = true; + isNullResult = true; delegate.reset(selector); } @Override public void fold(ColumnValueSelector selector) { - if (isNull && !selector.isNull()) { - isNull = false; + boolean isCurrentValNull = selector.isNull(); + if (!isCurrentValNull) { + if (isNullResult) { + isNullResult = false; + } + delegate.fold(selector); } - delegate.fold(selector); + } + + @Override + public float getFloat() + { + return delegate.getFloat(); } @Override @@ -54,9 +70,15 @@ public double getDouble() return delegate.getDouble(); } + @Override + public long getLong() + { + return delegate.getLong(); + } + @Override public boolean isNull() { - return isNull; + return isNullResult; } } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index 044183f97730..30cb9a6c8fe3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -21,11 +21,17 @@ import io.druid.segment.ColumnValueSelector; +/** + * The result of a NullableAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. + * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. + * Note that the delegate aggregator is not required to perform check for isNull on the columnValueSelector as only non-null values + * will be passed to the delegate aggregator. + */ public class NullableAggregator implements Aggregator { private final Aggregator delegate; private final ColumnValueSelector selector; - private boolean isNull = true; + private boolean isNullResult = true; public NullableAggregator(Aggregator delegate, ColumnValueSelector selector) { @@ -36,22 +42,28 @@ public NullableAggregator(Aggregator delegate, ColumnValueSelector selector) @Override public void aggregate() { - if (isNull && !selector.isNull()) { - isNull = false; + boolean isCurrentValNull = selector.isNull(); + if (!isCurrentValNull) { + if (isNullResult) { + isNullResult = false; + } + delegate.aggregate(); } - delegate.aggregate(); } @Override public void reset() { - isNull = true; + isNullResult = true; delegate.reset(); } @Override public Object get() { + if(isNullResult){ + return null; + } return delegate.get(); } @@ -76,7 +88,7 @@ public double getDouble() @Override public boolean isNull() { - return isNull; + return isNullResult; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 2f70f086edc0..8c24132e7b12 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -23,6 +23,13 @@ import java.nio.ByteBuffer; +/** + * The result of a NullableBufferAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. + * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. + * Note that the delegate aggregator is not required to perform check for isNull on the columnValueSelector as only non-null values + * will be passed to the delegate aggregator. + */ + public class NullableBufferAggregator implements BufferAggregator { public static final byte IS_NULL_BYTE = (byte) 1; @@ -47,15 +54,21 @@ public void init(ByteBuffer buf, int position) @Override public void aggregate(ByteBuffer buf, int position) { - if (buf.get(position) == IS_NULL_BYTE && !selector.isNull()) { - buf.put(position, IS_NOT_NULL_BYTE); + boolean isCurrentValNull = selector.isNull(); + if (!isCurrentValNull) { + if (buf.get(position) == IS_NULL_BYTE) { + buf.put(position, IS_NOT_NULL_BYTE); + } + delegate.aggregate(buf, position + Byte.BYTES); } - delegate.aggregate(buf, position + Byte.BYTES); } @Override public Object get(ByteBuffer buf, int position) { + if(buf.get(position) == IS_NULL_BYTE){ + return null; + } return delegate.get(buf, position + Byte.BYTES); } @@ -80,12 +93,12 @@ public double getDouble(ByteBuffer buf, int position) @Override public boolean isNull(ByteBuffer buf, int position) { - return buf.get() == IS_NULL_BYTE; + return buf.get(position) == IS_NULL_BYTE; } @Override public void close() { - + // Nothing to close. } } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java deleted file mode 100644 index 3c5664cad245..000000000000 --- a/processing/src/main/java/io/druid/query/aggregation/NullableLongAggregateCombiner.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.query.aggregation; - -import io.druid.segment.ColumnValueSelector; - -public class NullableLongAggregateCombiner extends LongAggregateCombiner -{ - private boolean isNull = true; - - private LongAggregateCombiner delegate; - - public NullableLongAggregateCombiner(LongAggregateCombiner delegate) - { - this.delegate = delegate; - } - - @Override - public void reset(ColumnValueSelector selector) - { - isNull = true; - } - - @Override - public void fold(ColumnValueSelector selector) - { - if (isNull && !selector.isNull()) { - isNull = true; - } - delegate.fold(selector); - } - - @Override - public long getLong() - { - return delegate.getLong(); - } - - @Override - public boolean isNull() - { - return isNull; - } -} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java deleted file mode 100644 index dd2cc0ba179d..000000000000 --- a/processing/src/main/java/io/druid/query/aggregation/NullableObjectAggregateCombiner.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.query.aggregation; - -import io.druid.segment.ColumnValueSelector; - -public class NullableObjectAggregateCombiner extends ObjectAggregateCombiner -{ - private boolean isNull = true; - - private ObjectAggregateCombiner delegate; - - public NullableObjectAggregateCombiner(ObjectAggregateCombiner delegate) - { - this.delegate = delegate; - } - - @Override - public void reset(ColumnValueSelector selector) - { - isNull = true; - } - - @Override - public void fold(ColumnValueSelector selector) - { - if (isNull && !selector.isNull()) { - isNull = true; - } - delegate.fold(selector); - } - - @Override - public Class classOfObject() - { - return delegate.classOfObject(); - } - - @Override - public Object get() - { - return delegate.get(); - } - - @Override - public boolean isNull() - { - return isNull; - } -} diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index 72125cc43ceb..949892533fb5 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -82,7 +82,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Double.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Double.BYTES + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java index 4f6337c436ed..adc1d8f7c127 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -81,7 +81,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Float.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Float.BYTES + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index fc0d270c347b..d66b8fd1585a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -233,7 +233,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES + Double.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Long.BYTES + Double.BYTES + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 3af4462b3494..a85f019f08e7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -232,7 +232,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES + Float.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Long.BYTES + Float.BYTES + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 9fd80a908825..9d927ff85ccb 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -225,7 +225,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES * 2 + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Long.BYTES * 2 + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 35f7c13888a5..5c8fd15835f2 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -223,7 +223,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES + Double.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Long.BYTES + Double.BYTES + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 44e613083333..341029695948 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -225,7 +225,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES + Float.BYTES + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Long.BYTES + Float.BYTES + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 18ef42250d0a..673d538da00f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -221,7 +221,7 @@ public String getTypeName() @Override public int getMaxIntermediateSize() { - return Long.BYTES * 2 + (NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES); + return Long.BYTES * 2 + NullHandlingHelper.extraAggregatorBytes(); } @Override diff --git a/processing/src/main/java/io/druid/segment/ColumnValueSelector.java b/processing/src/main/java/io/druid/segment/ColumnValueSelector.java index 0d788a9143d8..333ec39df264 100644 --- a/processing/src/main/java/io/druid/segment/ColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnValueSelector.java @@ -24,15 +24,30 @@ * * This interface has methods to get the value in all primitive types, that have corresponding basic aggregators in * Druid: Sum, Min, Max, etc: {@link #getFloat()}, {@link #getDouble()} and {@link #getLong()} to support "polymorphic" - * rollup aggregation during index merging. + * rollup aggregation during index merging. Additionally, it also has a {@link #isNull()} method which can be used to + * check whether the column has null value or not. */ public interface ColumnValueSelector { + /** + * @return float column value, 0.0F if the value is null. + */ float getFloat(); + /** + * @return double column value, 0.0D if the value is null. + */ double getDouble(); + /** + * @return long column value, 0L if the value is null. + */ long getLong(); + /** + * checks if the column value is null. + * + * @return true if the column value for is null + */ boolean isNull(); } diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index dac18d223854..ec53dcbaef3d 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -21,16 +21,12 @@ import com.google.common.base.Strings; import com.google.inject.Inject; +import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.BufferAggregator; -import io.druid.query.aggregation.DoubleAggregateCombiner; -import io.druid.query.aggregation.LongAggregateCombiner; +import io.druid.query.aggregation.NullableAggregateCombiner; import io.druid.query.aggregation.NullableAggregator; import io.druid.query.aggregation.NullableBufferAggregator; -import io.druid.query.aggregation.NullableDoubleAggregateCombiner; -import io.druid.query.aggregation.NullableLongAggregateCombiner; -import io.druid.query.aggregation.NullableObjectAggregateCombiner; -import io.druid.query.aggregation.ObjectAggregateCombiner; public class NullHandlingHelper { @@ -89,18 +85,13 @@ public static BufferAggregator getNullableAggregator(BufferAggregator aggregator return INSTANCE.isUseDefaultValuesForNull() ? aggregator : new NullableBufferAggregator(aggregator, selector); } - public static DoubleAggregateCombiner getNullableCombiner(DoubleAggregateCombiner combiner) + public static AggregateCombiner getNullableCombiner(AggregateCombiner combiner) { - return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableDoubleAggregateCombiner(combiner); + return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableAggregateCombiner(combiner); } - public static LongAggregateCombiner getNullableCombiner(LongAggregateCombiner combiner) + public static int extraAggregatorBytes() { - return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableLongAggregateCombiner(combiner); - } - - public static ObjectAggregateCombiner getNullableCombiner(ObjectAggregateCombiner combiner) - { - return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableObjectAggregateCombiner(combiner); + return NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES; } } diff --git a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java b/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java index 8104aefd1663..7e457534bbf8 100644 --- a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java +++ b/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java @@ -24,20 +24,18 @@ public class NullValueHandlingConfig { - private static boolean DEFAULT_USE_DEFAULT_VALUES_FOR_NULL = true; @JsonProperty("useDefaultValueForNull") - private Boolean useDefaultValuesForNull = DEFAULT_USE_DEFAULT_VALUES_FOR_NULL; + private final boolean useDefaultValuesForNull; @JsonCreator public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) { this.useDefaultValuesForNull = useDefaultValuesForNull == null - ? DEFAULT_USE_DEFAULT_VALUES_FOR_NULL + ? true : useDefaultValuesForNull; } - public boolean isUseDefaultValuesForNull() { return useDefaultValuesForNull; diff --git a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java index bbf56499fc01..fb531bbbc710 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java @@ -30,7 +30,6 @@ public class ColumnBuilder { private ValueType type = null; private boolean hasMultipleValues = false; - private boolean hasNullValues = false; private Supplier dictionaryEncodedColumn = null; private Supplier runLengthColumn = null; @@ -100,17 +99,6 @@ public ColumnBuilder setSpatialIndex(Supplier spatialIndex) return this; } - public ColumnBuilder setHasNullValues(boolean hasNullValues) - { - this.hasNullValues = hasNullValues; - return this; - } - - public boolean isHasNullValues() - { - return hasNullValues; - } - public ColumnBuilder setNullValueBitmap(Supplier nullValueBitmap) { this.nullValueBitmap = nullValueBitmap; @@ -128,8 +116,7 @@ public Column build() .setHasBitmapIndexes(bitmapIndex != null) .setHasSpatialIndexes(spatialIndex != null) .setRunLengthEncoded(runLengthColumn != null) - .setHasMultipleValues(hasMultipleValues) - .setHasNullValues(hasNullValues), + .setHasMultipleValues(hasMultipleValues), dictionaryEncodedColumn, runLengthColumn, genericColumn, diff --git a/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java b/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java index ea88a8985e0e..f6b416415326 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java @@ -31,7 +31,5 @@ public interface ColumnCapabilities public boolean hasSpatialIndexes(); public boolean hasMultipleValues(); - public boolean hasNullValues(); - public ColumnCapabilitiesImpl merge(ColumnCapabilities other); } diff --git a/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java b/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java index 2ba4bd7a5389..fff3d1c05dc4 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java @@ -32,7 +32,6 @@ public class ColumnCapabilitiesImpl implements ColumnCapabilities private boolean hasInvertedIndexes = false; private boolean hasSpatialIndexes = false; private boolean hasMultipleValues = false; - private boolean hasNullValues = false; @Override @JsonProperty @@ -112,19 +111,6 @@ public ColumnCapabilitiesImpl setHasMultipleValues(boolean hasMultipleValues) return this; } - @Override - @JsonProperty("hasNullValues") - public boolean hasNullValues() - { - return hasNullValues; - } - - public ColumnCapabilitiesImpl setHasNullValues(boolean hasNullValues) - { - this.hasNullValues = hasNullValues; - return this; - } - @Override public ColumnCapabilitiesImpl merge(ColumnCapabilities other) { @@ -145,7 +131,6 @@ public ColumnCapabilitiesImpl merge(ColumnCapabilities other) this.hasInvertedIndexes |= other.hasBitmapIndexes(); this.hasSpatialIndexes |= other.hasSpatialIndexes(); this.hasMultipleValues |= other.hasMultipleValues(); - this.hasNullValues |= other.hasNullValues(); return this; } diff --git a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java index b51f41fb7612..0aa194075726 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java @@ -44,20 +44,17 @@ public static Builder builder() private final ValueType valueType; private final boolean hasMultipleValues; - private final boolean hasNullValues; private final List parts; @JsonCreator public ColumnDescriptor( @JsonProperty("valueType") ValueType valueType, @JsonProperty("hasMultipleValues") boolean hasMultipleValues, - @JsonProperty("hasNullValues") boolean hasNullValues, @JsonProperty("parts") List parts ) { this.valueType = valueType; this.hasMultipleValues = hasMultipleValues; - this.hasNullValues = hasNullValues; this.parts = parts; } @@ -73,12 +70,6 @@ public boolean isHasMultipleValues() return hasMultipleValues; } - @JsonProperty - public boolean isHasNullValues() - { - return hasNullValues; - } - @JsonProperty public List getParts() { @@ -108,7 +99,6 @@ public Column read(ByteBuffer buffer, ColumnConfig columnConfig, SmooshedFileMap final ColumnBuilder builder = new ColumnBuilder() .setType(valueType) .setHasMultipleValues(hasMultipleValues) - .setHasNullValues(hasNullValues) .setFileMapper(smooshedFiles); for (ColumnPartSerde part : parts) { @@ -164,7 +154,6 @@ public ColumnDescriptor build() return new ColumnDescriptor( valueType, hasMultipleValues == null ? false : hasMultipleValues, - hasNullValues, parts ); } diff --git a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java index 6fed80fea70c..5d4a9e21e6aa 100644 --- a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java @@ -24,6 +24,7 @@ import io.druid.segment.data.IndexedInts; import io.druid.segment.data.ReadableOffset; +import javax.annotation.Nullable; import java.io.Closeable; /** @@ -34,6 +35,8 @@ public interface DictionaryEncodedColumn extends public boolean hasMultipleValues(); public int getSingleValueRow(int rowNum); public IndexedInts getMultiValueRow(int rowNum); + + @Nullable public ActualType lookupName(int id); public int lookupId(ActualType name); public int getCardinality(); diff --git a/processing/src/main/java/io/druid/segment/column/DoubleColumn.java b/processing/src/main/java/io/druid/segment/column/DoubleColumn.java index 654495a3bd55..1abf8dbff46b 100644 --- a/processing/src/main/java/io/druid/segment/column/DoubleColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DoubleColumn.java @@ -30,9 +30,6 @@ public class DoubleColumn extends AbstractColumn private static final ColumnCapabilitiesImpl CAPABILITIES = new ColumnCapabilitiesImpl() .setType(ValueType.DOUBLE); - private static final ColumnCapabilitiesImpl CAPABILITIES_WITH_NULL = new ColumnCapabilitiesImpl() - .setType(ValueType.DOUBLE).setHasNullValues(true); - private final CompressedDoublesIndexedSupplier column; private final ImmutableBitmap nullValueBitmap; @@ -52,7 +49,7 @@ public int getLength() @Override public ColumnCapabilities getCapabilities() { - return nullValueBitmap.isEmpty() ? CAPABILITIES : CAPABILITIES_WITH_NULL; + return CAPABILITIES; } @Override diff --git a/processing/src/main/java/io/druid/segment/column/FloatColumn.java b/processing/src/main/java/io/druid/segment/column/FloatColumn.java index 8d16a8761b1a..1905593ae13d 100644 --- a/processing/src/main/java/io/druid/segment/column/FloatColumn.java +++ b/processing/src/main/java/io/druid/segment/column/FloatColumn.java @@ -30,8 +30,6 @@ public class FloatColumn extends AbstractColumn private static final ColumnCapabilitiesImpl CAPABILITIES = new ColumnCapabilitiesImpl() .setType(ValueType.FLOAT); - private static final ColumnCapabilitiesImpl CAPABILITIES_WITH_NULL = new ColumnCapabilitiesImpl() - .setType(ValueType.FLOAT).setHasNullValues(true); private final CompressedFloatsIndexedSupplier column; private final ImmutableBitmap nullValueBitmap; @@ -46,7 +44,7 @@ public FloatColumn(CompressedFloatsIndexedSupplier column, ImmutableBitmap nullV @Override public ColumnCapabilities getCapabilities() { - return nullValueBitmap.isEmpty() ? CAPABILITIES : CAPABILITIES_WITH_NULL; + return CAPABILITIES; } @Override diff --git a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java index a91314db963d..6048f6f30342 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java @@ -85,6 +85,7 @@ public IndexedInts getMultiValueRow(int rowNum) } @Override + @Nullable public String lookupName(int id) { return cachedLookups.get(id); diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 4f2a56859c18..6aa2d77b8e9b 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -90,13 +90,13 @@ public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) if (numBytes < 0) { return null; } - return NullHandlingHelper.defaultToNull(StringUtils.fromUtf8(buffer, numBytes)); + return NullHandlingHelper.defaultToNull(StringUtils.fromUtf8Nullable(buffer, numBytes)); } @Override public byte[] toBytes(String val) { - return StringUtils.toUtf8(val); + return StringUtils.toUtf8Nullable(val); } @Override diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index c0912248a018..c7cecafb0850 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -55,7 +55,6 @@ import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; import io.druid.segment.Metadata; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; @@ -565,10 +564,6 @@ TimeAndDims toTimeAndDims(InputRow row) throws IndexSizeExceededException capabilities.setHasMultipleValues(true); } - if (!NullHandlingHelper.useDefaultValuesForNull() && !capabilities.hasNullValues() && raw == null) { - capabilities.setHasNullValues(true); - } - if (wasNewDim) { if (overflow == null) { overflow = Lists.newArrayList(); diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java b/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java index 5141cb4c7d55..5f364ddfbc7e 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java @@ -287,7 +287,7 @@ public Object getRaw(String dimension) @Override public Double getDoubleMetric(String metric) { - return new Double(metrics.get(metric)).doubleValue(); + return new Double(metrics.get(metric)); } @Override diff --git a/server/src/test/java/io/druid/realtime/firehose/CombiningFirehoseFactoryTest.java b/server/src/test/java/io/druid/realtime/firehose/CombiningFirehoseFactoryTest.java index f8dadfa86001..39269e01b8d6 100644 --- a/server/src/test/java/io/druid/realtime/firehose/CombiningFirehoseFactoryTest.java +++ b/server/src/test/java/io/druid/realtime/firehose/CombiningFirehoseFactoryTest.java @@ -109,7 +109,6 @@ public Double getDoubleMetric(String metric) return new Float(metricValue).doubleValue(); } - @Override public Object getRaw(String dimension) { diff --git a/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java b/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java index d4a4781460e2..5f5cb854c9e1 100644 --- a/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java +++ b/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java @@ -35,6 +35,7 @@ import org.junit.Assert; import org.junit.Test; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; @@ -231,24 +232,28 @@ public List getDimension(String s) } @Override + @Nullable public Object getRaw(String s) { return null; } @Override + @Nullable public Float getFloatMetric(String s) { return 0F; } @Override + @Nullable public Long getLongMetric(String s) { return 0L; } @Override + @Nullable public Double getDoubleMetric(String metric) { return 0.0d; From 22f23352a0ad834da0ba6a3e7effda4cf49ff5f5 Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 18 Sep 2017 20:59:25 +0530 Subject: [PATCH 12/50] checkstyle errors fix --- .../query/aggregation/NullableAggregator.java | 2 +- .../aggregation/NullableBufferAggregator.java | 2 +- .../segment/SchemalessTestSimpleTest.java | 64 +++++++++---------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index 30cb9a6c8fe3..b13e442c732f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -61,7 +61,7 @@ public void reset() @Override public Object get() { - if(isNullResult){ + if (isNullResult) { return null; } return delegate.get(); diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 8c24132e7b12..6978e6d56e8a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -66,7 +66,7 @@ public void aggregate(ByteBuffer buf, int position) @Override public Object get(ByteBuffer buf, int position) { - if(buf.get(position) == IS_NULL_BYTE){ + if (buf.get(position) == IS_NULL_BYTE) { return null; } return delegate.get(buf, position + Byte.BYTES); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java index a5b0adc09808..89837fab7323 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java @@ -41,9 +41,9 @@ import io.druid.query.aggregation.post.ArithmeticPostAggregator; import io.druid.query.aggregation.post.ConstantPostAggregator; import io.druid.query.aggregation.post.FieldAccessPostAggregator; -import io.druid.query.search.SearchResultValue; import io.druid.query.search.SearchHit; import io.druid.query.search.SearchQuery; +import io.druid.query.search.SearchResultValue; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.spec.QuerySegmentSpec; import io.druid.query.timeboundary.TimeBoundaryQuery; @@ -156,13 +156,13 @@ public void testFullOnTimeseries() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 11L) - .put("index", 900.0) - .put("addRowsIndexConstant", 912.0) - .put("uniques", 2.000977198748901D) - .put("maxIndex", 100.0) - .put("minIndex", 0.0) - .build() + .put("rows", 11L) + .put("index", 900.0) + .put("addRowsIndexConstant", 912.0) + .put("uniques", 2.000977198748901D) + .put("maxIndex", 100.0) + .put("minIndex", 0.0) + .build() ) ) ); @@ -204,36 +204,36 @@ public void testFullOnTopN() Arrays.asList( new DimensionAndMetricValueExtractor( ImmutableMap.builder() - .put("market", "spot") - .put("rows", 4L) - .put("index", 400.0D) - .put("addRowsIndexConstant", 405.0D) - .put("uniques", 1.0002442201269182D) - .put("maxIndex", 100.0) - .put("minIndex", 100.0) - .build() + .put("market", "spot") + .put("rows", 4L) + .put("index", 400.0D) + .put("addRowsIndexConstant", 405.0D) + .put("uniques", 1.0002442201269182D) + .put("maxIndex", 100.0) + .put("minIndex", 100.0) + .build() ), new DimensionAndMetricValueExtractor( ImmutableMap.builder() - .put("market", "") - .put("rows", 2L) - .put("index", 200.0D) - .put("addRowsIndexConstant", 203.0D) - .put("uniques", 0.0) - .put("maxIndex", 100.0D) - .put("minIndex", 100.0D) - .build() + .put("market", "") + .put("rows", 2L) + .put("index", 200.0D) + .put("addRowsIndexConstant", 203.0D) + .put("uniques", 0.0) + .put("maxIndex", 100.0D) + .put("minIndex", 100.0D) + .build() ), new DimensionAndMetricValueExtractor( ImmutableMap.builder() - .put("market", "total_market") - .put("rows", 2L) - .put("index", 200.0D) - .put("addRowsIndexConstant", 203.0D) - .put("uniques", 1.0002442201269182D) - .put("maxIndex", 100.0D) - .put("minIndex", 100.0D) - .build() + .put("market", "total_market") + .put("rows", 2L) + .put("index", 200.0D) + .put("addRowsIndexConstant", 203.0D) + .put("uniques", 1.0002442201269182D) + .put("maxIndex", 100.0D) + .put("minIndex", 100.0D) + .build() ) ) ) From 800e79514426e283942664de1fe49f64e0d092e6 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 19 Sep 2017 04:38:44 +0530 Subject: [PATCH 13/50] Fix Null Aggregate Serde and handle nulls in post aggs. --- .../TimestampAggregatorFactory.java | 13 +- .../query/aggregation/AggregatorFactory.java | 6 +- .../DoubleMaxAggregatorFactory.java | 10 +- .../DoubleMinAggregatorFactory.java | 10 +- .../DoubleSumAggregatorFactory.java | 10 +- .../FloatMaxAggregatorFactory.java | 10 +- .../FloatMinAggregatorFactory.java | 10 +- .../FloatSumAggregatorFactory.java | 10 +- .../aggregation/LongMaxAggregatorFactory.java | 10 +- .../aggregation/LongMinAggregatorFactory.java | 10 +- .../query/aggregation/NullableAggregator.java | 3 + .../aggregation/NullableBufferAggregator.java | 2 + .../query/aggregation/PostAggregator.java | 2 + .../first/DoubleFirstAggregatorFactory.java | 13 +- .../first/FloatFirstAggregatorFactory.java | 12 +- .../first/LongFirstAggregatorFactory.java | 13 +- .../last/DoubleLastAggregatorFactory.java | 12 +- .../last/FloatLastAggregatorFactory.java | 13 +- .../last/LongLastAggregatorFactory.java | 12 +- .../post/ArithmeticPostAggregator.java | 17 +- .../post/DoubleGreatestPostAggregator.java | 25 ++- .../post/DoubleLeastPostAggregator.java | 25 ++- .../post/LongGreatestPostAggregator.java | 26 ++- .../post/LongLeastPostAggregator.java | 26 ++- .../druid/segment/DoubleColumnSerializer.java | 3 +- .../segment/DoubleDimensionMergerV9.java | 22 +-- .../druid/segment/FloatColumnSerializer.java | 3 +- .../druid/segment/FloatDimensionMergerV9.java | 22 +-- .../segment/GenericColumnSerializer.java | 3 +- .../java/io/druid/segment/IndexMergerV9.java | 89 ++++++++++- .../druid/segment/LongColumnSerializer.java | 3 +- .../druid/segment/LongDimensionMergerV9.java | 22 +-- .../io/druid/segment/NullHandlingHelper.java | 2 +- .../QueryableIndexIndexableAdapter.java | 9 +- .../druid/segment/column/GenericColumn.java | 3 + .../column/IndexedDoublesGenericColumn.java | 6 + .../column/IndexedFloatsGenericColumn.java | 6 + .../column/IndexedLongsGenericColumn.java | 6 + .../io/druid/query/SchemaEvolutionTest.java | 11 +- .../aggregation/DoubleMaxAggregationTest.java | 7 +- .../aggregation/DoubleMinAggregationTest.java | 7 +- .../aggregation/FilteredAggregatorTest.java | 7 +- .../aggregation/LongMaxAggregationTest.java | 7 +- .../aggregation/LongMinAggregationTest.java | 7 +- .../first/DoubleFirstAggregationTest.java | 13 +- .../first/FloatFirstAggregationTest.java | 13 +- .../first/LongFirstAggregationTest.java | 13 +- .../last/DoubleLastAggregationTest.java | 11 +- .../last/FloatLastAggregationTest.java | 13 +- .../last/LongLastAggregationTest.java | 13 +- .../query/groupby/GroupByQueryRunnerTest.java | 142 +++++++++++++++-- .../timeseries/TimeseriesQueryRunnerTest.java | 149 +++++++++++------- .../druid/segment/SchemalessTestFullTest.java | 78 ++++----- .../segment/SchemalessTestSimpleTest.java | 2 +- .../java/io/druid/segment/TestHelper.java | 14 ++ .../druid/sql/calcite/CalciteQueryTest.java | 48 +++--- 56 files changed, 740 insertions(+), 314 deletions(-) diff --git a/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java b/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java index 7b9fe52f86ad..851b773c2cb8 100644 --- a/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java +++ b/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java @@ -29,6 +29,7 @@ import io.druid.segment.ObjectColumnSelector; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.sql.Timestamp; import java.util.Arrays; @@ -83,8 +84,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return TimestampAggregator.combineValues(comparator, lhs, rhs); } @@ -159,9 +167,10 @@ public Object deserialize(Object object) } @Override + @Nullable public Object finalizeComputation(Object object) { - return DateTimes.utc((long) object); + return object == null ? null : DateTimes.utc((long) object); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java index f4e67c34e26b..c8ae7fface56 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java @@ -58,7 +58,8 @@ public abstract class AggregatorFactory implements Cacheable * * @return an object representing the combination of lhs and rhs, this can be a new object or a mutation of the inputs */ - public abstract Object combine(Object lhs, Object rhs); + @Nullable + public abstract Object combine(@Nullable Object lhs, @Nullable Object rhs); /** * Creates an AggregateCombiner to fold rollup aggregation results from serveral "rows" of different indexes during @@ -120,7 +121,8 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre * * @return the finalized value that should be returned for the initial query */ - public abstract Object finalizeComputation(Object object); + @Nullable + public abstract Object finalizeComputation(@Nullable Object object); public abstract String getName(); diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index 972bbece60f4..058d6811a0fe 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -28,6 +28,7 @@ import io.druid.segment.DoubleColumnSelector; import io.druid.segment.NullHandlingHelper; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -74,8 +75,15 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleMaxAggregator.combineValues(lhs, rhs); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index a57d3d423858..23c9752e1cd8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -28,6 +28,7 @@ import io.druid.segment.DoubleColumnSelector; import io.druid.segment.NullHandlingHelper; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -74,8 +75,15 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleMinAggregator.combineValues(lhs, rhs); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index 97ad1ba24449..ff50771e3c3d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -28,6 +28,7 @@ import io.druid.segment.DoubleColumnSelector; import io.druid.segment.NullHandlingHelper; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -75,8 +76,15 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleSumAggregator.combineValues(lhs, rhs); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java index 8ef29d6d0357..96acd4cd99d6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -28,6 +28,7 @@ import io.druid.segment.FloatColumnSelector; import io.druid.segment.NullHandlingHelper; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -70,8 +71,15 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatMaxAggregator.combineValues(finalizeComputation(lhs), finalizeComputation(rhs)); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java index 9d84dbb4d80a..2b61e0ea3583 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java @@ -28,6 +28,7 @@ import io.druid.segment.FloatColumnSelector; import io.druid.segment.NullHandlingHelper; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -70,8 +71,15 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatMinAggregator.combineValues(lhs, rhs); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java index 451b7124bc28..4137efd29001 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java @@ -28,6 +28,7 @@ import io.druid.segment.FloatColumnSelector; import io.druid.segment.NullHandlingHelper; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -70,8 +71,15 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatSumAggregator.combineValues(lhs, rhs); } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index fa82619c6d95..56abdf8afb6b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -32,6 +32,7 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.NullHandlingHelper; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -102,8 +103,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return LongMaxAggregator.combineValues(lhs, rhs); } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 1c735304c233..c632491f7d6f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -33,6 +33,7 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.NullHandlingHelper; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -105,8 +106,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return LongMinAggregator.combineValues(lhs, rhs); } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index b13e442c732f..27a3a0e7fce5 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -21,6 +21,8 @@ import io.druid.segment.ColumnValueSelector; +import javax.annotation.Nullable; + /** * The result of a NullableAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. @@ -59,6 +61,7 @@ public void reset() } @Override + @Nullable public Object get() { if (isNullResult) { diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 6978e6d56e8a..724da9b8ea16 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -21,6 +21,7 @@ import io.druid.segment.ColumnValueSelector; +import javax.annotation.Nullable; import java.nio.ByteBuffer; /** @@ -64,6 +65,7 @@ public void aggregate(ByteBuffer buf, int position) } @Override + @Nullable public Object get(ByteBuffer buf, int position) { if (buf.get(position) == IS_NULL_BYTE) { diff --git a/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java index 171d4f39ef18..3d789d1d3dab 100644 --- a/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java @@ -22,6 +22,7 @@ import io.druid.guice.annotations.ExtensionPoint; import io.druid.java.util.common.Cacheable; +import javax.annotation.Nullable; import java.util.Comparator; import java.util.Map; import java.util.Set; @@ -36,6 +37,7 @@ public interface PostAggregator extends Cacheable Comparator getComparator(); + @Nullable Object compute(Map combinedAggregators); String getName(); diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index d66b8fd1585a..461e0331e90f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -40,6 +40,7 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -104,8 +105,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; } @@ -189,9 +197,10 @@ public Object deserialize(Object object) } @Override + @Nullable public Object finalizeComputation(Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index a85f019f08e7..59bf363096ae 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -40,6 +40,7 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -103,8 +104,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; } @@ -190,7 +198,7 @@ public Object deserialize(Object object) @Override public Object finalizeComputation(Object object) { - return ((SerializablePair) object).rhs; + return object == null ? object : ((SerializablePair) object).rhs; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 9d927ff85ccb..36cb104062d0 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -39,6 +39,7 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -96,8 +97,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (lhs == null) { + return rhs; + } + if (rhs == null) { + return lhs; + } return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; } @@ -181,9 +189,10 @@ public Object deserialize(Object object) } @Override + @Nullable public Object finalizeComputation(Object object) { - return ((SerializablePair) object).rhs; + return object == null ? object : ((SerializablePair) object).rhs; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 5c8fd15835f2..3c571fc5f7c3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -41,6 +41,7 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -94,8 +95,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; } @@ -181,7 +189,7 @@ public Object deserialize(Object object) @Override public Object finalizeComputation(Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 341029695948..705d371d4611 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -41,6 +41,7 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -95,8 +96,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; } @@ -180,9 +188,10 @@ public Object deserialize(Object object) } @Override + @Nullable public Object finalizeComputation(Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 673d538da00f..5407cde15f1d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -40,6 +40,7 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -92,8 +93,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; } @@ -179,7 +187,7 @@ public Object deserialize(Object object) @Override public Object finalizeComputation(Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java index 6f6b7f003aac..1e4e3af0bcb4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -29,6 +29,7 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; +import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -110,11 +111,21 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - double retVal = 0.0; + Double retVal = NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null; if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); + Number nextVal = (Number) fieldsIter.next().compute(values); + if (nextVal == null) { + // As per SQL standard if any of the value is null, arithmetic operators will return null. + return null; + } + retVal = nextVal.doubleValue(); while (fieldsIter.hasNext()) { - retVal = op.compute(retVal, ((Number) fieldsIter.next().compute(values)).doubleValue()); + nextVal = (Number) fieldsIter.next().compute(values); + if (nextVal == null) { + // As per SQL standard if any of the value is null, arithmetic operators will return null. + return null; + } + retVal = op.compute(retVal, (nextVal).doubleValue()); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index 584d022675b5..b85b236008ec 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -23,10 +23,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; +import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -37,14 +39,7 @@ public class DoubleGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return ((Double) o).compareTo((Double) o1); - } - }; + private static final Comparator COMPARATOR = Comparators.naturalNullsFirst(); private final String name; private final List fields; @@ -81,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - double retVal = Double.NEGATIVE_INFINITY; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); - while (fieldsIter.hasNext()) { - double other = ((Number) fieldsIter.next().compute(values)).doubleValue(); - if (other > retVal) { - retVal = other; - } + Double retVal = NullHandlingHelper.useDefaultValuesForNull() ? Double.NEGATIVE_INFINITY : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.doubleValue(), retVal) > 0) { + retVal = nextVal.doubleValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java index f66c81d9e769..3ffa723244f9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -22,11 +22,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; +import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -37,14 +39,7 @@ public class DoubleLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return ((Double) o).compareTo((Double) o1); - } - }; + private static final Comparator COMPARATOR = Ordering.natural().nullsLast(); private final String name; private final List fields; @@ -81,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - double retVal = Double.POSITIVE_INFINITY; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); - while (fieldsIter.hasNext()) { - double other = ((Number) fieldsIter.next().compute(values)).doubleValue(); - if (other < retVal) { - retVal = other; - } + Double retVal = NullHandlingHelper.useDefaultValuesForNull() ? Double.POSITIVE_INFINITY : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.doubleValue(), retVal) < 0) { + retVal = nextVal.doubleValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java index 695568c4dad4..e4fadad31de1 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -23,11 +23,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; -import com.google.common.primitives.Longs; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; +import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -38,14 +39,7 @@ public class LongGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return Longs.compare(((Number) o).longValue(), ((Number) o1).longValue()); - } - }; + private static final Comparator COMPARATOR = Comparators.naturalNullsFirst(); private final String name; private final List fields; @@ -82,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - long retVal = Long.MIN_VALUE; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).longValue(); - while (fieldsIter.hasNext()) { - long other = ((Number) fieldsIter.next().compute(values)).longValue(); - if (other > retVal) { - retVal = other; - } + Long retVal = NullHandlingHelper.useDefaultValuesForNull() ? Long.MIN_VALUE : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.longValue(), retVal) > 0) { + retVal = nextVal.longValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java index f14bcd8228f3..03a0b18e6bc0 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -22,12 +22,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import com.google.common.primitives.Longs; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; +import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -38,14 +39,7 @@ public class LongLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return Longs.compare(((Number) o).longValue(), ((Number) o1).longValue()); - } - }; + private static final Comparator COMPARATOR = Ordering.natural().nullsLast(); private final String name; private final List fields; @@ -82,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - long retVal = Long.MAX_VALUE; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).longValue(); - while (fieldsIter.hasNext()) { - long other = ((Number) fieldsIter.next().compute(values)).longValue(); - if (other < retVal) { - retVal = other; - } + Long retVal = NullHandlingHelper.useDefaultValuesForNull() ? Long.MAX_VALUE : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.longValue(), retVal) < 0) { + retVal = nextVal.longValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java index 0e74e23990ac..0531804372e1 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java @@ -27,6 +27,7 @@ import io.druid.segment.data.DoubleSupplierSerializer; import io.druid.segment.data.IOPeon; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -74,7 +75,7 @@ public void open() throws IOException } @Override - public void serialize(Object obj) throws IOException + public void serialize(@Nullable Object obj) throws IOException { double val = (obj == null) ? 0 : ((Number) obj).doubleValue(); writer.add(val); diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java index 3bf90855023f..76a529f45b1c 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java @@ -21,7 +21,6 @@ import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.MutableBitmap; -import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; @@ -31,7 +30,6 @@ import io.druid.segment.data.IOPeon; import io.druid.segment.serde.DoubleGenericColumnPartSerdeV2; -import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.IntBuffer; @@ -126,20 +124,12 @@ public void processMergedRow(Double rowValues) throws IOException @Override public void writeIndexes(List segmentRowNumConversions, Closer closer) throws IOException { - boolean hasNullValues = !nullRowsBitmap.isEmpty(); - if (hasNullValues) { - nullValueBitmapWriter = new ByteBufferWriter<>( - ioPeon, - StringUtils.format("%s.nullBitmap", dimensionName), - indexSpec.getBitmapSerdeFactory().getObjectStrategy() - ); - try (Closeable bitmapWriter = nullValueBitmapWriter) { - nullValueBitmapWriter.open(); - nullValueBitmapWriter.write(indexSpec.getBitmapSerdeFactory() - .getBitmapFactory() - .makeImmutableBitmap(nullRowsBitmap)); - } - } + this.nullValueBitmapWriter = IndexMergerV9.createNullRowsBitmapWriter( + ioPeon, + dimensionName, + nullRowsBitmap, + indexSpec + ); } @Override diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java index 15560f14d855..110b6e77737e 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java @@ -26,6 +26,7 @@ import io.druid.segment.data.FloatSupplierSerializer; import io.druid.segment.data.IOPeon; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -73,7 +74,7 @@ public void open() throws IOException } @Override - public void serialize(Object obj) throws IOException + public void serialize(@Nullable Object obj) throws IOException { float val = (obj == null) ? 0 : ((Number) obj).floatValue(); writer.add(val); diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java index 31fb92121079..facd0c939fe7 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java @@ -21,7 +21,6 @@ import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.MutableBitmap; -import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; @@ -31,7 +30,6 @@ import io.druid.segment.data.IOPeon; import io.druid.segment.serde.FloatGenericColumnPartSerdeV2; -import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.IntBuffer; @@ -108,20 +106,12 @@ public void processMergedRow(Float rowValues) throws IOException @Override public void writeIndexes(List segmentRowNumConversions, Closer closer) throws IOException { - boolean hasNullValues = !nullRowsBitmap.isEmpty(); - if (hasNullValues) { - nullValueBitmapWriter = new ByteBufferWriter<>( - ioPeon, - StringUtils.format("%s.nullBitmap", dimensionName), - indexSpec.getBitmapSerdeFactory().getObjectStrategy() - ); - try (Closeable bitmapWriter = nullValueBitmapWriter) { - nullValueBitmapWriter.open(); - nullValueBitmapWriter.write(indexSpec.getBitmapSerdeFactory() - .getBitmapFactory() - .makeImmutableBitmap(nullRowsBitmap)); - } - } + this.nullValueBitmapWriter = IndexMergerV9.createNullRowsBitmapWriter( + ioPeon, + dimensionName, + nullRowsBitmap, + indexSpec + ); } @Override diff --git a/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java b/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java index 474d7dafe524..6f56c0c59be0 100644 --- a/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java @@ -22,6 +22,7 @@ import io.druid.guice.annotations.ExtensionPoint; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; import java.nio.channels.WritableByteChannel; @@ -31,7 +32,7 @@ public interface GenericColumnSerializer extends Closeable { public void open() throws IOException; - public void serialize(Object obj) throws IOException; + public void serialize(@Nullable Object obj) throws IOException; public long getSerializedSize(); diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index 694c657db390..4b90fb268f05 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -34,11 +34,14 @@ import com.google.common.primitives.Longs; import com.google.inject.Inject; import io.druid.collections.CombiningIterable; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.io.ZeroCopyByteArrayOutputStream; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.JodaUtils; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.Comparators; import io.druid.java.util.common.guava.FunctionalIterable; import io.druid.java.util.common.guava.MergeIterable; @@ -52,6 +55,7 @@ import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.GenericIndexed; @@ -225,17 +229,31 @@ public Metadata apply(IndexableAdapter input) final ArrayList metWriters = setupMetricsWriters( ioPeon, mergedMetrics, metricsValueTypes, metricTypeNames, indexSpec ); + + List metricNullRowsBitmap = setupMetricNullRowsBitmaps(metWriters.size(), indexSpec); + final List rowNumConversions = Lists.newArrayListWithCapacity(adapters.size()); mergeIndexesAndWriteColumns( - adapters, progress, theRows, timeWriter, metWriters, rowNumConversions, mergers + adapters, progress, theRows, timeWriter, metWriters, rowNumConversions, mergers, + metricNullRowsBitmap ); /************ Create Inverted Indexes and Finalize Build Columns *************/ final String section = "build inverted index and columns"; progress.startSection(section); makeTimeColumn(v9Smoosher, progress, timeWriter); - makeMetricsColumns(v9Smoosher, progress, mergedMetrics, metricsValueTypes, metricTypeNames, metWriters); + makeMetricsColumns( + v9Smoosher, + progress, + mergedMetrics, + metricsValueTypes, + metricTypeNames, + metWriters, + indexSpec, + metricNullRowsBitmap, + ioPeon + ); for (int i = 0; i < mergedDimensions.size(); i++) { DimensionMergerV9 merger = (DimensionMergerV9) mergers.get(i); @@ -269,6 +287,41 @@ public Metadata apply(IndexableAdapter input) } } + private List setupMetricNullRowsBitmaps(int size, IndexSpec indexSpec) + { + List rv = Lists.newArrayListWithCapacity(size); + for (int i = 0; i < size; i++) { + rv.add(indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap()); + } + return rv; + } + + public static ByteBufferWriter createNullRowsBitmapWriter( + IOPeon ioPeon, + String columnName, + MutableBitmap nullRowsBitmap, + IndexSpec indexSpec + ) + throws IOException + { + if (nullRowsBitmap.isEmpty()) { + return null; + } + ; + ByteBufferWriter nullValueBitmapWriter = new ByteBufferWriter<>( + ioPeon, + StringUtils.format("%s.nullBitmap", columnName), + indexSpec.getBitmapSerdeFactory().getObjectStrategy() + ); + try (Closeable bitmapWriter = nullValueBitmapWriter) { + nullValueBitmapWriter.open(); + nullValueBitmapWriter.write(indexSpec.getBitmapSerdeFactory() + .getBitmapFactory() + .makeImmutableBitmap(nullRowsBitmap)); + } + return nullValueBitmapWriter; + } + private void makeMetadataBinary( final FileSmoosher v9Smoosher, final ProgressIndicator progress, @@ -350,7 +403,10 @@ private void makeMetricsColumns( final List mergedMetrics, final Map metricsValueTypes, final Map metricTypeNames, - final List metWriters + final List metWriters, + final IndexSpec indexSpec, + final List metricNullRowsBitmap, + IOPeon ioPeon ) throws IOException { final String section = "make metric columns"; @@ -373,6 +429,12 @@ private void makeMetricsColumns( .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((LongColumnSerializer) writer) + .withNullValueBitmapWriter(createNullRowsBitmapWriter( + ioPeon, + metric, + metricNullRowsBitmap.get(i), + indexSpec + )) .build() ); break; @@ -383,6 +445,12 @@ private void makeMetricsColumns( .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((FloatColumnSerializer) writer) + .withNullValueBitmapWriter(createNullRowsBitmapWriter( + ioPeon, + metric, + metricNullRowsBitmap.get(i), + indexSpec + )) .build() ); break; @@ -393,6 +461,12 @@ private void makeMetricsColumns( .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((DoubleColumnSerializer) writer) + .withNullValueBitmapWriter(createNullRowsBitmapWriter( + ioPeon, + metric, + metricNullRowsBitmap.get(i), + indexSpec + )) .build() ); break; @@ -467,7 +541,8 @@ private void mergeIndexesAndWriteColumns( final LongColumnSerializer timeWriter, final ArrayList metWriters, final List rowNumConversions, - final List mergers + final List mergers, + final List metricNullRowsBitmap ) throws IOException { final String section = "walk through and merge rows"; @@ -488,7 +563,11 @@ private void mergeIndexesAndWriteColumns( final Object[] metrics = theRow.getMetrics(); for (int i = 0; i < metrics.length; ++i) { - metWriters.get(i).serialize(metrics[i]); + Object metricVal = metrics[i]; + if (metricVal == null) { + metricNullRowsBitmap.get(i).add(rowCount); + } + metWriters.get(i).serialize(metricVal); } Object[] dims = theRow.getDims(); diff --git a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java index b31a5f504545..5f77c13f13af 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java @@ -26,6 +26,7 @@ import io.druid.segment.data.IOPeon; import io.druid.segment.data.LongSupplierSerializer; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -81,7 +82,7 @@ public void open() throws IOException } @Override - public void serialize(Object obj) throws IOException + public void serialize(@Nullable Object obj) throws IOException { long val = (obj == null) ? 0 : ((Number) obj).longValue(); writer.add(val); diff --git a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java index 5d5d7b755966..8e31079997a1 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java @@ -22,7 +22,6 @@ import com.google.common.base.Throwables; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.MutableBitmap; -import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.Closer; import io.druid.java.util.common.logger.Logger; import io.druid.segment.column.ColumnCapabilities; @@ -34,7 +33,6 @@ import io.druid.segment.data.IOPeon; import io.druid.segment.serde.LongGenericColumnPartSerdeV2; -import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.IntBuffer; @@ -113,20 +111,12 @@ public void processMergedRow(Long rowValues) throws IOException @Override public void writeIndexes(List segmentRowNumConversions, Closer closer) throws IOException { - boolean hasNullValues = !nullRowsBitmap.isEmpty(); - if (hasNullValues) { - nullValueBitmapWriter = new ByteBufferWriter<>( - ioPeon, - StringUtils.format("%s.nullBitmap", dimensionName), - indexSpec.getBitmapSerdeFactory().getObjectStrategy() - ); - try (Closeable bitmapWriter = nullValueBitmapWriter) { - nullValueBitmapWriter.open(); - nullValueBitmapWriter.write(indexSpec.getBitmapSerdeFactory() - .getBitmapFactory() - .makeImmutableBitmap(nullRowsBitmap)); - } - } + this.nullValueBitmapWriter = IndexMergerV9.createNullRowsBitmapWriter( + ioPeon, + dimensionName, + nullRowsBitmap, + indexSpec + ); } @Override diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index ec53dcbaef3d..8ffc68bed375 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -38,7 +38,7 @@ public class NullHandlingHelper // Using static injection to avoid adding JacksonInject annotations all over the code. @Inject - private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig(false); + private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig(true); public static boolean useDefaultValuesForNull() { diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java index 951929faf8dc..7b055ddee681 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java @@ -271,11 +271,14 @@ public Rowboat next() Object[] metricArray = new Object[numMetrics]; for (int i = 0; i < metricArray.length; ++i) { if (metrics[i] instanceof IndexedFloatsGenericColumn) { - metricArray[i] = ((GenericColumn) metrics[i]).getFloatSingleValueRow(currRow); + GenericColumn genericColumn = (GenericColumn) metrics[i]; + metricArray[i] = genericColumn.isNull(currRow) ? null : genericColumn.getFloatSingleValueRow(currRow); } else if (metrics[i] instanceof IndexedDoublesGenericColumn) { - metricArray[i] = ((GenericColumn) metrics[i]).getDoubleSingleValueRow(currRow); + GenericColumn genericColumn = (GenericColumn) metrics[i]; + metricArray[i] = genericColumn.isNull(currRow) ? null : genericColumn.getDoubleSingleValueRow(currRow); } else if (metrics[i] instanceof IndexedLongsGenericColumn) { - metricArray[i] = ((GenericColumn) metrics[i]).getLongSingleValueRow(currRow); + GenericColumn genericColumn = (GenericColumn) metrics[i]; + metricArray[i] = genericColumn.isNull(currRow) ? null : genericColumn.getLongSingleValueRow(currRow); } else if (metrics[i] instanceof ComplexColumn) { metricArray[i] = ((ComplexColumn) metrics[i]).getRowValue(currRow); } diff --git a/processing/src/main/java/io/druid/segment/column/GenericColumn.java b/processing/src/main/java/io/druid/segment/column/GenericColumn.java index 69b6d431f44e..66d8ce5e40c5 100644 --- a/processing/src/main/java/io/druid/segment/column/GenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/GenericColumn.java @@ -51,6 +51,9 @@ public interface GenericColumn extends HotLoopCallee, Closeable double getDoubleSingleValueRow(int rowNum); DoubleColumnSelector makeDoubleSingleValueRowSelector(ReadableOffset offset); + @CalledFromHotLoop + boolean isNull(int rowNum); + @Override void close(); } diff --git a/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java index b028e2903597..f49d0ceb24cd 100644 --- a/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java @@ -99,6 +99,12 @@ public DoubleColumnSelector makeDoubleSingleValueRowSelector(ReadableOffset offs return column.makeDoubleColumnSelector(offset, nullValueBitmap); } + @Override + public boolean isNull(int rowNum) + { + return nullValueBitmap.get(rowNum); + } + @Override public void close() { diff --git a/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java index 70fe5872d389..30c4ee4c6a45 100644 --- a/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java @@ -103,6 +103,12 @@ public DoubleColumnSelector makeDoubleSingleValueRowSelector(ReadableOffset offs return column.makeDoubleColumnSelector(offset, nullValueBitmap); } + @Override + public boolean isNull(int rowNum) + { + return nullValueBitmap.get(rowNum); + } + @Override public void close() { diff --git a/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java index 14b758dc5130..fee683cb4e69 100644 --- a/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java @@ -103,6 +103,12 @@ public DoubleColumnSelector makeDoubleSingleValueRowSelector(ReadableOffset offs return column.makeDoubleColumnSelector(offset, nullValueBitmap); } + @Override + public boolean isNull(int rowNum) + { + return nullValueBitmap.get(rowNum); + } + @Override public void close() { diff --git a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java index 71b30f927898..ca0c051354e5 100644 --- a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java +++ b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java @@ -48,8 +48,10 @@ import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; import io.druid.query.timeseries.TimeseriesResultValue; import io.druid.segment.IndexBuilder; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.TestHelper; import io.druid.segment.incremental.IncrementalIndexSchema; import org.junit.After; import org.junit.Assert; @@ -356,7 +358,14 @@ public void testNumericEvolutionFiltering() // Only nonexistent(4) Assert.assertEquals( - timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L)), + timeseriesResult(TestHelper.createExpectedMap( + "a", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null, + "b", + NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : null, + "c", + 0L + )), runQuery(query, factory, ImmutableList.of(index4)) ); diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java index f6737ac4e4bb..5a9cdbcc1f9c 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Doubles; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import org.easymock.EasyMock; import org.junit.Assert; @@ -69,7 +70,11 @@ public void testDoubleMaxAggregator() Assert.assertEquals(values[2], agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(Double.NEGATIVE_INFINITY, (Double) agg.get(), 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(Double.NEGATIVE_INFINITY, (Double) agg.get(), 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java index f5c1607bd882..353c3b1a4d28 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Doubles; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import org.easymock.EasyMock; import org.junit.Assert; @@ -69,7 +70,11 @@ public void testDoubleMinAggregator() Assert.assertEquals(values[2], agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(Double.POSITIVE_INFINITY, (Double) agg.get(), 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(Double.POSITIVE_INFINITY, (Double) agg.get(), 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index d1b16ef78f59..893fc4f8ab5a 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -47,6 +47,7 @@ import io.druid.segment.FloatColumnSelector; import io.druid.segment.IdLookup; import io.druid.segment.LongColumnSelector; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; @@ -257,9 +258,9 @@ public ColumnCapabilities getColumnCapabilities(String columnName) private void assertValues(FilteredAggregator agg, TestFloatColumnSelector selector, double... expectedVals) { - Assert.assertEquals(0.0d, agg.get()); - Assert.assertEquals(0.0d, agg.get()); - Assert.assertEquals(0.0d, agg.get()); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 0.0d : null, agg.get()); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 0.0d : null, agg.get()); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 0.0d : null, agg.get()); for (double expectedVal : expectedVals) { aggregate(selector, agg); Assert.assertEquals(expectedVal, agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java index 19b6cbc819cc..d1939c2cc14c 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Longs; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import org.easymock.EasyMock; import org.junit.Assert; @@ -69,7 +70,11 @@ public void testLongMaxAggregator() Assert.assertEquals((float) values[2], agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(Long.MIN_VALUE, agg.getLong()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(Long.MIN_VALUE, ((Long) agg.get()).longValue()); + } else { + Assert.assertNull(agg.get()); + } } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java index 265a455457ca..85edd0f1139d 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Longs; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import org.easymock.EasyMock; import org.junit.Assert; @@ -69,7 +70,11 @@ public void testLongMinAggregator() Assert.assertEquals((float) values[2], agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(Long.MAX_VALUE, agg.getLong()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(Long.MAX_VALUE, ((Long) agg.get()).longValue()); + } else { + Assert.assertNull(agg.get()); + } } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java index 8ef75b2bf51e..0b455d4363cd 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -29,6 +29,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -88,7 +89,11 @@ public void testDoubleFirstAggregator() Assert.assertEquals(doubleValues[1], agg.getDouble(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test @@ -140,7 +145,11 @@ public void testDoubleFirstCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getDouble(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java index 47bbd172d3fa..20cf21b0ad80 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -29,6 +29,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -88,7 +89,11 @@ public void testDoubleFirstAggregator() Assert.assertEquals(floats[1], agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test @@ -140,7 +145,11 @@ public void testDoubleFirstCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java index 9731e299569a..11e65bfbc318 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -28,6 +28,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -87,7 +88,11 @@ public void testLongFirstAggregator() Assert.assertEquals(longValues[3], agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test @@ -139,7 +144,11 @@ public void testLongFirstCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java index 6ff8547f6fc8..756eb1ae6cfa 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -29,6 +29,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -88,7 +89,11 @@ public void testDoubleLastAggregator() Assert.assertEquals(doubles[0], agg.getDouble(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test @@ -140,7 +145,9 @@ public void testDoubleLastCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getDouble(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java index 217467cb7a02..9fc9bf8e5cab 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -29,6 +29,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -88,7 +89,11 @@ public void testDoubleLastAggregator() Assert.assertEquals(floats[0], agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test @@ -140,7 +145,11 @@ public void testDoubleLastCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getFloat(), 0.0001); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); + } else { + Assert.assertNull(agg.get()); + } } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java index 5ca9c26053e6..0a001e033396 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java @@ -28,6 +28,7 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -87,7 +88,11 @@ public void testLongLastAggregator() Assert.assertEquals(longValues[2], agg.getFloat(), 1); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs.longValue()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs.longValue()); + } else { + Assert.assertNull(agg.get()); + } } @Test @@ -139,7 +144,11 @@ public void testLongLastCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getFloat(), 1); agg.reset(); - Assert.assertEquals(0, ((Pair) agg.get()).rhs.longValue()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(0, ((Pair) agg.get()).rhs.longValue()); + } else { + Assert.assertNull(agg.get()); + } } @Test diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 3b208bc1540a..b17d65fdb117 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -7098,25 +7098,137 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() .setGranularity(QueryRunnerTestHelper.dayGran) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "automotive", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "business", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "entertainment", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "health", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 0L, "idx", 0L), - - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "premium", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "technology", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "travel", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "automotive", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "business", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "entertainment", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "health", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 0L, "idx", 0L) + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "premium", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "technology", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "travel", + "rows", + 0L, + "idx", + NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + ) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index 8d7b57056fb8..02809c0fc89e 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.primitives.Doubles; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; @@ -59,6 +60,7 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.ordering.StringComparators; import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; @@ -145,16 +147,15 @@ public void testEmptyTimeseries() ) .descending(descending) .build(); - + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandlingHelper.useDefaultValuesForNull() ? 0D : null); + resultMap.put("first", NullHandlingHelper.useDefaultValuesForNull() ? 0D : null); List> expectedResults = ImmutableList.of( new Result<>( DateTimes.of("2020-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0D, - "first", 0D - ) + resultMap ) ) ); @@ -215,24 +216,61 @@ public void testFullOnTimeseries() QueryRunnerTestHelper.skippedDay.equals(current) ? 0L : 13L, value.getLongMetric("rows").longValue() ); - Assert.assertEquals( - result.toString(), - Doubles.tryParse(expectedIndex[count]).doubleValue(), - value.getDoubleMetric("index").doubleValue(), - value.getDoubleMetric("index").doubleValue() * 1e-6 - ); - Assert.assertEquals( - result.toString(), - new Double(expectedIndex[count]) + - (QueryRunnerTestHelper.skippedDay.equals(current) ? 0L : 13L) + 1L, - value.getDoubleMetric("addRowsIndexConstant"), - value.getDoubleMetric("addRowsIndexConstant") * 1e-6 - ); - Assert.assertEquals( - value.getDoubleMetric("uniques"), - QueryRunnerTestHelper.skippedDay.equals(current) ? 0.0d : 9.0d, - 0.02 - ); + + if (!QueryRunnerTestHelper.skippedDay.equals(current)) { + Assert.assertEquals( + result.toString(), + Doubles.tryParse(expectedIndex[count]).doubleValue(), + value.getDoubleMetric("index").doubleValue(), + value.getDoubleMetric("index").doubleValue() * 1e-6 + ); + Assert.assertEquals( + result.toString(), + new Double(expectedIndex[count]) + + 13L + 1L, + value.getDoubleMetric("addRowsIndexConstant"), + value.getDoubleMetric("addRowsIndexConstant") * 1e-6 + ); + Assert.assertEquals( + value.getDoubleMetric("uniques"), + 9.0d, + 0.02 + ); + } else { + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals( + result.toString(), + 0.0D, + value.getDoubleMetric("index").doubleValue(), + value.getDoubleMetric("index").doubleValue() * 1e-6 + ); + Assert.assertEquals( + result.toString(), + new Double(expectedIndex[count]) + 1L, + value.getDoubleMetric("addRowsIndexConstant"), + value.getDoubleMetric("addRowsIndexConstant") * 1e-6 + ); + Assert.assertEquals( + 0.0D, + value.getDoubleMetric("uniques"), + 0.02 + ); + } else { + Assert.assertNull( + result.toString(), + value.getDoubleMetric("index") + ); + Assert.assertNull( + result.toString(), + value.getDoubleMetric("addRowsIndexConstant") + ); + Assert.assertEquals( + value.getDoubleMetric("uniques"), + 0.0d, + 0.02 + ); + } + } lastResult = result; ++count; @@ -666,12 +704,15 @@ public void testTimeseriesQueryZeroFilling() final Iterable iterable = Granularities.HOUR.getIterable( new Interval(DateTimes.of("2011-04-14T01"), DateTimes.of("2011-04-15")) ); + Map noRowsResult = Maps.newHashMap(); + noRowsResult.put("rows", 0L); + noRowsResult.put("idx", NullHandlingHelper.useDefaultValuesForNull() ? 0L : null); for (Interval interval : iterable) { lotsOfZeroes.add( new Result<>( interval.getStart(), new TimeseriesResultValue( - ImmutableMap.of("rows", 0L, "idx", 0L) + noRowsResult ) ) ); @@ -1495,27 +1536,23 @@ public void testTimeseriesWithFilterOnNonExistentDimension() .descending(descending) .build(); + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : null); + resultMap.put("addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 1.0 : null); + resultMap.put("uniques", 0.0); + List> expectedResults = Arrays.asList( new Result<>( DateTimes.of("2011-04-01"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ), new Result<>( DateTimes.of("2011-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ) ); @@ -1656,28 +1693,23 @@ public void testTimeseriesWithNonExistentFilter() .addRowsIndexConstant)) .descending(descending) .build(); + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : null); + resultMap.put("addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 1.0 : null); + resultMap.put("uniques", 0.0); List> expectedResults = Arrays.asList( new Result<>( DateTimes.of("2011-04-01"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ), new Result<>( DateTimes.of("2011-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ) ); @@ -1716,28 +1748,23 @@ public void testTimeseriesWithNonExistentFilterAndMultiDim() .addRowsIndexConstant)) .descending(descending) .build(); + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : null); + resultMap.put("addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 1.0 : null); + resultMap.put("uniques", 0.0); List> expectedResults = Arrays.asList( new Result<>( DateTimes.of("2011-04-01"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ), new Result<>( DateTimes.of("2011-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ) ); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java index ed535a769dda..e4deaaf77729 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java @@ -45,9 +45,9 @@ import io.druid.query.aggregation.post.ArithmeticPostAggregator; import io.druid.query.aggregation.post.ConstantPostAggregator; import io.druid.query.aggregation.post.FieldAccessPostAggregator; -import io.druid.query.search.SearchResultValue; import io.druid.query.search.SearchHit; import io.druid.query.search.SearchQuery; +import io.druid.query.search.SearchResultValue; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.spec.QuerySegmentSpec; import io.druid.query.timeboundary.TimeBoundaryQuery; @@ -379,13 +379,13 @@ public void testNonIntersectingSchemas() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 2L) - .put("index", 100.0D) - .put("addRowsIndexConstant", 103.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", 0.0D) - .build() + .put("rows", 2L) + .put("index", 100.0D) + .put("addRowsIndexConstant", 103.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : 100.0D) + .build() ) ) ); @@ -726,13 +726,13 @@ public void testValueAndEmptySchemas() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 2L) - .put("index", 100.0D) - .put("addRowsIndexConstant", 103.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", 0.0D) - .build() + .put("rows", 2L) + .put("index", 100.0D) + .put("addRowsIndexConstant", 103.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : 100.0D) + .build() ) ) ); @@ -844,19 +844,19 @@ public void testValueAndEmptySchemas() @Test public void testEmptySchemas() { + List> expectedTimeseriesResults = Arrays.asList( new Result<>( DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( - ImmutableMap.builder() - .put("rows", 1L) - .put("index", 0.0D) - .put("addRowsIndexConstant", 2.0D) - .put("uniques", 0.0D) - .put("maxIndex", 0.0D) - .put("minIndex", 0.0D) - .build() - ) + TestHelper.createExpectedMap( + "rows", 1L, + "index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null, + "addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 2.0D : null, + "uniques", 0.0D, + "maxIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null, + "minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null + )) ) ); @@ -864,14 +864,14 @@ public void testEmptySchemas() new Result<>( DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( - ImmutableMap.builder() - .put("rows", 0L) - .put("index", 0.0D) - .put("addRowsIndexConstant", 1.0D) - .put("uniques", 0.0D) - .put("maxIndex", Double.NEGATIVE_INFINITY) - .put("minIndex", Double.POSITIVE_INFINITY) - .build() + TestHelper.createExpectedMap( + "rows", 0L, + "index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null, + "addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 1.0D : null, + "uniques", 0.0D, + "maxIndex", NullHandlingHelper.useDefaultValuesForNull() ? Double.NEGATIVE_INFINITY : null, + "minIndex", NullHandlingHelper.useDefaultValuesForNull() ? Double.POSITIVE_INFINITY : null + ) ) ) ); @@ -1165,13 +1165,13 @@ public void testDifferentMetrics() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 11L) - .put("index", 900.0D) - .put("addRowsIndexConstant", 912.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", 0.0D) - .build() + .put("rows", 11L) + .put("index", 900.0D) + .put("addRowsIndexConstant", 912.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : 100.0D) + .build() ) ) ); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java index 89837fab7323..56f242223e5a 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java @@ -161,7 +161,7 @@ public void testFullOnTimeseries() .put("addRowsIndexConstant", 912.0) .put("uniques", 2.000977198748901D) .put("maxIndex", 100.0) - .put("minIndex", 0.0) + .put("minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : 100.0) .build() ) ) diff --git a/processing/src/test/java/io/druid/segment/TestHelper.java b/processing/src/test/java/io/druid/segment/TestHelper.java index 6a3db8e24bb7..1dcb914b98c9 100644 --- a/processing/src/test/java/io/druid/segment/TestHelper.java +++ b/processing/src/test/java/io/druid/segment/TestHelper.java @@ -21,7 +21,9 @@ import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; import io.druid.jackson.DefaultObjectMapper; @@ -338,4 +340,16 @@ private static void assertRow(String msg, Row expected, Row actual) } } } + + + public static Map createExpectedMap(Object... vals) + { + Preconditions.checkArgument(vals.length % 2 == 0); + + Map theVals = Maps.newHashMap(); + for (int i = 0; i < vals.length; i += 2) { + theVals.put(vals[i].toString(), vals[i + 1]); + } + return theVals; + } } diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 7f50eb185add..20bc4f538b4f 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -5018,7 +5018,7 @@ public void testTimeseriesLosAngelesUsingTimeFloorConnectionLosAngeles() throws public void testTimeseriesDontSkipEmptyBuckets() throws Exception { // Tests that query context parameters are passed through to the underlying query engine. - + Long defaultVal = NullHandlingHelper.useDefaultValuesForNull() ? 0L : null; testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_DONT_SKIP_EMPTY_BUCKETS, @@ -5039,29 +5039,29 @@ public void testTimeseriesDontSkipEmptyBuckets() throws Exception ), ImmutableList.builder() .add(new Object[]{1L, T("2000-01-01")}) - .add(new Object[]{0L, T("2000-01-01T01")}) - .add(new Object[]{0L, T("2000-01-01T02")}) - .add(new Object[]{0L, T("2000-01-01T03")}) - .add(new Object[]{0L, T("2000-01-01T04")}) - .add(new Object[]{0L, T("2000-01-01T05")}) - .add(new Object[]{0L, T("2000-01-01T06")}) - .add(new Object[]{0L, T("2000-01-01T07")}) - .add(new Object[]{0L, T("2000-01-01T08")}) - .add(new Object[]{0L, T("2000-01-01T09")}) - .add(new Object[]{0L, T("2000-01-01T10")}) - .add(new Object[]{0L, T("2000-01-01T11")}) - .add(new Object[]{0L, T("2000-01-01T12")}) - .add(new Object[]{0L, T("2000-01-01T13")}) - .add(new Object[]{0L, T("2000-01-01T14")}) - .add(new Object[]{0L, T("2000-01-01T15")}) - .add(new Object[]{0L, T("2000-01-01T16")}) - .add(new Object[]{0L, T("2000-01-01T17")}) - .add(new Object[]{0L, T("2000-01-01T18")}) - .add(new Object[]{0L, T("2000-01-01T19")}) - .add(new Object[]{0L, T("2000-01-01T20")}) - .add(new Object[]{0L, T("2000-01-01T21")}) - .add(new Object[]{0L, T("2000-01-01T22")}) - .add(new Object[]{0L, T("2000-01-01T23")}) + .add(new Object[]{defaultVal, T("2000-01-01T01")}) + .add(new Object[]{defaultVal, T("2000-01-01T02")}) + .add(new Object[]{defaultVal, T("2000-01-01T03")}) + .add(new Object[]{defaultVal, T("2000-01-01T04")}) + .add(new Object[]{defaultVal, T("2000-01-01T05")}) + .add(new Object[]{defaultVal, T("2000-01-01T06")}) + .add(new Object[]{defaultVal, T("2000-01-01T07")}) + .add(new Object[]{defaultVal, T("2000-01-01T08")}) + .add(new Object[]{defaultVal, T("2000-01-01T09")}) + .add(new Object[]{defaultVal, T("2000-01-01T10")}) + .add(new Object[]{defaultVal, T("2000-01-01T11")}) + .add(new Object[]{defaultVal, T("2000-01-01T12")}) + .add(new Object[]{defaultVal, T("2000-01-01T13")}) + .add(new Object[]{defaultVal, T("2000-01-01T14")}) + .add(new Object[]{defaultVal, T("2000-01-01T15")}) + .add(new Object[]{defaultVal, T("2000-01-01T16")}) + .add(new Object[]{defaultVal, T("2000-01-01T17")}) + .add(new Object[]{defaultVal, T("2000-01-01T18")}) + .add(new Object[]{defaultVal, T("2000-01-01T19")}) + .add(new Object[]{defaultVal, T("2000-01-01T20")}) + .add(new Object[]{defaultVal, T("2000-01-01T21")}) + .add(new Object[]{defaultVal, T("2000-01-01T22")}) + .add(new Object[]{defaultVal, T("2000-01-01T23")}) .build() ); } From d4b0339cc4ebd28ddf491f7f85382821fc51a570 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 27 Sep 2017 03:39:07 +0530 Subject: [PATCH 14/50] review comments --- .../main/java/io/druid/math/expr/Expr.java | 2 ++ .../java/io/druid/math/expr/Function.java | 3 ++ .../DistinctCountAggregator.java | 35 +++++++++++++----- .../DistinctCountBufferAggregator.java | 36 ++++++++++++++----- .../java/io/druid/indexer/InputRowSerde.java | 2 +- .../aggregation/LongSumAggregatorFactory.java | 6 ++++ .../post/ArithmeticPostAggregator.java | 3 +- .../ColumnSelectorBitmapIndexSelector.java | 2 +- .../druid/segment/StringDimensionHandler.java | 2 +- .../segment/data/ByteBufferSerializer.java | 6 +++- .../io/druid/segment/data/GenericIndexed.java | 3 ++ .../io/druid/segment/data/ObjectStrategy.java | 6 +++- .../serde/DoubleGenericColumnPartSerdeV2.java | 4 +-- 13 files changed, 84 insertions(+), 26 deletions(-) diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index 1630bf8e5fbf..c7d05a0bd93c 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -490,6 +490,8 @@ class BinPlusExpr extends BinaryEvalOpExprBase @Override protected ExprEval evalString(@Nullable String left, @Nullable String right) { + // Result of expression is Null if any of the argument is null. + // e.g "select null * 2 as c;" will return null as per Standard SQL spec. if (left == null || right == null) { return ExprEval.of(null); } diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index ac95075a7788..2243f3604ad9 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -904,6 +904,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) builder.append(s); } else { // Result of concatenation is null if any of the Values is null. + // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. return ExprEval.of(null); } } @@ -964,6 +965,8 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) return ExprEval.of(arg.substring(index)); } } else { + // If starting index of substring is greater then the lengh of string, the result will be a zero length string. + // e.g. 'select substring("abc", 4,5) as c;' will return an empty string return ExprEval.of(""); } } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java index 6a39a5826a28..eb8d867e47a4 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java @@ -23,6 +23,8 @@ import io.druid.query.aggregation.Aggregator; import io.druid.segment.DimensionSelector; import io.druid.segment.NullHandlingHelper; +import io.druid.segment.data.IndexedInts; +import org.roaringbitmap.IntIterator; public class DistinctCountAggregator implements Aggregator { @@ -42,11 +44,10 @@ public DistinctCountAggregator( @Override public void aggregate() { - boolean countNulls = NullHandlingHelper.useDefaultValuesForNull(); - for (final Integer index : selector.getRow()) { - if (countNulls || selector.lookupName(index) != null) { - mutableBitmap.add(index); - } + IndexedInts row = selector.getRow(); + for (int i = 0; i < row.size(); i++) { + int index = row.get(i); + mutableBitmap.add(index); } } @@ -59,13 +60,13 @@ public void reset() @Override public Object get() { - return mutableBitmap.size(); + return countValues(); } @Override public float getFloat() { - return (float) mutableBitmap.size(); + return (float) countValues(); } @Override @@ -77,12 +78,28 @@ public void close() @Override public long getLong() { - return (long) mutableBitmap.size(); + return (long) countValues(); } @Override public double getDouble() { - return (double) mutableBitmap.size(); + return (double) countValues(); + } + + private int countValues() + { + if (NullHandlingHelper.useDefaultValuesForNull()) { + return mutableBitmap.size(); + } + int retVal = 0; + IntIterator iterator = mutableBitmap.iterator(); + while (iterator.hasNext()) { + String val = selector.lookupName(iterator.next()); + if (val != null) { + retVal++; + } + } + return retVal; } } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index 63039ad9f978..8e43173f8944 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -25,8 +25,10 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionSelector; import io.druid.segment.NullHandlingHelper; +import io.druid.segment.data.IndexedInts; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import org.roaringbitmap.IntIterator; import java.nio.ByteBuffer; @@ -51,12 +53,11 @@ public void init(ByteBuffer buf, int position) @Override public void aggregate(ByteBuffer buf, int position) { - boolean countNulls = NullHandlingHelper.useDefaultValuesForNull(); MutableBitmap mutableBitmap = getMutableBitmap(buf, position); - for (final Integer index : selector.getRow()) { - if (countNulls || selector.lookupName(index) != null) { - mutableBitmap.add(index); - } + IndexedInts row = selector.getRow(); + for (int i = 0; i < row.size(); i++) { + int index = row.get(i); + mutableBitmap.add(index); } buf.putLong(position, mutableBitmap.size()); } @@ -74,25 +75,25 @@ private MutableBitmap getMutableBitmap(ByteBuffer buf, int position) @Override public Object get(ByteBuffer buf, int position) { - return buf.getLong(position); + return countValues(buf, position); } @Override public float getFloat(ByteBuffer buf, int position) { - return (float) buf.getLong(position); + return (float) countValues(buf, position); } @Override public long getLong(ByteBuffer buf, int position) { - return buf.getLong(position); + return countValues(buf, position); } @Override public double getDouble(ByteBuffer buf, int position) { - return (double) buf.getLong(position); + return (double) countValues(buf, position); } @Override @@ -106,4 +107,21 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("selector", selector); } + + private long countValues(ByteBuffer buf, int position) + { + MutableBitmap mutableBitmap = getMutableBitmap(buf, position); + if (NullHandlingHelper.useDefaultValuesForNull()) { + return mutableBitmap.size(); + } + int retVal = 0; + IntIterator iterator = mutableBitmap.iterator(); + while (iterator.hasNext()) { + String val = selector.lookupName(iterator.next()); + if (val != null) { + retVal++; + } + } + return retVal; + } } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index 98a6f691eea4..1bfcb7a893be 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -163,7 +163,7 @@ private static void writeStringArray(List values, ByteArrayDataOutput ou private static String readString(DataInput in) throws IOException { byte[] result = readBytes(in); - return StringUtils.fromUtf8(result); + return StringUtils.fromUtf8Nullable(result); } @Nullable diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 445d46e7b9e6..d3cf982d9194 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -103,6 +103,12 @@ public Comparator getComparator() @Override public Object combine(Object lhs, Object rhs) { + if (lhs == null) { + return rhs; + } + if (rhs == null) { + return lhs; + } return LongSumAggregator.combineValues(lhs, rhs); } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java index 1e4e3af0bcb4..078aa1b571bb 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -29,6 +29,7 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; +import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.NullHandlingHelper; import java.util.Comparator; @@ -111,7 +112,7 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - Double retVal = NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null; + Double retVal = NullHandlingHelper.useDefaultValuesForNull() ? DimensionHandlerUtils.ZERO_DOUBLE : null; if (fieldsIter.hasNext()) { Number nextVal = (Number) fieldsIter.next().compute(values); if (nextVal == null) { diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index 936016b574ba..0ed41dac64cc 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -209,7 +209,7 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) final Column column = index.getColumn(dimension); if (column == null || !columnSupportsFiltering(column)) { - if (value == null) { + if (NullHandlingHelper.isNullOrDefault(value)) { return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows()); } else { return bitmapFactory.makeEmptyImmutableBitmap(); diff --git a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java index 516f3c18c610..b2bc051a31d1 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java @@ -72,7 +72,7 @@ public int compareSortedEncodedKeyComponents(int[] lhs, int[] rhs) return retVal; } - public boolean isNUllRow(int[] row, Indexed encodings) + private boolean isNUllRow(int[] row, Indexed encodings) { if (row == null) { return true; diff --git a/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java b/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java index 650b9a338848..cd59e067d515 100644 --- a/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Ints; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; @@ -29,10 +30,12 @@ */ public class ByteBufferSerializer { + @Nullable public static T read(ByteBuffer buffer, ObjectStrategy strategy) { int size = buffer.getInt(); if (size < 0) { + // Size is less than 0 for null values. return null; } ByteBuffer bufferToUse = buffer.asReadOnlyBuffer(); @@ -42,10 +45,11 @@ public static T read(ByteBuffer buffer, ObjectStrategy strategy) return strategy.fromByteBuffer(bufferToUse, size); } - public static void writeToChannel(T obj, ObjectStrategy strategy, WritableByteChannel channel) + public static void writeToChannel(@Nullable T obj, ObjectStrategy strategy, WritableByteChannel channel) throws IOException { byte[] toWrite = strategy.toBytes(obj); + // For null byte array write -1 as size. 0 size will denote empty byte array. int size = toWrite == null ? -1 : toWrite.length; channel.write(ByteBuffer.allocate(Ints.BYTES).putInt(0, size)); if (size > 0) { diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 6aa2d77b8e9b..e4a432d3c81a 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -31,6 +31,7 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.NullHandlingHelper; +import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; @@ -85,9 +86,11 @@ public Class getClazz() } @Override + @Nullable public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) { if (numBytes < 0) { + // nulBytes will be -1 for null values. return null; } return NullHandlingHelper.defaultToNull(StringUtils.fromUtf8Nullable(buffer, numBytes)); diff --git a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java index dfcb60ed4434..a22927a021aa 100644 --- a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java @@ -21,6 +21,7 @@ import io.druid.guice.annotations.ExtensionPoint; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Comparator; @@ -42,6 +43,9 @@ public interface ObjectStrategy extends Comparator * @param numBytes number of bytes used to store the value, starting at buffer.position() * @return an object created from the given byte buffer representation */ + @Nullable public T fromByteBuffer(ByteBuffer buffer, int numBytes); - public byte[] toBytes(T val); + + @Nullable + public byte[] toBytes(@Nullable T val); } diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java index d9f048ef1280..d0fc57c9c33a 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -31,6 +31,7 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferSerializer; import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedDoublesIndexedSupplier; @@ -103,8 +104,7 @@ public Deserializer getDeserializer() buffer.position(initialPos + offset); final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { - int bitmapSize = buffer.getInt(); - bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBuffer(buffer, bitmapSize); + bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); From 4004029597c0fca4880962a655abf12c3264788c Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 3 Oct 2017 17:36:22 +0530 Subject: [PATCH 15/50] review comments - move numeric column null handling to single place --- .../druid/segment/DoubleColumnSerializer.java | 46 +++++++- .../segment/DoubleDimensionMergerV9.java | 28 ++--- .../druid/segment/FloatColumnSerializer.java | 46 +++++++- .../druid/segment/FloatDimensionMergerV9.java | 28 ++--- .../java/io/druid/segment/IndexMergerV9.java | 104 +++--------------- .../druid/segment/LongColumnSerializer.java | 53 ++++++++- .../druid/segment/LongDimensionMergerV9.java | 28 ++--- .../segment/column/ColumnDescriptor.java | 8 -- .../serde/DoubleGenericColumnPartSerdeV2.java | 20 +--- .../serde/FloatGenericColumnPartSerdeV2.java | 20 +--- .../serde/LongGenericColumnPartSerdeV2.java | 19 +--- 11 files changed, 168 insertions(+), 232 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java index 0531804372e1..f5e5f78ae701 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java @@ -20,8 +20,13 @@ package io.druid.segment; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.DoubleSupplierSerializer; @@ -29,6 +34,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -37,29 +43,36 @@ public class DoubleColumnSerializer implements GenericColumnSerializer public static DoubleColumnSerializer create( IOPeon ioPeon, String filenameBase, - CompressedObjectStrategy.CompressionStrategy compression + CompressedObjectStrategy.CompressionStrategy compression, + BitmapSerdeFactory bitmapSerdeFactory ) { - return new DoubleColumnSerializer(ioPeon, filenameBase, IndexIO.BYTE_ORDER, compression); + return new DoubleColumnSerializer(ioPeon, filenameBase, IndexIO.BYTE_ORDER, compression, bitmapSerdeFactory); } private final IOPeon ioPeon; private final String filenameBase; private final ByteOrder byteOrder; private final CompressedObjectStrategy.CompressionStrategy compression; + private final BitmapSerdeFactory bitmapSerdeFactory; private DoubleSupplierSerializer writer; + private ByteBufferWriter nullValueBitmapWriter; + private MutableBitmap nullRowsBitmap; + private int rowCount = 0; public DoubleColumnSerializer( IOPeon ioPeon, String filenameBase, ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression + CompressedObjectStrategy.CompressionStrategy compression, + BitmapSerdeFactory bitmapSerdeFactory ) { this.ioPeon = ioPeon; this.filenameBase = filenameBase; this.byteOrder = byteOrder; this.compression = compression; + this.bitmapSerdeFactory = bitmapSerdeFactory; } @Override @@ -72,31 +85,52 @@ public void open() throws IOException compression ); writer.open(); + nullValueBitmapWriter = new ByteBufferWriter<>( + ioPeon, + StringUtils.format("%s.nullBitmap", filenameBase), + bitmapSerdeFactory.getObjectStrategy() + ); + nullValueBitmapWriter.open(); + nullRowsBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); } @Override public void serialize(@Nullable Object obj) throws IOException { - double val = (obj == null) ? 0 : ((Number) obj).doubleValue(); - writer.add(val); + if (obj == null) { + nullRowsBitmap.add(rowCount); + writer.add(0D); + } else { + writer.add(((Number) obj).doubleValue()); + } + rowCount++; } @Override public void close() throws IOException { writer.close(); + nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); + nullValueBitmapWriter.close(); } @Override public long getSerializedSize() { - return writer.getSerializedSize(); + long bitmapSize = nullRowsBitmap.isEmpty() + ? 0L + : nullValueBitmapWriter.getSerializedSize(); + return Integer.BYTES + writer.getSerializedSize() + bitmapSize; } @Override public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { + channel.write(ByteBuffer.wrap(Ints.toByteArray((int) writer.getSerializedSize()))); writer.writeToChannel(channel, smoosher); + if (!nullRowsBitmap.isEmpty()) { + nullValueBitmapWriter.writeToChannel(channel, smoosher); + } } } diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java index 76a529f45b1c..f6ebd8acefea 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java @@ -19,13 +19,10 @@ package io.druid.segment; -import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.IOPeon; import io.druid.segment.serde.DoubleGenericColumnPartSerdeV2; @@ -44,10 +41,6 @@ public class DoubleDimensionMergerV9 implements DimensionMergerV9 protected final File outDir; protected IOPeon ioPeon; private DoubleColumnSerializer serializer; - private MutableBitmap nullRowsBitmap; - private int rowCount = 0; - private ByteBufferWriter nullValueBitmapWriter; - public DoubleDimensionMergerV9( String dimensionName, @@ -64,7 +57,6 @@ public DoubleDimensionMergerV9( this.outDir = outDir; this.ioPeon = ioPeon; this.progress = progress; - this.nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); try { setupEncodedValueWriter(); @@ -77,7 +69,12 @@ public DoubleDimensionMergerV9( protected void setupEncodedValueWriter() throws IOException { final CompressedObjectStrategy.CompressionStrategy metCompression = indexSpec.getMetricCompression(); - this.serializer = DoubleColumnSerializer.create(ioPeon, dimensionName, metCompression); + this.serializer = DoubleColumnSerializer.create( + ioPeon, + dimensionName, + metCompression, + indexSpec.getBitmapSerdeFactory() + ); serializer.open(); } @@ -87,13 +84,11 @@ public ColumnDescriptor makeColumnDescriptor() throws IOException serializer.close(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.DOUBLE); - builder.setHasNullValues(!nullRowsBitmap.isEmpty()); builder.addSerde( DoubleGenericColumnPartSerdeV2.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate(serializer) .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) - .withNullValueBitmapWriter(nullValueBitmapWriter) .build() ); return builder.build(); @@ -114,22 +109,13 @@ public Double convertSegmentRowValuesToMergedRowValues(Double segmentRow, int se @Override public void processMergedRow(Double rowValues) throws IOException { - if (rowValues == null) { - nullRowsBitmap.add(rowCount); - } serializer.serialize(rowValues); - rowCount++; } @Override public void writeIndexes(List segmentRowNumConversions, Closer closer) throws IOException { - this.nullValueBitmapWriter = IndexMergerV9.createNullRowsBitmapWriter( - ioPeon, - dimensionName, - nullRowsBitmap, - indexSpec - ); + // doubless have no indices to write } @Override diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java index 110b6e77737e..3dae3109b717 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java @@ -19,8 +19,13 @@ package io.druid.segment; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.FloatSupplierSerializer; @@ -28,6 +33,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -36,29 +42,36 @@ public class FloatColumnSerializer implements GenericColumnSerializer public static FloatColumnSerializer create( IOPeon ioPeon, String filenameBase, - CompressedObjectStrategy.CompressionStrategy compression + CompressedObjectStrategy.CompressionStrategy compression, + BitmapSerdeFactory bitmapSerdeFactory ) { - return new FloatColumnSerializer(ioPeon, filenameBase, IndexIO.BYTE_ORDER, compression); + return new FloatColumnSerializer(ioPeon, filenameBase, IndexIO.BYTE_ORDER, compression, bitmapSerdeFactory); } private final IOPeon ioPeon; private final String filenameBase; private final ByteOrder byteOrder; private final CompressedObjectStrategy.CompressionStrategy compression; + private final BitmapSerdeFactory bitmapSerdeFactory; private FloatSupplierSerializer writer; + private ByteBufferWriter nullValueBitmapWriter; + private MutableBitmap nullRowsBitmap; + private int rowCount = 0; public FloatColumnSerializer( IOPeon ioPeon, String filenameBase, ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression + CompressedObjectStrategy.CompressionStrategy compression, + BitmapSerdeFactory bitmapSerdeFactory ) { this.ioPeon = ioPeon; this.filenameBase = filenameBase; this.byteOrder = byteOrder; this.compression = compression; + this.bitmapSerdeFactory = bitmapSerdeFactory; } @Override @@ -71,31 +84,52 @@ public void open() throws IOException compression ); writer.open(); + nullValueBitmapWriter = new ByteBufferWriter<>( + ioPeon, + StringUtils.format("%s.nullBitmap", filenameBase), + bitmapSerdeFactory.getObjectStrategy() + ); + nullValueBitmapWriter.open(); + nullRowsBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); } @Override public void serialize(@Nullable Object obj) throws IOException { - float val = (obj == null) ? 0 : ((Number) obj).floatValue(); - writer.add(val); + if (obj == null) { + nullRowsBitmap.add(rowCount); + writer.add(0L); + } else { + writer.add(((Number) obj).floatValue()); + } + rowCount++; } @Override public void close() throws IOException { writer.close(); + nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); + nullValueBitmapWriter.close(); } @Override public long getSerializedSize() { - return writer.getSerializedSize(); + long bitmapSize = nullRowsBitmap.isEmpty() + ? 0L + : nullValueBitmapWriter.getSerializedSize(); + return Integer.BYTES + writer.getSerializedSize() + bitmapSize; } @Override public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { + channel.write(ByteBuffer.wrap(Ints.toByteArray((int) writer.getSerializedSize()))); writer.writeToChannel(channel, smoosher); + if (!nullRowsBitmap.isEmpty()) { + nullValueBitmapWriter.writeToChannel(channel, smoosher); + } } } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java index facd0c939fe7..7a8da6d3eeb0 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java @@ -19,13 +19,10 @@ package io.druid.segment; -import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.io.Closer; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.IOPeon; import io.druid.segment.serde.FloatGenericColumnPartSerdeV2; @@ -45,9 +42,6 @@ public class FloatDimensionMergerV9 implements DimensionMergerV9 protected IOPeon ioPeon; private FloatColumnSerializer serializer; - private MutableBitmap nullRowsBitmap; - private int rowCount = 0; - private ByteBufferWriter nullValueBitmapWriter; public FloatDimensionMergerV9( String dimensionName, @@ -64,8 +58,6 @@ public FloatDimensionMergerV9( this.outDir = outDir; this.ioPeon = ioPeon; this.progress = progress; - this.nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); - try { setupEncodedValueWriter(); } @@ -77,7 +69,12 @@ public FloatDimensionMergerV9( protected void setupEncodedValueWriter() throws IOException { final CompressedObjectStrategy.CompressionStrategy metCompression = indexSpec.getMetricCompression(); - this.serializer = FloatColumnSerializer.create(ioPeon, dimensionName, metCompression); + this.serializer = FloatColumnSerializer.create( + ioPeon, + dimensionName, + metCompression, + indexSpec.getBitmapSerdeFactory() + ); serializer.open(); } @@ -96,22 +93,13 @@ public Float convertSegmentRowValuesToMergedRowValues(Float segmentRow, int segm @Override public void processMergedRow(Float rowValues) throws IOException { - if (rowValues == null) { - nullRowsBitmap.add(rowCount); - } serializer.serialize(rowValues); - rowCount++; } @Override public void writeIndexes(List segmentRowNumConversions, Closer closer) throws IOException { - this.nullValueBitmapWriter = IndexMergerV9.createNullRowsBitmapWriter( - ioPeon, - dimensionName, - nullRowsBitmap, - indexSpec - ); + // floats have no indices to write } @Override @@ -126,12 +114,10 @@ public ColumnDescriptor makeColumnDescriptor() throws IOException serializer.close(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.FLOAT); - builder.setHasNullValues(!nullRowsBitmap.isEmpty()); builder.addSerde( FloatGenericColumnPartSerdeV2.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) - .withNullValueBitmapWriter(nullValueBitmapWriter) .withDelegate(serializer) .build() ); diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index 4b90fb268f05..3be29bb7ee86 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -34,14 +34,11 @@ import com.google.common.primitives.Longs; import com.google.inject.Inject; import io.druid.collections.CombiningIterable; -import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.collections.bitmap.MutableBitmap; import io.druid.io.ZeroCopyByteArrayOutputStream; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.JodaUtils; -import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.Comparators; import io.druid.java.util.common.guava.FunctionalIterable; import io.druid.java.util.common.guava.MergeIterable; @@ -55,7 +52,6 @@ import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.GenericIndexed; @@ -229,31 +225,17 @@ public Metadata apply(IndexableAdapter input) final ArrayList metWriters = setupMetricsWriters( ioPeon, mergedMetrics, metricsValueTypes, metricTypeNames, indexSpec ); - - List metricNullRowsBitmap = setupMetricNullRowsBitmaps(metWriters.size(), indexSpec); - final List rowNumConversions = Lists.newArrayListWithCapacity(adapters.size()); mergeIndexesAndWriteColumns( - adapters, progress, theRows, timeWriter, metWriters, rowNumConversions, mergers, - metricNullRowsBitmap + adapters, progress, theRows, timeWriter, metWriters, rowNumConversions, mergers ); /************ Create Inverted Indexes and Finalize Build Columns *************/ final String section = "build inverted index and columns"; progress.startSection(section); makeTimeColumn(v9Smoosher, progress, timeWriter); - makeMetricsColumns( - v9Smoosher, - progress, - mergedMetrics, - metricsValueTypes, - metricTypeNames, - metWriters, - indexSpec, - metricNullRowsBitmap, - ioPeon - ); + makeMetricsColumns(v9Smoosher, progress, mergedMetrics, metricsValueTypes, metricTypeNames, metWriters); for (int i = 0; i < mergedDimensions.size(); i++) { DimensionMergerV9 merger = (DimensionMergerV9) mergers.get(i); @@ -287,41 +269,6 @@ public Metadata apply(IndexableAdapter input) } } - private List setupMetricNullRowsBitmaps(int size, IndexSpec indexSpec) - { - List rv = Lists.newArrayListWithCapacity(size); - for (int i = 0; i < size; i++) { - rv.add(indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap()); - } - return rv; - } - - public static ByteBufferWriter createNullRowsBitmapWriter( - IOPeon ioPeon, - String columnName, - MutableBitmap nullRowsBitmap, - IndexSpec indexSpec - ) - throws IOException - { - if (nullRowsBitmap.isEmpty()) { - return null; - } - ; - ByteBufferWriter nullValueBitmapWriter = new ByteBufferWriter<>( - ioPeon, - StringUtils.format("%s.nullBitmap", columnName), - indexSpec.getBitmapSerdeFactory().getObjectStrategy() - ); - try (Closeable bitmapWriter = nullValueBitmapWriter) { - nullValueBitmapWriter.open(); - nullValueBitmapWriter.write(indexSpec.getBitmapSerdeFactory() - .getBitmapFactory() - .makeImmutableBitmap(nullRowsBitmap)); - } - return nullValueBitmapWriter; - } - private void makeMetadataBinary( final FileSmoosher v9Smoosher, final ProgressIndicator progress, @@ -403,10 +350,7 @@ private void makeMetricsColumns( final List mergedMetrics, final Map metricsValueTypes, final Map metricTypeNames, - final List metWriters, - final IndexSpec indexSpec, - final List metricNullRowsBitmap, - IOPeon ioPeon + final List metWriters ) throws IOException { final String section = "make metric columns"; @@ -429,12 +373,6 @@ private void makeMetricsColumns( .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((LongColumnSerializer) writer) - .withNullValueBitmapWriter(createNullRowsBitmapWriter( - ioPeon, - metric, - metricNullRowsBitmap.get(i), - indexSpec - )) .build() ); break; @@ -445,12 +383,6 @@ private void makeMetricsColumns( .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((FloatColumnSerializer) writer) - .withNullValueBitmapWriter(createNullRowsBitmapWriter( - ioPeon, - metric, - metricNullRowsBitmap.get(i), - indexSpec - )) .build() ); break; @@ -461,12 +393,6 @@ private void makeMetricsColumns( .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((DoubleColumnSerializer) writer) - .withNullValueBitmapWriter(createNullRowsBitmapWriter( - ioPeon, - metric, - metricNullRowsBitmap.get(i), - indexSpec - )) .build() ); break; @@ -541,8 +467,7 @@ private void mergeIndexesAndWriteColumns( final LongColumnSerializer timeWriter, final ArrayList metWriters, final List rowNumConversions, - final List mergers, - final List metricNullRowsBitmap + final List mergers ) throws IOException { final String section = "walk through and merge rows"; @@ -563,11 +488,7 @@ private void mergeIndexesAndWriteColumns( final Object[] metrics = theRow.getMetrics(); for (int i = 0; i < metrics.length; ++i) { - Object metricVal = metrics[i]; - if (metricVal == null) { - metricNullRowsBitmap.get(i).add(rowCount); - } - metWriters.get(i).serialize(metricVal); + metWriters.get(i).serialize(metrics[i]); } Object[] dims = theRow.getDims(); @@ -610,7 +531,8 @@ private LongColumnSerializer setupTimeWriter(final IOPeon ioPeon, final IndexSpe { LongColumnSerializer timeWriter = LongColumnSerializer.create( ioPeon, "little_end_time", CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY, - indexSpec.getLongEncoding() + indexSpec.getLongEncoding(), + indexSpec.getBitmapSerdeFactory() ); // we will close this writer after we added all the timestamps timeWriter.open(); @@ -633,13 +555,19 @@ private ArrayList setupMetricsWriters( GenericColumnSerializer writer; switch (type) { case LONG: - writer = LongColumnSerializer.create(ioPeon, metric, metCompression, longEncoding); + writer = LongColumnSerializer.create( + ioPeon, + metric, + metCompression, + longEncoding, + indexSpec.getBitmapSerdeFactory() + ); break; case FLOAT: - writer = FloatColumnSerializer.create(ioPeon, metric, metCompression); + writer = FloatColumnSerializer.create(ioPeon, metric, metCompression, indexSpec.getBitmapSerdeFactory()); break; case DOUBLE: - writer = DoubleColumnSerializer.create(ioPeon, metric, metCompression); + writer = DoubleColumnSerializer.create(ioPeon, metric, metCompression, indexSpec.getBitmapSerdeFactory()); break; case COMPLEX: final String typeName = metricTypeNames.get(metric); diff --git a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java index 5f77c13f13af..9e44aa30934c 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java @@ -19,8 +19,13 @@ package io.druid.segment; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.IOPeon; @@ -28,6 +33,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -40,10 +46,18 @@ public static LongColumnSerializer create( IOPeon ioPeon, String filenameBase, CompressedObjectStrategy.CompressionStrategy compression, - CompressionFactory.LongEncodingStrategy encoding + CompressionFactory.LongEncodingStrategy encoding, + BitmapSerdeFactory bitmapSerdeFactory ) { - return new LongColumnSerializer(ioPeon, filenameBase, IndexIO.BYTE_ORDER, compression, encoding); + return new LongColumnSerializer( + ioPeon, + filenameBase, + IndexIO.BYTE_ORDER, + compression, + encoding, + bitmapSerdeFactory + ); } private final IOPeon ioPeon; @@ -51,14 +65,19 @@ public static LongColumnSerializer create( private final ByteOrder byteOrder; private final CompressedObjectStrategy.CompressionStrategy compression; private final CompressionFactory.LongEncodingStrategy encoding; + private final BitmapSerdeFactory bitmapSerdeFactory; private LongSupplierSerializer writer; + private ByteBufferWriter nullValueBitmapWriter; + private MutableBitmap nullRowsBitmap; + private int rowCount = 0; public LongColumnSerializer( IOPeon ioPeon, String filenameBase, ByteOrder byteOrder, CompressedObjectStrategy.CompressionStrategy compression, - CompressionFactory.LongEncodingStrategy encoding + CompressionFactory.LongEncodingStrategy encoding, + BitmapSerdeFactory bitmapSerdeFactory ) { this.ioPeon = ioPeon; @@ -66,6 +85,7 @@ public LongColumnSerializer( this.byteOrder = byteOrder; this.compression = compression; this.encoding = encoding; + this.bitmapSerdeFactory = bitmapSerdeFactory; } @Override @@ -79,31 +99,52 @@ public void open() throws IOException compression ); writer.open(); + nullValueBitmapWriter = new ByteBufferWriter<>( + ioPeon, + StringUtils.format("%s.nullBitmap", filenameBase), + bitmapSerdeFactory.getObjectStrategy() + ); + nullValueBitmapWriter.open(); + nullRowsBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); } @Override public void serialize(@Nullable Object obj) throws IOException { - long val = (obj == null) ? 0 : ((Number) obj).longValue(); - writer.add(val); + if (obj == null) { + nullRowsBitmap.add(rowCount); + writer.add(0L); + } else { + writer.add(((Number) obj).longValue()); + } + rowCount++; } @Override public void close() throws IOException { writer.close(); + nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); + nullValueBitmapWriter.close(); } @Override public long getSerializedSize() { - return writer.getSerializedSize(); + long bitmapSize = nullRowsBitmap.isEmpty() + ? 0L + : nullValueBitmapWriter.getSerializedSize(); + return Integer.BYTES + writer.getSerializedSize() + bitmapSize; } @Override public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { + channel.write(ByteBuffer.wrap(Ints.toByteArray((int) writer.getSerializedSize()))); writer.writeToChannel(channel, smoosher); + if (!nullRowsBitmap.isEmpty()) { + nullValueBitmapWriter.writeToChannel(channel, smoosher); + } } } diff --git a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java index 8e31079997a1..9e5e4bf19f32 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java @@ -20,14 +20,11 @@ package io.druid.segment; import com.google.common.base.Throwables; -import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.io.Closer; import io.druid.java.util.common.logger.Logger; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.IOPeon; @@ -49,9 +46,6 @@ public class LongDimensionMergerV9 implements DimensionMergerV9 protected final File outDir; protected IOPeon ioPeon; protected LongColumnSerializer serializer; - private MutableBitmap nullRowsBitmap; - private int rowCount = 0; - private ByteBufferWriter nullValueBitmapWriter; public LongDimensionMergerV9( String dimensionName, @@ -68,7 +62,6 @@ public LongDimensionMergerV9( this.outDir = outDir; this.ioPeon = ioPeon; this.progress = progress; - this.nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); try { setupEncodedValueWriter(); @@ -82,7 +75,13 @@ protected void setupEncodedValueWriter() throws IOException { final CompressedObjectStrategy.CompressionStrategy metCompression = indexSpec.getMetricCompression(); final CompressionFactory.LongEncodingStrategy longEncoding = indexSpec.getLongEncoding(); - this.serializer = LongColumnSerializer.create(ioPeon, dimensionName, metCompression, longEncoding); + this.serializer = LongColumnSerializer.create( + ioPeon, + dimensionName, + metCompression, + longEncoding, + indexSpec.getBitmapSerdeFactory() + ); serializer.open(); } @@ -101,22 +100,13 @@ public Long convertSegmentRowValuesToMergedRowValues(Long segmentRow, int segmen @Override public void processMergedRow(Long rowValues) throws IOException { - if (rowValues == null) { - nullRowsBitmap.add(rowCount); - } serializer.serialize(rowValues); - rowCount++; } @Override public void writeIndexes(List segmentRowNumConversions, Closer closer) throws IOException { - this.nullValueBitmapWriter = IndexMergerV9.createNullRowsBitmapWriter( - ioPeon, - dimensionName, - nullRowsBitmap, - indexSpec - ); + // longs have no indices to write } @Override @@ -131,12 +121,10 @@ public ColumnDescriptor makeColumnDescriptor() throws IOException serializer.close(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.LONG); - builder.setHasNullValues(!nullRowsBitmap.isEmpty()); builder.addSerde( LongGenericColumnPartSerdeV2.serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) - .withNullValueBitmapWriter(nullValueBitmapWriter) .withDelegate(serializer) .build() ); diff --git a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java index 0aa194075726..277b990ce8c2 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java @@ -112,8 +112,6 @@ public static class Builder { private ValueType valueType = null; private Boolean hasMultipleValues = null; - private boolean hasNullValues = false; - private final List parts = Lists.newArrayList(); public Builder setValueType(ValueType valueType) @@ -136,12 +134,6 @@ public Builder setHasMultipleValues(boolean hasMultipleValues) return this; } - public Builder setHasNullValues(boolean hasNullValues) - { - this.hasNullValues = hasNullValues || this.hasNullValues; - return this; - } - public Builder addSerde(ColumnPartSerde serde) { parts.add(serde); diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java index d0fc57c9c33a..ed7e9cd78361 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Suppliers; -import com.google.common.primitives.Ints; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.FileSmoosher; @@ -32,7 +31,6 @@ import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.ByteBufferSerializer; -import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedDoublesIndexedSupplier; import javax.annotation.Nullable; @@ -92,7 +90,6 @@ public Deserializer getDeserializer() byte versionFromBuffer = buffer.get(); if (VERSION_ONE == versionFromBuffer) { - int offset = buffer.getInt(); int initialPos = buffer.position(); final CompressedDoublesIndexedSupplier column = CompressedDoublesIndexedSupplier.fromByteBuffer( @@ -128,7 +125,6 @@ public static class SerializerBuilder private ByteOrder byteOrder = null; private DoubleColumnSerializer delegate = null; private BitmapSerdeFactory bitmapSerdeFactory = null; - private ByteBufferWriter nullValueBitmapWriter = null; public SerializerBuilder withByteOrder(final ByteOrder byteOrder) { @@ -148,12 +144,6 @@ public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFa return this; } - public SerializerBuilder withNullValueBitmapWriter(ByteBufferWriter nullValueBitmapWriter) - { - this.nullValueBitmapWriter = nullValueBitmapWriter; - return this; - } - public DoubleGenericColumnPartSerdeV2 build() { return new DoubleGenericColumnPartSerdeV2( @@ -164,22 +154,14 @@ public DoubleGenericColumnPartSerdeV2 build() @Override public long numBytes() { - long size = delegate.getSerializedSize() + Ints.BYTES + Byte.BYTES; - if (nullValueBitmapWriter != null) { - size += nullValueBitmapWriter.getSerializedSize(); - } - return size; + return delegate.getSerializedSize() + Byte.BYTES; } @Override public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException { channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE})); - channel.write(ByteBuffer.wrap(Ints.toByteArray((int) delegate.getSerializedSize()))); delegate.writeToChannel(channel, fileSmoosher); - if (nullValueBitmapWriter != null) { - nullValueBitmapWriter.writeToChannel(channel, fileSmoosher); - } } } ); diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java index ee2dcacdedcc..a0c9ab76830c 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Suppliers; -import com.google.common.primitives.Ints; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.FileSmoosher; @@ -31,7 +30,6 @@ import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.ByteBufferSerializer; -import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedFloatsIndexedSupplier; import javax.annotation.Nullable; @@ -90,7 +88,6 @@ public static class SerializerBuilder private ByteOrder byteOrder = null; private FloatColumnSerializer delegate = null; private BitmapSerdeFactory bitmapSerdeFactory = null; - private ByteBufferWriter nullValueBitmapWriter = null; public SerializerBuilder withByteOrder(final ByteOrder byteOrder) { @@ -110,12 +107,6 @@ public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFa return this; } - public SerializerBuilder withNullValueBitmapWriter(ByteBufferWriter nullValueBitmapWriter) - { - this.nullValueBitmapWriter = nullValueBitmapWriter; - return this; - } - public FloatGenericColumnPartSerdeV2 build() { return new FloatGenericColumnPartSerdeV2( @@ -124,28 +115,19 @@ byteOrder, bitmapSerdeFactory, new Serializer() @Override public long numBytes() { - long size = delegate.getSerializedSize() + Ints.BYTES + Byte.BYTES; - if (nullValueBitmapWriter != null) { - size += nullValueBitmapWriter.getSerializedSize(); - } - return size; + return delegate.getSerializedSize() + Byte.BYTES; } @Override public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException { channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE})); - channel.write(ByteBuffer.wrap(Ints.toByteArray((int) delegate.getSerializedSize()))); delegate.writeToChannel(channel, fileSmoosher); - if (nullValueBitmapWriter != null) { - nullValueBitmapWriter.writeToChannel(channel, fileSmoosher); - } } } ); } - } @Override diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java index 1cd86316a86e..3aa34f4707da 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Suppliers; -import com.google.common.primitives.Ints; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.FileSmoosher; @@ -31,7 +30,6 @@ import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.ByteBufferSerializer; -import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedLongsIndexedSupplier; import javax.annotation.Nullable; @@ -89,7 +87,6 @@ public static class SerializerBuilder private ByteOrder byteOrder = null; private LongColumnSerializer delegate = null; private BitmapSerdeFactory bitmapSerdeFactory = null; - private ByteBufferWriter nullValueBitmapWriter = null; public SerializerBuilder withByteOrder(final ByteOrder byteOrder) { @@ -109,12 +106,6 @@ public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFa return this; } - public SerializerBuilder withNullValueBitmapWriter(ByteBufferWriter nullValueBitmapWriter) - { - this.nullValueBitmapWriter = nullValueBitmapWriter; - return this; - } - public LongGenericColumnPartSerdeV2 build() { return new LongGenericColumnPartSerdeV2( @@ -123,22 +114,14 @@ byteOrder, bitmapSerdeFactory, new Serializer() @Override public long numBytes() { - long size = delegate.getSerializedSize() + Ints.BYTES + Byte.BYTES; - if (nullValueBitmapWriter != null) { - size += nullValueBitmapWriter.getSerializedSize(); - } - return size; + return delegate.getSerializedSize() + Byte.BYTES; } @Override public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE})); - channel.write(ByteBuffer.wrap(Ints.toByteArray((int) delegate.getSerializedSize()))); delegate.writeToChannel(channel, smoosher); - if (nullValueBitmapWriter != null) { - nullValueBitmapWriter.writeToChannel(channel, smoosher); - } } } ); From c9300dde4b13b9c9bf74850bc2bae4aa15c0d3b5 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 4 Oct 2017 22:12:08 +0530 Subject: [PATCH 16/50] review comments - remove version from numeric column serve v2 --- .../serde/DoubleGenericColumnPartSerdeV2.java | 42 ++++++++---------- .../serde/FloatGenericColumnPartSerdeV2.java | 42 +++++++----------- .../serde/LongGenericColumnPartSerdeV2.java | 43 +++++++------------ 3 files changed, 49 insertions(+), 78 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java index ed7e9cd78361..08071af430cc 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Suppliers; import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.DoubleColumnSerializer; import io.druid.segment.column.ValueType; @@ -41,7 +40,6 @@ public class DoubleGenericColumnPartSerdeV2 implements ColumnPartSerde { - static final byte VERSION_ONE = 0x1; private final ByteOrder byteOrder; private Serializer serialize; private final BitmapSerdeFactory bitmapSerdeFactory; @@ -87,31 +85,25 @@ public Serializer getSerializer() public Deserializer getDeserializer() { return (buffer, builder, columnConfig) -> { - byte versionFromBuffer = buffer.get(); - - if (VERSION_ONE == versionFromBuffer) { - int offset = buffer.getInt(); - int initialPos = buffer.position(); - final CompressedDoublesIndexedSupplier column = CompressedDoublesIndexedSupplier.fromByteBuffer( - buffer, - byteOrder, - builder.getFileMapper() - ); - - buffer.position(initialPos + offset); - final ImmutableBitmap bitmap; - if (buffer.hasRemaining()) { - bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); - builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); - } else { - bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); - } - builder.setType(ValueType.DOUBLE) - .setHasMultipleValues(false) - .setGenericColumn(new DoubleGenericColumnSupplier(column, bitmap)); + int offset = buffer.getInt(); + int initialPos = buffer.position(); + final CompressedDoublesIndexedSupplier column = CompressedDoublesIndexedSupplier.fromByteBuffer( + buffer, + byteOrder, + builder.getFileMapper() + ); + + buffer.position(initialPos + offset); + final ImmutableBitmap bitmap; + if (buffer.hasRemaining()) { + bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); + builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); } else { - throw new IAE("Unknown version[%d]", (int) versionFromBuffer); + bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } + builder.setType(ValueType.DOUBLE) + .setHasMultipleValues(false) + .setGenericColumn(new DoubleGenericColumnSupplier(column, bitmap)); }; } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java index fa73a9ed8d17..6dcea1a62a77 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Suppliers; import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.FloatColumnSerializer; import io.druid.segment.column.ValueType; @@ -34,7 +33,6 @@ import javax.annotation.Nullable; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -42,8 +40,6 @@ */ public class FloatGenericColumnPartSerdeV2 implements ColumnPartSerde { - static final byte VERSION_ONE = 0x1; - @JsonCreator public static FloatGenericColumnPartSerdeV2 createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder, @@ -122,7 +118,6 @@ public long numBytes() @Override public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE})); delegate.writeToChannel(channel, fileSmoosher); } } @@ -141,29 +136,24 @@ public Serializer getSerializer() public Deserializer getDeserializer() { return (buffer, builder, columnConfig) -> { - byte versionFromBuffer = buffer.get(); - if (VERSION_ONE == versionFromBuffer) { - int offset = buffer.getInt(); - int initialPos = buffer.position(); - final CompressedFloatsIndexedSupplier column = CompressedFloatsIndexedSupplier.fromByteBuffer( - buffer, - byteOrder, - builder.getFileMapper() - ); - buffer.position(initialPos + offset); - final ImmutableBitmap bitmap; - if (buffer.hasRemaining()) { - bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); - builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); - } else { - bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); - } - builder.setType(ValueType.FLOAT) - .setHasMultipleValues(false) - .setGenericColumn(new FloatGenericColumnSupplier(column, byteOrder, bitmap)); + int offset = buffer.getInt(); + int initialPos = buffer.position(); + final CompressedFloatsIndexedSupplier column = CompressedFloatsIndexedSupplier.fromByteBuffer( + buffer, + byteOrder, + builder.getFileMapper() + ); + buffer.position(initialPos + offset); + final ImmutableBitmap bitmap; + if (buffer.hasRemaining()) { + bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); + builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); } else { - throw new IAE("Unknown version[%d]", (int) versionFromBuffer); + bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } + builder.setType(ValueType.FLOAT) + .setHasMultipleValues(false) + .setGenericColumn(new FloatGenericColumnSupplier(column, byteOrder, bitmap)); }; } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java index d3d0d095807d..d7f7a743e47e 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Suppliers; import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.LongColumnSerializer; import io.druid.segment.column.ValueType; @@ -34,7 +33,6 @@ import javax.annotation.Nullable; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -42,8 +40,6 @@ */ public class LongGenericColumnPartSerdeV2 implements ColumnPartSerde { - static final byte VERSION_ONE = 0x1; - @JsonCreator public static LongGenericColumnPartSerdeV2 createDeserializer( @JsonProperty("byteOrder") ByteOrder byteOrder, @@ -121,7 +117,6 @@ public long numBytes() @Override public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE})); delegate.writeToChannel(channel, smoosher); } } @@ -139,30 +134,24 @@ public Serializer getSerializer() public Deserializer getDeserializer() { return (buffer, builder, columnConfig) -> { - byte versionFromBuffer = buffer.get(); - - if (VERSION_ONE == versionFromBuffer) { - int offset = buffer.getInt(); - int initialPos = buffer.position(); - final CompressedLongsIndexedSupplier column = CompressedLongsIndexedSupplier.fromByteBuffer( - buffer, - byteOrder, - builder.getFileMapper() - ); - buffer.position(initialPos + offset); - final ImmutableBitmap bitmap; - if (buffer.hasRemaining()) { - bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); - builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); - } else { - bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); - } - builder.setType(ValueType.LONG) - .setHasMultipleValues(false) - .setGenericColumn(new LongGenericColumnSupplier(column, bitmap)); + int offset = buffer.getInt(); + int initialPos = buffer.position(); + final CompressedLongsIndexedSupplier column = CompressedLongsIndexedSupplier.fromByteBuffer( + buffer, + byteOrder, + builder.getFileMapper() + ); + buffer.position(initialPos + offset); + final ImmutableBitmap bitmap; + if (buffer.hasRemaining()) { + bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); + builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); } else { - throw new IAE("Unknown version[%d]", (int) versionFromBuffer); + bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } + builder.setType(ValueType.LONG) + .setHasMultipleValues(false) + .setGenericColumn(new LongGenericColumnSupplier(column, bitmap)); }; } From c46c6b61d0a0258433523015c32d276f2ba73565 Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 5 Oct 2017 09:27:58 +0530 Subject: [PATCH 17/50] Fix compilation and tests --- .../druid/segment/serde/DoubleGenericColumnPartSerdeV2.java | 4 +--- .../io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java | 2 +- .../io/druid/segment/serde/LongGenericColumnPartSerdeV2.java | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java index 08071af430cc..d482e5f14dd0 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -34,7 +34,6 @@ import javax.annotation.Nullable; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -146,13 +145,12 @@ public DoubleGenericColumnPartSerdeV2 build() @Override public long numBytes() { - return delegate.getSerializedSize() + Byte.BYTES; + return delegate.getSerializedSize(); } @Override public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{VERSION_ONE})); delegate.writeToChannel(channel, fileSmoosher); } } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java index 6dcea1a62a77..e27f70b1fa38 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -112,7 +112,7 @@ public FloatGenericColumnPartSerdeV2 build() @Override public long numBytes() { - return delegate.getSerializedSize() + Byte.BYTES; + return delegate.getSerializedSize(); } @Override diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java index d7f7a743e47e..6f651238e2db 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -111,7 +111,7 @@ public LongGenericColumnPartSerdeV2 build() @Override public long numBytes() { - return delegate.getSerializedSize() + Byte.BYTES; + return delegate.getSerializedSize(); } @Override From 2e3ba31370986e4ddde263384882e1b4d83442e2 Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 6 Oct 2017 04:22:55 +0530 Subject: [PATCH 18/50] review comments --- .../main/java/io/druid/data/input/MapBasedRow.java | 12 +++++++++--- .../src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 | 3 ++- .../java/io/druid/math/expr/ExprListenerImpl.java | 9 +++++++++ .../query/aggregation/NullableBufferAggregator.java | 2 +- .../io/druid/segment/StringDimensionMergerV9.java | 4 ++-- 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/io/druid/data/input/MapBasedRow.java b/api/src/main/java/io/druid/data/input/MapBasedRow.java index 462e9c8e8ee5..f30bb54d925f 100644 --- a/api/src/main/java/io/druid/data/input/MapBasedRow.java +++ b/api/src/main/java/io/druid/data/input/MapBasedRow.java @@ -112,7 +112,9 @@ public Float getFloatMetric(String metric) return null; } - if (metricValue instanceof Number) { + if (metricValue instanceof Float) { + return (Float) metricValue; + } else if (metricValue instanceof Number) { return ((Number) metricValue).floatValue(); } else if (metricValue instanceof String) { try { @@ -136,7 +138,9 @@ public Long getLongMetric(String metric) return null; } - if (metricValue instanceof Number) { + if (metricValue instanceof Long) { + return (Long) metricValue; + } else if (metricValue instanceof Number) { return ((Number) metricValue).longValue(); } else if (metricValue instanceof String) { try { @@ -161,7 +165,9 @@ public Double getDoubleMetric(String metric) return null; } - if (metricValue instanceof Number) { + if (metricValue instanceof Double) { + return (Double) metricValue; + } else if (metricValue instanceof Number) { return ((Number) metricValue).doubleValue(); } else if (metricValue instanceof String) { try { diff --git a/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 b/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 index c3f88029b031..c3547e70d2ed 100644 --- a/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 +++ b/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 @@ -1,6 +1,7 @@ grammar Expr; -expr : ('-'|'!') expr # unaryOpExpr +expr : 'null' # null + | ('-'|'!') expr # unaryOpExpr | expr '^' expr # powOpExpr | expr ('*'|'/'|'%') expr # mulDivModuloExpr | expr ('+'|'-') expr # addSubExpr diff --git a/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java index 87d9b8e558dd..b74e69df1fe5 100644 --- a/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java +++ b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java @@ -333,4 +333,13 @@ public void exitFunctionArgs(ExprParser.FunctionArgsContext ctx) nodes.put(ctx, args); } + + @Override + public void exitNull(ExprParser.NullContext ctx) + { + nodes.put( + ctx, + new StringExpr(null) + ); + } } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 724da9b8ea16..180a72b8e9a6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -101,6 +101,6 @@ public boolean isNull(ByteBuffer buf, int position) @Override public void close() { - // Nothing to close. + delegate.close(); } } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index 3014a275cd1f..be05a2bddc40 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -20,6 +20,7 @@ package io.druid.segment; import com.google.common.base.Splitter; +import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.io.ByteStreams; @@ -57,7 +58,6 @@ import it.unimi.dsi.fastutil.ints.AbstractIntIterator; import it.unimi.dsi.fastutil.ints.IntIterable; import it.unimi.dsi.fastutil.ints.IntIterator; -import org.apache.logging.log4j.util.Strings; import javax.annotation.Nonnull; import java.io.Closeable; @@ -409,7 +409,7 @@ static void mergeBitmaps( if (hasSpatial) { String dimVal = dimVals.get(dictId); - if (Strings.isNotEmpty(dimVal)) { + if (!Strings.isNullOrEmpty(dimVal)) { List stringCoords = Lists.newArrayList(SPLITTER.split(dimVal)); float[] coords = new float[stringCoords.size()]; for (int j = 0; j < coords.length; j++) { From d1028013d02eb944bbaeb33bfc6453fe584406f5 Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 6 Oct 2017 22:48:18 +0530 Subject: [PATCH 19/50] Add @Override annotation --- .../src/main/java/io/druid/segment/ObjectColumnSelector.java | 1 + 1 file changed, 1 insertion(+) diff --git a/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java index 2c0519e58884..e945bdf307d7 100644 --- a/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java @@ -76,6 +76,7 @@ default long getLong() * objects. */ @Deprecated + @Override default boolean isNull() { T value = getObject(); From 400be55840012592b6118ad8187df7432b576537 Mon Sep 17 00:00:00 2001 From: Nishant Date: Sun, 8 Oct 2017 01:13:26 +0530 Subject: [PATCH 20/50] more review comments and fix failing test --- .../query/aggregation/NullableAggregateCombiner.java | 4 ++-- .../io/druid/query/aggregation/NullableAggregator.java | 4 ++-- .../druid/query/aggregation/NullableBufferAggregator.java | 8 ++++---- .../main/java/io/druid/segment/NullHandlingHelper.java | 2 +- .../java/io/druid/segment/ZeroFloatColumnSelector.java | 1 + .../druid/segment/virtual/ExpressionObjectSelector.java | 4 +++- .../io/druid/segment/filter/ExpressionFilterTest.java | 6 +++++- 7 files changed, 18 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java index b59e7b6a612b..c261da320c5d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -51,8 +51,8 @@ public void reset(ColumnValueSelector selector) @Override public void fold(ColumnValueSelector selector) { - boolean isCurrentValNull = selector.isNull(); - if (!isCurrentValNull) { + boolean isNotNull = !selector.isNull(); + if (isNotNull) { if (isNullResult) { isNullResult = false; } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index 27a3a0e7fce5..8a2bff623ca2 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -44,8 +44,8 @@ public NullableAggregator(Aggregator delegate, ColumnValueSelector selector) @Override public void aggregate() { - boolean isCurrentValNull = selector.isNull(); - if (!isCurrentValNull) { + boolean isNotNull = !selector.isNull(); + if (isNotNull) { if (isNullResult) { isNullResult = false; } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 180a72b8e9a6..36527f8e0976 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -33,8 +33,8 @@ public class NullableBufferAggregator implements BufferAggregator { - public static final byte IS_NULL_BYTE = (byte) 1; - public static final byte IS_NOT_NULL_BYTE = (byte) 0; + private static final byte IS_NULL_BYTE = (byte) 1; + private static final byte IS_NOT_NULL_BYTE = (byte) 0; private final BufferAggregator delegate; private final ColumnValueSelector selector; @@ -55,8 +55,8 @@ public void init(ByteBuffer buf, int position) @Override public void aggregate(ByteBuffer buf, int position) { - boolean isCurrentValNull = selector.isNull(); - if (!isCurrentValNull) { + boolean isNotNull = !selector.isNull(); + if (isNotNull) { if (buf.get(position) == IS_NULL_BYTE) { buf.put(position, IS_NOT_NULL_BYTE); } diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index ec53dcbaef3d..8ffc68bed375 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -38,7 +38,7 @@ public class NullHandlingHelper // Using static injection to avoid adding JacksonInject annotations all over the code. @Inject - private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig(false); + private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig(true); public static boolean useDefaultValuesForNull() { diff --git a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java index 5c2a21b8236d..5f52bb2e11f0 100644 --- a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java @@ -48,6 +48,7 @@ public double getDouble(int offset) return 0.0; } + @Override public boolean isNull() { return false; diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java index 4e6c52bf983f..f3d58ea47308 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java @@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.Maps; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -114,7 +115,8 @@ static Supplier supplierFromDimensionSelector(final DimensionSelector se static Supplier supplierFromObjectSelector(final ObjectColumnSelector selector) { if (selector == null) { - return null; + // Missing column. + return Suppliers.ofInstance(NullHandlingHelper.nullToDefault((String) null)); } final Class clazz = selector.classOfObject(); diff --git a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java index 10e8892732ca..6c6bab684952 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java @@ -193,7 +193,11 @@ public void testCompareColumns() @Test public void testMissingColumn() { - assertFilterMatches(EDF("missing == null"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandlingHelper.useDefaultValuesForNull()) { + assertFilterMatches(EDF("missing == ''"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + } else { + assertFilterMatches(EDF("missing == null"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + } assertFilterMatches(EDF("missing == '1'"), ImmutableList.of()); assertFilterMatches(EDF("missing == 2"), ImmutableList.of()); assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); From c5fbbd532f859f48bc6583f92e8866cd01fcb824 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 11 Oct 2017 02:28:02 +0530 Subject: [PATCH 21/50] Remove Guice Static Injection - Static Injection is not recommended in guice, one e.g saw random failures on travis due to change in order in which tests are run. - It can cause things to break unexpectedly when druid classes are used in embedded library. - Instead of using guice read null handling behavior via System property in a consistent way. --- .../java/io/druid/guice/GuiceInjectors.java | 1 - .../io/druid/guice/NullHandlingModule.java | 37 ----------- .../io/druid/segment/NullHandlingHelper.java | 25 +++---- .../segment/NullValueHandlingConfig.java | 43 ------------ .../NullHandlingHelperInjectionTest.java | 65 ------------------- 5 files changed, 14 insertions(+), 157 deletions(-) delete mode 100644 processing/src/main/java/io/druid/guice/NullHandlingModule.java delete mode 100644 processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java delete mode 100644 processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java diff --git a/processing/src/main/java/io/druid/guice/GuiceInjectors.java b/processing/src/main/java/io/druid/guice/GuiceInjectors.java index 8adaae4dd25d..f3bac2362ce4 100644 --- a/processing/src/main/java/io/druid/guice/GuiceInjectors.java +++ b/processing/src/main/java/io/druid/guice/GuiceInjectors.java @@ -42,7 +42,6 @@ public static Collection makeDefaultStartupModules() new JacksonModule(), new PropertiesModule(Arrays.asList("common.runtime.properties", "runtime.properties")), new ConfigModule(), - new NullHandlingModule(), new Module() { @Override diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java deleted file mode 100644 index ba6ca343098d..000000000000 --- a/processing/src/main/java/io/druid/guice/NullHandlingModule.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.guice; - -import com.google.inject.Binder; -import com.google.inject.Module; -import io.druid.segment.NullHandlingHelper; -import io.druid.segment.NullValueHandlingConfig; - -/** - */ -public class NullHandlingModule implements Module -{ - @Override - public void configure(Binder binder) - { - JsonConfigProvider.bind(binder, "druid.null.handling", NullValueHandlingConfig.class); - binder.requestStaticInjection(NullHandlingHelper.class); - } -} diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index 8ffc68bed375..2ca1d297445e 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -38,56 +38,59 @@ public class NullHandlingHelper // Using static injection to avoid adding JacksonInject annotations all over the code. @Inject - private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig(true); + private static boolean useDefaultValuesForNull = Boolean.valueOf(System.getProperty( + "druid.null.handling.useDefaultValueForNull", + "true" + )); public static boolean useDefaultValuesForNull() { - return INSTANCE.isUseDefaultValuesForNull(); + return useDefaultValuesForNull; } public static String nullToDefault(String value) { - return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; + return useDefaultValuesForNull ? Strings.nullToEmpty(value) : value; } public static String defaultToNull(String value) { - return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; + return useDefaultValuesForNull ? Strings.emptyToNull(value) : value; } public static boolean isNullOrDefault(String value) { - return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; + return useDefaultValuesForNull ? Strings.isNullOrEmpty(value) : value == null; } public static Long nullToDefault(Long value) { - return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_LONG : value; + return useDefaultValuesForNull && value == null ? ZERO_LONG : value; } public static Double nullToDefault(Double value) { - return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_DOUBLE : value; + return useDefaultValuesForNull && value == null ? ZERO_DOUBLE : value; } public static Float nullToDefault(Float value) { - return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_FLOAT : value; + return useDefaultValuesForNull && value == null ? ZERO_FLOAT : value; } public static Aggregator getNullableAggregator(Aggregator aggregator, ColumnValueSelector selector) { - return INSTANCE.isUseDefaultValuesForNull() ? aggregator : new NullableAggregator(aggregator, selector); + return useDefaultValuesForNull ? aggregator : new NullableAggregator(aggregator, selector); } public static BufferAggregator getNullableAggregator(BufferAggregator aggregator, ColumnValueSelector selector) { - return INSTANCE.isUseDefaultValuesForNull() ? aggregator : new NullableBufferAggregator(aggregator, selector); + return useDefaultValuesForNull ? aggregator : new NullableBufferAggregator(aggregator, selector); } public static AggregateCombiner getNullableCombiner(AggregateCombiner combiner) { - return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableAggregateCombiner(combiner); + return useDefaultValuesForNull ? combiner : new NullableAggregateCombiner(combiner); } public static int extraAggregatorBytes() diff --git a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java b/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java deleted file mode 100644 index 7e457534bbf8..000000000000 --- a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class NullValueHandlingConfig -{ - - @JsonProperty("useDefaultValueForNull") - private final boolean useDefaultValuesForNull; - - @JsonCreator - public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) - { - this.useDefaultValuesForNull = useDefaultValuesForNull == null - ? true - : useDefaultValuesForNull; - } - - public boolean isUseDefaultValuesForNull() - { - return useDefaultValuesForNull; - } -} diff --git a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java deleted file mode 100644 index ad29938cdbaf..000000000000 --- a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.guice; - -import com.google.inject.Injector; -import io.druid.segment.NullHandlingHelper; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -@Ignore -public class NullHandlingHelperInjectionTest -{ - public static String NULL_HANDLING_CONFIG_STRING = ("druid.null.handling.useDefaultValueForNull"); - - @Test - public void testNullHandlingHelperUseDefaultValues() - { - String prev = System.getProperty(NULL_HANDLING_CONFIG_STRING); - try { - System.setProperty(NULL_HANDLING_CONFIG_STRING, "true"); - Injector injector = GuiceInjectors.makeStartupInjector(); - Assert.assertEquals(true, NullHandlingHelper.useDefaultValuesForNull()); - } - finally { - if (prev != null) { - System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); - } - } - } - - @Test - public void testNullHandlingHelperNoDefaultValues() - { - String prev = System.getProperty(NULL_HANDLING_CONFIG_STRING); - try { - System.setProperty(NULL_HANDLING_CONFIG_STRING, "false"); - Injector injector = GuiceInjectors.makeStartupInjector(); - Assert.assertEquals(false, NullHandlingHelper.useDefaultValuesForNull()); - } - finally { - if (prev != null) { - System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); - } - } - } - -} From b5a0880d7e0e85ba51be4ec1f40a67a42d3758c7 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 11 Oct 2017 03:19:29 +0530 Subject: [PATCH 22/50] Revert "Remove Guice Static Injection" This reverts commit c5fbbd532f859f48bc6583f92e8866cd01fcb824. --- .../java/io/druid/guice/GuiceInjectors.java | 1 + .../io/druid/guice/NullHandlingModule.java | 37 +++++++++++ .../io/druid/segment/NullHandlingHelper.java | 25 ++++--- .../segment/NullValueHandlingConfig.java | 43 ++++++++++++ .../NullHandlingHelperInjectionTest.java | 65 +++++++++++++++++++ 5 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 processing/src/main/java/io/druid/guice/NullHandlingModule.java create mode 100644 processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java create mode 100644 processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java diff --git a/processing/src/main/java/io/druid/guice/GuiceInjectors.java b/processing/src/main/java/io/druid/guice/GuiceInjectors.java index f3bac2362ce4..8adaae4dd25d 100644 --- a/processing/src/main/java/io/druid/guice/GuiceInjectors.java +++ b/processing/src/main/java/io/druid/guice/GuiceInjectors.java @@ -42,6 +42,7 @@ public static Collection makeDefaultStartupModules() new JacksonModule(), new PropertiesModule(Arrays.asList("common.runtime.properties", "runtime.properties")), new ConfigModule(), + new NullHandlingModule(), new Module() { @Override diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java new file mode 100644 index 000000000000..ba6ca343098d --- /dev/null +++ b/processing/src/main/java/io/druid/guice/NullHandlingModule.java @@ -0,0 +1,37 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.segment.NullHandlingHelper; +import io.druid.segment.NullValueHandlingConfig; + +/** + */ +public class NullHandlingModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.null.handling", NullValueHandlingConfig.class); + binder.requestStaticInjection(NullHandlingHelper.class); + } +} diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index 2ca1d297445e..8ffc68bed375 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -38,59 +38,56 @@ public class NullHandlingHelper // Using static injection to avoid adding JacksonInject annotations all over the code. @Inject - private static boolean useDefaultValuesForNull = Boolean.valueOf(System.getProperty( - "druid.null.handling.useDefaultValueForNull", - "true" - )); + private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig(true); public static boolean useDefaultValuesForNull() { - return useDefaultValuesForNull; + return INSTANCE.isUseDefaultValuesForNull(); } public static String nullToDefault(String value) { - return useDefaultValuesForNull ? Strings.nullToEmpty(value) : value; + return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; } public static String defaultToNull(String value) { - return useDefaultValuesForNull ? Strings.emptyToNull(value) : value; + return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; } public static boolean isNullOrDefault(String value) { - return useDefaultValuesForNull ? Strings.isNullOrEmpty(value) : value == null; + return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; } public static Long nullToDefault(Long value) { - return useDefaultValuesForNull && value == null ? ZERO_LONG : value; + return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_LONG : value; } public static Double nullToDefault(Double value) { - return useDefaultValuesForNull && value == null ? ZERO_DOUBLE : value; + return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_DOUBLE : value; } public static Float nullToDefault(Float value) { - return useDefaultValuesForNull && value == null ? ZERO_FLOAT : value; + return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_FLOAT : value; } public static Aggregator getNullableAggregator(Aggregator aggregator, ColumnValueSelector selector) { - return useDefaultValuesForNull ? aggregator : new NullableAggregator(aggregator, selector); + return INSTANCE.isUseDefaultValuesForNull() ? aggregator : new NullableAggregator(aggregator, selector); } public static BufferAggregator getNullableAggregator(BufferAggregator aggregator, ColumnValueSelector selector) { - return useDefaultValuesForNull ? aggregator : new NullableBufferAggregator(aggregator, selector); + return INSTANCE.isUseDefaultValuesForNull() ? aggregator : new NullableBufferAggregator(aggregator, selector); } public static AggregateCombiner getNullableCombiner(AggregateCombiner combiner) { - return useDefaultValuesForNull ? combiner : new NullableAggregateCombiner(combiner); + return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableAggregateCombiner(combiner); } public static int extraAggregatorBytes() diff --git a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java b/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java new file mode 100644 index 000000000000..7e457534bbf8 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java @@ -0,0 +1,43 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class NullValueHandlingConfig +{ + + @JsonProperty("useDefaultValueForNull") + private final boolean useDefaultValuesForNull; + + @JsonCreator + public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) + { + this.useDefaultValuesForNull = useDefaultValuesForNull == null + ? true + : useDefaultValuesForNull; + } + + public boolean isUseDefaultValuesForNull() + { + return useDefaultValuesForNull; + } +} diff --git a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java new file mode 100644 index 000000000000..ad29938cdbaf --- /dev/null +++ b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.guice; + +import com.google.inject.Injector; +import io.druid.segment.NullHandlingHelper; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class NullHandlingHelperInjectionTest +{ + public static String NULL_HANDLING_CONFIG_STRING = ("druid.null.handling.useDefaultValueForNull"); + + @Test + public void testNullHandlingHelperUseDefaultValues() + { + String prev = System.getProperty(NULL_HANDLING_CONFIG_STRING); + try { + System.setProperty(NULL_HANDLING_CONFIG_STRING, "true"); + Injector injector = GuiceInjectors.makeStartupInjector(); + Assert.assertEquals(true, NullHandlingHelper.useDefaultValuesForNull()); + } + finally { + if (prev != null) { + System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); + } + } + } + + @Test + public void testNullHandlingHelperNoDefaultValues() + { + String prev = System.getProperty(NULL_HANDLING_CONFIG_STRING); + try { + System.setProperty(NULL_HANDLING_CONFIG_STRING, "false"); + Injector injector = GuiceInjectors.makeStartupInjector(); + Assert.assertEquals(false, NullHandlingHelper.useDefaultValuesForNull()); + } + finally { + if (prev != null) { + System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); + } + } + } + +} From 23ee4f1f32ec345afe2b8b445cfd767dc0c53580 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 11 Oct 2017 04:10:45 +0530 Subject: [PATCH 23/50] Attempt to fix UT in travis runs --- .../java/io/druid/segment/NullHandlingHelper.java | 10 ++++++++-- .../guice/NullHandlingHelperInjectionTest.java | 13 +++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index 8ffc68bed375..92a616d9f39c 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -30,15 +30,21 @@ public class NullHandlingHelper { + private static String NULL_HANDLING_CONFIG_STRING = "druid.null.handling.useDefaultValueForNull"; + // use these values to ensure that convertObjectToLong(), convertObjectToDouble() and convertObjectToFloat() // return the same boxed object when returning a constant zero. public static final Double ZERO_DOUBLE = 0.0d; public static final Float ZERO_FLOAT = 0.0f; public static final Long ZERO_LONG = 0L; - // Using static injection to avoid adding JacksonInject annotations all over the code. + // INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. + // See NullHandlingModule for details. + // The default system property is supposed to be used only in tests. @Inject - private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig(true); + private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( + Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) + ); public static boolean useDefaultValuesForNull() { diff --git a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java index ad29938cdbaf..28673d0c6f51 100644 --- a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java +++ b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java @@ -22,10 +22,8 @@ import com.google.inject.Injector; import io.druid.segment.NullHandlingHelper; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; -@Ignore public class NullHandlingHelperInjectionTest { public static String NULL_HANDLING_CONFIG_STRING = ("druid.null.handling.useDefaultValueForNull"); @@ -41,7 +39,7 @@ public void testNullHandlingHelperUseDefaultValues() } finally { if (prev != null) { - System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); + resetNullHandlingHelper(prev); } } } @@ -57,9 +55,16 @@ public void testNullHandlingHelperNoDefaultValues() } finally { if (prev != null) { - System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); + resetNullHandlingHelper(prev); } } } + private void resetNullHandlingHelper(String prev) + { + System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); + Injector injector = GuiceInjectors.makeStartupInjector(); + Assert.assertEquals(Boolean.valueOf(prev), NullHandlingHelper.useDefaultValuesForNull()); + } + } From 01ec864549c400b56002a54cbe2b941760dd4b66 Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 14 Oct 2017 01:32:34 +0530 Subject: [PATCH 24/50] Fix metric serialization missing bitmapSerdeFactory and QueryableIndexColumnSelectorFactory Will add tests in subsequent commits. --- .../java/io/druid/segment/IndexMergerV9.java | 20 ++++++++++++++++--- .../QueryableIndexColumnSelectorFactory.java | 9 +++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index 3be29bb7ee86..280b852a256f 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -235,7 +235,15 @@ public Metadata apply(IndexableAdapter input) final String section = "build inverted index and columns"; progress.startSection(section); makeTimeColumn(v9Smoosher, progress, timeWriter); - makeMetricsColumns(v9Smoosher, progress, mergedMetrics, metricsValueTypes, metricTypeNames, metWriters); + makeMetricsColumns( + v9Smoosher, + progress, + mergedMetrics, + metricsValueTypes, + metricTypeNames, + metWriters, + indexSpec + ); for (int i = 0; i < mergedDimensions.size(); i++) { DimensionMergerV9 merger = (DimensionMergerV9) mergers.get(i); @@ -350,7 +358,8 @@ private void makeMetricsColumns( final List mergedMetrics, final Map metricsValueTypes, final Map metricTypeNames, - final List metWriters + final List metWriters, + final IndexSpec indexSpec ) throws IOException { final String section = "make metric columns"; @@ -373,6 +382,7 @@ private void makeMetricsColumns( .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((LongColumnSerializer) writer) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .build() ); break; @@ -383,6 +393,7 @@ private void makeMetricsColumns( .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((FloatColumnSerializer) writer) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .build() ); break; @@ -393,6 +404,7 @@ private void makeMetricsColumns( .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((DoubleColumnSerializer) writer) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .build() ); break; @@ -500,7 +512,9 @@ private void mergeIndexesAndWriteColumns( merger.processMergedRow(dims[i]); } - Iterator> rowsIterator = theRow.getComprisedRows().int2ObjectEntrySet().fastIterator(); + Iterator> rowsIterator = theRow.getComprisedRows() + .int2ObjectEntrySet() + .fastIterator(); while (rowsIterator.hasNext()) { Int2ObjectMap.Entry comprisedRow = rowsIterator.next(); diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java index 1db2cc295f3a..fbb88c3619a6 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java @@ -254,6 +254,9 @@ public Class classOfObject() @Override public Float getObject() { + if (columnVals.isNull(offset.getOffset())) { + return null; + } return columnVals.getFloatSingleValueRow(offset.getOffset()); } }; @@ -270,6 +273,9 @@ public Class classOfObject() @Override public Double getObject() { + if (columnVals.isNull(offset.getOffset())) { + return null; + } return columnVals.getDoubleSingleValueRow(offset.getOffset()); } }; @@ -286,6 +292,9 @@ public Class classOfObject() @Override public Long getObject() { + if (columnVals.isNull(offset.getOffset())) { + return null; + } return columnVals.getLongSingleValueRow(offset.getOffset()); } }; From 7dc2efd6077d79e38f6b6da397fddc8e6a7c4ad2 Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 16 Oct 2017 22:39:32 +0530 Subject: [PATCH 25/50] [BUG-90164] Addendum patch to fix serde of bitmap factory used. TESTSUITE_FILE=DruidHive2 Change-Id: I6a9757b19d2b4b8b71760ea155ab51e6bffe87ba --- .../serde/DoubleGenericColumnPartSerdeV2.java | 5 ++ .../serde/FloatGenericColumnPartSerdeV2.java | 6 +++ .../serde/LongGenericColumnPartSerdeV2.java | 6 +++ .../DoubleGenericColumnPartSerdeTest.java | 53 +++++++++++++++++++ .../FloatGenericColumnPartSerdeTest.java | 53 +++++++++++++++++++ .../serde/LongGenericColumnPartSerdeTest.java | 53 +++++++++++++++++++ 6 files changed, 176 insertions(+) create mode 100644 processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java create mode 100644 processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java create mode 100644 processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java index d482e5f14dd0..296f5e25f5b3 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -62,6 +62,11 @@ public ByteOrder getByteOrder() return byteOrder; } + @JsonProperty + public BitmapSerdeFactory getBitmapSerdeFactory() + { + return bitmapSerdeFactory; + } public DoubleGenericColumnPartSerdeV2( ByteOrder byteOrder, diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java index e27f70b1fa38..b76289cc4ae2 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -74,6 +74,12 @@ public ByteOrder getByteOrder() return byteOrder; } + @JsonProperty + public BitmapSerdeFactory getBitmapSerdeFactory() + { + return bitmapSerdeFactory; + } + public static SerializerBuilder serializerBuilder() { return new SerializerBuilder(); diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java index 6f651238e2db..ac0ede4ea5a6 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -73,6 +73,12 @@ public ByteOrder getByteOrder() return byteOrder; } + @JsonProperty + public BitmapSerdeFactory getBitmapSerdeFactory() + { + return bitmapSerdeFactory; + } + public static SerializerBuilder serializerBuilder() { return new SerializerBuilder(); diff --git a/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java new file mode 100644 index 000000000000..f15b73896989 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.TestHelper; +import io.druid.segment.data.RoaringBitmapSerdeFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteOrder; + +public class DoubleGenericColumnPartSerdeTest +{ + private static ObjectMapper jsonMapper = TestHelper.getJsonMapper(); + + @Test + public void testSerdeV2() throws IOException + { + DoubleGenericColumnPartSerdeV2 object = DoubleGenericColumnPartSerdeV2.getDoubleGenericColumnPartSerde( + ByteOrder.BIG_ENDIAN, + new RoaringBitmapSerdeFactory(true) + ); + ColumnPartSerde columnPartSerde = jsonMapper.readValue( + jsonMapper.writeValueAsString(object), + ColumnPartSerde.class + ); + Assert.assertTrue(columnPartSerde instanceof DoubleGenericColumnPartSerdeV2); + Assert.assertEquals( + object.getBitmapSerdeFactory(), + ((DoubleGenericColumnPartSerdeV2) columnPartSerde).getBitmapSerdeFactory() + ); + + } +} diff --git a/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java new file mode 100644 index 000000000000..41680d3b5b8e --- /dev/null +++ b/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.TestHelper; +import io.druid.segment.data.RoaringBitmapSerdeFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteOrder; + +public class FloatGenericColumnPartSerdeTest +{ + private static ObjectMapper jsonMapper = TestHelper.getJsonMapper(); + + @Test + public void testSerdeV2() throws IOException + { + FloatGenericColumnPartSerdeV2 object = FloatGenericColumnPartSerdeV2.createDeserializer( + ByteOrder.BIG_ENDIAN, + new RoaringBitmapSerdeFactory(true) + ); + ColumnPartSerde columnPartSerde = jsonMapper.readValue( + jsonMapper.writeValueAsString(object), + ColumnPartSerde.class + ); + Assert.assertTrue(columnPartSerde instanceof FloatGenericColumnPartSerdeV2); + Assert.assertEquals( + object.getBitmapSerdeFactory(), + ((FloatGenericColumnPartSerdeV2) columnPartSerde).getBitmapSerdeFactory() + ); + + } +} diff --git a/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java new file mode 100644 index 000000000000..63c155995284 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.TestHelper; +import io.druid.segment.data.RoaringBitmapSerdeFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteOrder; + +public class LongGenericColumnPartSerdeTest +{ + private static ObjectMapper jsonMapper = TestHelper.getJsonMapper(); + + @Test + public void testSerdeV2() throws IOException + { + LongGenericColumnPartSerdeV2 object = LongGenericColumnPartSerdeV2.createDeserializer( + ByteOrder.BIG_ENDIAN, + new RoaringBitmapSerdeFactory(true) + ); + ColumnPartSerde columnPartSerde = jsonMapper.readValue( + jsonMapper.writeValueAsString(object), + ColumnPartSerde.class + ); + Assert.assertTrue(columnPartSerde instanceof LongGenericColumnPartSerdeV2); + Assert.assertEquals( + object.getBitmapSerdeFactory(), + ((LongGenericColumnPartSerdeV2) columnPartSerde).getBitmapSerdeFactory() + ); + + } +} From 2af1f7349fec4878a2738fde63d936c3ff84a3f4 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 24 Oct 2017 20:38:18 +0530 Subject: [PATCH 26/50] [BUG-90687] Fix null handling for extraction functions. Change-Id: Ie220530c4d549959a962569b0bb82b775ddf41d8 --- .../io/druid/query/extraction/BucketExtractionFn.java | 3 +-- .../io/druid/query/extraction/FunctionalExtraction.java | 3 +-- .../io/druid/query/extraction/IdentityExtractionFn.java | 6 +++--- .../java/io/druid/query/extraction/LowerExtractionFn.java | 3 ++- .../druid/query/extraction/MatchingDimExtractionFn.java | 4 ++-- .../query/extraction/SearchQuerySpecDimExtractionFn.java | 4 ++-- .../io/druid/query/extraction/StrlenExtractionFn.java | 4 ++++ .../druid/query/extraction/SubstringDimExtractionFn.java | 4 ++-- .../io/druid/query/extraction/TimeDimExtractionFn.java | 3 ++- .../io/druid/query/extraction/TimeFormatExtractionFn.java | 1 + .../java/io/druid/query/extraction/UpperExtractionFn.java | 3 ++- .../druid/query/extraction/FunctionalExtractionTest.java | 8 ++++---- .../io/druid/query/extraction/LowerExtractionFnTest.java | 1 + .../query/extraction/MatchingDimExtractionFnTest.java | 3 ++- .../io/druid/query/extraction/StrlenExtractionFnTest.java | 3 ++- .../io/druid/query/extraction/UpperExtractionFnTest.java | 3 ++- .../query/lookup/RegisteredLookupExtractionFnTest.java | 3 ++- 17 files changed, 35 insertions(+), 24 deletions(-) diff --git a/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java index 55fee2e237bb..54bf69648559 100644 --- a/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.primitives.Doubles; import io.druid.java.util.common.StringUtils; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -62,7 +63,6 @@ public String apply(@Nullable Object value) if (value == null) { return null; } - if (value instanceof Number) { return bucket(((Number) value).doubleValue()); } else if (value instanceof String) { @@ -78,7 +78,6 @@ public String apply(@Nullable String value) if (value == null) { return null; } - try { return bucket(Double.parseDouble(value)); } diff --git a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java index 7cae1c31d7ec..9ad5d0656ed5 100644 --- a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java +++ b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java @@ -21,7 +21,6 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; @@ -70,7 +69,7 @@ public FunctionalExtraction( public String apply(@Nullable String dimValue) { final String retval = extractionFunction.apply(dimValue); - return Strings.isNullOrEmpty(retval) ? Strings.emptyToNull(dimValue) : retval; + return NullHandlingHelper.isNullOrDefault(retval) ? NullHandlingHelper.defaultToNull(dimValue) : retval; } }; } else { diff --git a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java index 1bf700feaaf0..39fb96285e7f 100644 --- a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java @@ -19,7 +19,7 @@ package io.druid.query.extraction; -import com.google.common.base.Strings; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; @@ -42,14 +42,14 @@ public byte[] getCacheKey() @Nullable public String apply(@Nullable Object value) { - return value == null ? null : Strings.emptyToNull(value.toString()); + return value == null ? null : NullHandlingHelper.defaultToNull(value.toString()); } @Override @Nullable public String apply(@Nullable String value) { - return Strings.emptyToNull(value); + return NullHandlingHelper.defaultToNull(value); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java index da2a8e3bc702..f3366ac7e80d 100644 --- a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Strings; import io.druid.java.util.common.StringUtils; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -52,7 +53,7 @@ public LowerExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (Strings.isNullOrEmpty(key)) { + if (NullHandlingHelper.isNullOrDefault(key)) { return null; } return key.toLowerCase(locale); diff --git a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java index 4d121be031fc..851fe2290aa1 100644 --- a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import io.druid.java.util.common.StringUtils; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -62,7 +62,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (Strings.isNullOrEmpty(dimValue)) { + if (NullHandlingHelper.isNullOrDefault(dimValue)) { // We'd return null whether or not the pattern matched return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java index 1f005523925b..0217b8a33b8d 100644 --- a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import io.druid.query.search.SearchQuerySpec; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -64,7 +64,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - return searchQuerySpec.accept(dimValue) ? Strings.emptyToNull(dimValue) : null; + return searchQuerySpec.accept(dimValue) ? NullHandlingHelper.defaultToNull(dimValue) : null; } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java index 5ad88fa38fce..9528541a5b52 100644 --- a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java @@ -20,6 +20,7 @@ package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; @@ -40,6 +41,9 @@ public static StrlenExtractionFn instance() @Override public String apply(@Nullable String value) { + if (!NullHandlingHelper.useDefaultValuesForNull() && value == null) { + return null; + } return String.valueOf(value == null ? 0 : value.length()); } diff --git a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java index 2d38806ddd45..2b71984726e0 100644 --- a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import io.druid.java.util.common.StringUtils; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -65,7 +65,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (Strings.isNullOrEmpty(dimValue)) { + if (NullHandlingHelper.isNullOrDefault(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java index 0aa7fdeb8acf..ca85457dd84c 100644 --- a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java @@ -25,6 +25,7 @@ import com.google.common.base.Strings; import com.ibm.icu.text.SimpleDateFormat; import io.druid.java.util.common.StringUtils; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -77,7 +78,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (Strings.isNullOrEmpty(dimValue)) { + if (NullHandlingHelper.isNullOrDefault(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/TimeFormatExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/TimeFormatExtractionFn.java index 3a2153b19668..0310f94061b3 100644 --- a/processing/src/main/java/io/druid/query/extraction/TimeFormatExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/TimeFormatExtractionFn.java @@ -26,6 +26,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.granularity.Granularity; +import io.druid.segment.NullHandlingHelper; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.chrono.ISOChronology; diff --git a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java index 34368f7f0fd6..8c6ddd9f3c1f 100644 --- a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Strings; import io.druid.java.util.common.StringUtils; +import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -51,7 +52,7 @@ public UpperExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (Strings.isNullOrEmpty(key)) { + if (NullHandlingHelper.isNullOrDefault(key)) { return null; } return key.toUpperCase(locale); diff --git a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java index 493cb9e2b741..147e208d8f83 100644 --- a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java +++ b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java @@ -136,7 +136,7 @@ public void testRetainMissing() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandlingHelper.isNullOrDefault(out) ? in : out, exFn.apply(in)); } @Test @@ -150,7 +150,7 @@ public void testRetainMissingButFound() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandlingHelper.isNullOrDefault(out) ? in : out, exFn.apply(in)); } @Test @@ -165,7 +165,7 @@ public void testReplaceMissing() ); final String out = fn.apply(in); if (NullHandlingHelper.useDefaultValuesForNull()) { - Assert.assertEquals(Strings.isNullOrEmpty(out) ? MISSING : out, exFn.apply(in)); + Assert.assertEquals(NullHandlingHelper.isNullOrDefault(out) ? MISSING : out, exFn.apply(in)); } else { Assert.assertEquals(out == null ? MISSING : out, exFn.apply(in)); } @@ -217,7 +217,7 @@ public void testNullInputs() null, false ); - if (Strings.isNullOrEmpty(fn.apply(null))) { + if (NullHandlingHelper.isNullOrDefault(fn.apply(null))) { Assert.assertEquals(null, exFn.apply(null)); } } diff --git a/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java index 1a7727adbaf1..a5c9abae1fc5 100644 --- a/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java @@ -19,6 +19,7 @@ package io.druid.query.extraction; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; diff --git a/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java index 4415056f542a..887f5248dca4 100644 --- a/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -75,7 +76,7 @@ public void testNullExtraction() Assert.assertNull(extractionFn.apply((Object) null)); Assert.assertNull(extractionFn.apply((String) null)); - Assert.assertNull(extractionFn.apply((String) "")); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "" , extractionFn.apply((String) "")); } @Test diff --git a/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java index 08b102c6cfb2..60ff23b183d6 100644 --- a/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -29,7 +30,7 @@ public class StrlenExtractionFnTest @Test public void testApply() { - Assert.assertEquals("0", StrlenExtractionFn.instance().apply(null)); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? "0" : null, StrlenExtractionFn.instance().apply(null)); Assert.assertEquals("0", StrlenExtractionFn.instance().apply("")); Assert.assertEquals("1", StrlenExtractionFn.instance().apply("x")); Assert.assertEquals("3", StrlenExtractionFn.instance().apply("foo")); diff --git a/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java index 1d038c0674ac..d8b8ce692524 100644 --- a/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java @@ -19,6 +19,7 @@ package io.druid.query.extraction; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -31,7 +32,7 @@ public class UpperExtractionFnTest public void testApply() { Assert.assertEquals("UPPER", extractionFn.apply("uPpeR")); - Assert.assertEquals(null, extractionFn.apply("")); + Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); Assert.assertEquals(null, extractionFn.apply(null)); Assert.assertEquals(null, extractionFn.apply((Object) null)); Assert.assertEquals("1", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java b/processing/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java index f1332e21ec5f..571aeb5f169a 100644 --- a/processing/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java @@ -38,7 +38,8 @@ public class RegisteredLookupExtractionFnTest { private static Map MAP = ImmutableMap.of( "foo", "bar", - "bat", "baz" + "bat", "baz", + "", "empty" ); private static final LookupExtractor LOOKUP_EXTRACTOR = new MapLookupExtractor(MAP, true); private static final String LOOKUP_NAME = "some lookup"; From 4e3231d10e4c98aa926a7b6a41f5ae0f66986f4a Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 7 Nov 2017 01:56:59 +0530 Subject: [PATCH 27/50] review comments --- .../io/druid/math/expr/ExprListenerImpl.java | 5 +- .../DistinctCountAggregator.java | 15 ++-- .../DistinctCountBufferAggregator.java | 8 ++- .../TimestampAggregatorFactory.java | 2 +- .../druid/java/util/common/StringUtils.java | 9 ++- pom.xml | 2 +- processing/pom.xml | 2 +- .../druid/query/aggregation/Aggregator.java | 6 +- .../query/aggregation/BufferAggregator.java | 70 ++++++++++--------- .../segment/NullValueHandlingConfig.java | 7 +- 10 files changed, 68 insertions(+), 58 deletions(-) diff --git a/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java index b74e69df1fe5..5b57cefadaa1 100644 --- a/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java +++ b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java @@ -337,9 +337,6 @@ public void exitFunctionArgs(ExprParser.FunctionArgsContext ctx) @Override public void exitNull(ExprParser.NullContext ctx) { - nodes.put( - ctx, - new StringExpr(null) - ); + nodes.put(ctx, new StringExpr(null)); } } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java index 27bdad5e118b..31d860359c69 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java @@ -23,7 +23,6 @@ import io.druid.query.aggregation.Aggregator; import io.druid.segment.DimensionSelector; import io.druid.segment.NullHandlingHelper; -import io.druid.segment.data.IndexedInts; public class DistinctCountAggregator implements Aggregator { @@ -43,11 +42,13 @@ public DistinctCountAggregator( @Override public void aggregate() { - IndexedInts row = selector.getRow(); - for (int i = 0; i < row.size(); i++) { - int index = row.get(i); - mutableBitmap.add(index); - } + + boolean countNulls = !selector.nameLookupPossibleInAdvance() || NullHandlingHelper.useDefaultValuesForNull(); + selector.getRow().forEach(index -> { + if (countNulls || selector.lookupName(index) != null) { + mutableBitmap.add(index); + } + }); } @Override @@ -88,7 +89,7 @@ public double getDouble() private int countValues() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (!selector.nameLookupPossibleInAdvance() || NullHandlingHelper.useDefaultValuesForNull()) { return mutableBitmap.size(); } int nullId = selector.idLookup().lookupId(null); diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index 7f3947b35ca8..0f15d13b28c7 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -52,11 +52,15 @@ public void init(ByteBuffer buf, int position) @Override public void aggregate(ByteBuffer buf, int position) { + boolean countNulls = !selector.nameLookupPossibleInAdvance() || NullHandlingHelper.useDefaultValuesForNull(); MutableBitmap mutableBitmap = getMutableBitmap(buf, position); IndexedInts row = selector.getRow(); for (int i = 0; i < row.size(); i++) { + int index = row.get(i); - mutableBitmap.add(index); + if (countNulls || selector.lookupName(index) != null) { + mutableBitmap.add(index); + } } buf.putLong(position, mutableBitmap.size()); } @@ -110,7 +114,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) private long countValues(ByteBuffer buf, int position) { MutableBitmap mutableBitmap = getMutableBitmap(buf, position); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (!selector.nameLookupPossibleInAdvance() || NullHandlingHelper.useDefaultValuesForNull()) { return mutableBitmap.size(); } int nullId = selector.idLookup().lookupId(null); diff --git a/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java b/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java index 6f7b2bd8c13c..46afdc470372 100644 --- a/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java +++ b/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java @@ -168,7 +168,7 @@ public Object deserialize(Object object) @Override @Nullable - public Object finalizeComputation(Object object) + public Object finalizeComputation(@Nullable Object object) { return object == null ? null : DateTimes.utc((long) object); } diff --git a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java index bdf6f14a5f4a..72738bb4a49c 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java @@ -79,7 +79,7 @@ public static String fromUtf8(final byte[] bytes) } @Nullable - public static String fromUtf8Nullable(final byte[] bytes) + public static String fromUtf8Nullable(@Nullable final byte[] bytes) { if (bytes == null) { return null; @@ -91,15 +91,18 @@ public static String fromUtf8(final ByteBuffer buffer, final int numBytes) { final byte[] bytes = new byte[numBytes]; buffer.get(bytes); - return StringUtils.fromUtf8(bytes); + return fromUtf8(bytes); } @Nullable public static String fromUtf8Nullable(final ByteBuffer buffer, final int numBytes) { + if(numBytes < 0){ + return null; + } final byte[] bytes = new byte[numBytes]; buffer.get(bytes); - return StringUtils.fromUtf8Nullable(bytes); + return fromUtf8(bytes); } public static String fromUtf8(final ByteBuffer buffer) diff --git a/pom.xml b/pom.xml index ebca7ed69aed..258f4e28b4c0 100644 --- a/pom.xml +++ b/pom.xml @@ -79,8 +79,8 @@ 1.10.77 - true 2.5.5 + true diff --git a/processing/pom.xml b/processing/pom.xml index 6e0808c78298..54963de0fb8c 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.11.1-SNAPSHOT + 0.11.1-SNAPSHOT diff --git a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java index cac559359fb9..d2978e2e0422 100644 --- a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java @@ -28,7 +28,7 @@ * An Aggregator is an object that can aggregate metrics. Its aggregation-related methods (namely, aggregate() and get()) * do not take any arguments as the assumption is that the Aggregator was given something in its constructor that * it can use to get at the next bit of data. - * + *

* Thus, an Aggregator can be thought of as a closure over some other thing that is stateful and changes between calls * to aggregate(). This is currently (as of this documentation) implemented through the use of {@link * io.druid.segment.ColumnValueSelector} objects. @@ -37,11 +37,14 @@ public interface Aggregator extends Closeable { void aggregate(); + void reset(); @Nullable Object get(); + float getFloat(); + long getLong(); /** @@ -57,6 +60,7 @@ default double getDouble() /** * returns true if the Aggregator supports returning null values and the aggregated value is Null. * The default implementation always return false to enable smooth backward compatibility, re-implement if your aggregator is nullable. + * For backwards compatibility, isNull() may return false even if get() returns null. Users of this method should account for this case. */ default boolean isNull() { diff --git a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java index f85ec81f6039..6b36d4fd10b9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java @@ -30,7 +30,7 @@ * A BufferAggregator is an object that can aggregate metrics into a ByteBuffer. Its aggregation-related methods * (namely, aggregate(...) and get(...)) only take the ByteBuffer and position because it is assumed that the Aggregator * was given something (one or more MetricSelector(s)) in its constructor that it can use to get at the next bit of data. - * + *

* Thus, an Aggregator can be thought of as a closure over some other thing that is stateful and changes between calls * to aggregate(...). */ @@ -39,15 +39,15 @@ public interface BufferAggregator extends HotLoopCallee { /** * Initializes the buffer location - * + *

* Implementations of this method must initialize the byte buffer at the given position - * + *

* Implementations must not change the position, limit or mark of the given buffer - * + *

* This method must not exceed the number of bytes returned by {@link AggregatorFactory#getMaxIntermediateSize()} * in the corresponding {@link AggregatorFactory} * - * @param buf byte buffer to initialize + * @param buf byte buffer to initialize * @param position offset within the byte buffer for initialization */ @CalledFromHotLoop @@ -55,13 +55,13 @@ public interface BufferAggregator extends HotLoopCallee /** * Aggregates metric values into the given aggregate byte representation - * + *

* Implementations of this method must read in the aggregate value from the buffer at the given position, * aggregate the next element of data and write the updated aggregate value back into the buffer. - * + *

* Implementations must not change the position, limit or mark of the given buffer * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the current aggregate value is stored */ @CalledFromHotLoop @@ -69,68 +69,72 @@ public interface BufferAggregator extends HotLoopCallee /** * Returns the intermediate object representation of the given aggregate. - * + *

* Converts the given byte buffer representation into an intermediate aggregate Object - * + *

* Implementations must not change the position, limit or mark of the given buffer * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the aggregate value is stored + * * @return the Object representation of the aggregate */ Object get(ByteBuffer buf, int position); /** * Returns the float representation of the given aggregate byte array - * + *

* Converts the given byte buffer representation into the intermediate aggregate value. - * + *

* Implementations must not change the position, limit or mark of the given buffer - * + *

* Implementations are only required to support this method if they are aggregations which * have an {@link AggregatorFactory#getTypeName()} of "float". * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the aggregate value is stored + * * @return the float representation of the aggregate */ float getFloat(ByteBuffer buf, int position); /** * Returns the long representation of the given aggregate byte array - * + *

* Converts the given byte buffer representation into the intermediate aggregate value. - * + *

* Implementations must not change the position, limit or mark of the given buffer - * + *

* Implementations are only required to support this method if they are aggregations which * have an {@link AggregatorFactory#getTypeName()} of "long". * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the aggregate value is stored + * * @return the long representation of the aggregate */ long getLong(ByteBuffer buf, int position); /** * Returns the double representation of the given aggregate byte array - * + *

* Converts the given byte buffer representation into the intermediate aggregate value. - * + *

* Implementations must not change the position, limit or mark of the given buffer - * + *

* Implementations are only required to support this method if they are aggregations which * have an {@link AggregatorFactory#getTypeName()} of "double". * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. - * + *

* The default implementation casts {@link BufferAggregator#getFloat(ByteBuffer, int)} to double. * This default method is added to enable smooth backward compatibility, please re-implement it if your aggregators * work with numeric double columns. * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the aggregate value is stored + * * @return the double representation of the aggregate */ default double getDouble(ByteBuffer buf, int position) @@ -145,7 +149,7 @@ default double getDouble(ByteBuffer buf, int position) /** * {@inheritDoc} - * + *

*

The default implementation inspects nothing. Classes that implement {@code BufferAggregator} are encouraged to * override this method, following the specification of {@link HotLoopCallee#inspectRuntimeShape}. */ @@ -161,18 +165,18 @@ default void inspectRuntimeShape(RuntimeShapeInspector inspector) * built on top of old ByteBuffer can not be used for further {@link BufferAggregator#aggregate(ByteBuffer, int)} * calls. This method tells the BufferAggregator that the cached objects at a certain location has been relocated to * a different location. - * + *

* Only used if there is any positional caches/objects in the BufferAggregator implementation. - * + *

* If relocate happens to be across multiple new ByteBuffers (say n ByteBuffers), this method should be called * multiple times(n times) given all the new positions/old positions should exist in newBuffer/OldBuffer. - * + *

* Implementations must not change the position, limit or mark of the given buffer * * @param oldPosition old position of a cached object before aggregation buffer relocates to a new ByteBuffer. * @param newPosition new position of a cached object after aggregation buffer relocates to a new ByteBuffer. - * @param oldBuffer old aggregation buffer. - * @param newBuffer new aggregation buffer. + * @param oldBuffer old aggregation buffer. + * @param newBuffer new aggregation buffer. */ default void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) { @@ -182,10 +186,7 @@ default void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, By * Returns true if the aggregator is nullable and the aggregated value is null *

* Implementations must not change the position, limit or mark of the given buffer - *

- * Implementations are only required to support this method if they the aggregator supports null values. - * If it doesn't support null always returning false is recommended. - *

+ * * The default implementation always returns false. * This default method is added to enable smooth backward compatibility, please re-implement it if your aggregators * support null values @@ -194,6 +195,7 @@ default void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, By * @param position offset within the byte buffer at which the aggregate value is stored * * @return true if the aggrgeated value is null otherwise false. + * For backwards compatibility, isNull() may return false even if get() returns null. Users of this method should account for this case. */ default boolean isNull(ByteBuffer buf, int position) { diff --git a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java b/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java index f1c6d4f0c835..7e457534bbf8 100644 --- a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java +++ b/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java @@ -31,10 +31,9 @@ public class NullValueHandlingConfig @JsonCreator public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) { -// this.useDefaultValuesForNull = useDefaultValuesForNull == null -// ? true -// : useDefaultValuesForNull; - this.useDefaultValuesForNull = false; + this.useDefaultValuesForNull = useDefaultValuesForNull == null + ? true + : useDefaultValuesForNull; } public boolean isUseDefaultValuesForNull() From 909a3d8dcfb3c1c756691f5b77a7bcc9c5e4eac2 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 7 Nov 2017 02:12:33 +0530 Subject: [PATCH 28/50] more review comments --- .../io/druid/query/aggregation/BufferAggregator.java | 2 +- .../java/io/druid/segment/column/ColumnBuilder.java | 10 +--------- .../main/java/io/druid/segment/column/LongColumn.java | 5 +---- .../java/io/druid/segment/column/SimpleColumn.java | 5 +---- .../segment/serde/DoubleGenericColumnPartSerdeV2.java | 1 - .../segment/serde/FloatGenericColumnPartSerdeV2.java | 1 - .../segment/serde/LongGenericColumnPartSerdeV2.java | 1 - 7 files changed, 4 insertions(+), 21 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java index 6b36d4fd10b9..2c253ab1d810 100644 --- a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java @@ -186,7 +186,7 @@ default void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, By * Returns true if the aggregator is nullable and the aggregated value is null *

* Implementations must not change the position, limit or mark of the given buffer - * + * * The default implementation always returns false. * This default method is added to enable smooth backward compatibility, please re-implement it if your aggregators * support null values diff --git a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java index fb531bbbc710..e8fa19bcc69d 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java @@ -38,7 +38,6 @@ public class ColumnBuilder private Supplier bitmapIndex = null; private Supplier spatialIndex = null; private SmooshedFileMapper fileMapper = null; - private Supplier nullValueBitmap = null; public ColumnBuilder setFileMapper(SmooshedFileMapper fileMapper) { @@ -99,12 +98,6 @@ public ColumnBuilder setSpatialIndex(Supplier spatialIndex) return this; } - public ColumnBuilder setNullValueBitmap(Supplier nullValueBitmap) - { - this.nullValueBitmap = nullValueBitmap; - return this; - } - public Column build() { Preconditions.checkState(type != null, "Type must be set."); @@ -122,8 +115,7 @@ public Column build() genericColumn, complexColumn, bitmapIndex, - spatialIndex, - nullValueBitmap + spatialIndex ); } } diff --git a/processing/src/main/java/io/druid/segment/column/LongColumn.java b/processing/src/main/java/io/druid/segment/column/LongColumn.java index d481275de212..bd81804b1f13 100644 --- a/processing/src/main/java/io/druid/segment/column/LongColumn.java +++ b/processing/src/main/java/io/druid/segment/column/LongColumn.java @@ -31,9 +31,6 @@ public class LongColumn extends AbstractColumn private static final ColumnCapabilitiesImpl CAPABILITIES = new ColumnCapabilitiesImpl() .setType(ValueType.LONG); - private static final ColumnCapabilitiesImpl CAPABILITIES_WITH_NULL = new ColumnCapabilitiesImpl() - .setType(ValueType.LONG); - private final CompressedLongsIndexedSupplier column; private final ImmutableBitmap nullValueBitmap; @@ -46,7 +43,7 @@ public LongColumn(CompressedLongsIndexedSupplier column, ImmutableBitmap nullVal @Override public ColumnCapabilities getCapabilities() { - return nullValueBitmap.isEmpty() ? CAPABILITIES : CAPABILITIES_WITH_NULL; + return CAPABILITIES; } @Override diff --git a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java index 2017d58c1774..3e79cbc8b2ca 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java @@ -33,7 +33,6 @@ class SimpleColumn implements Column private final Supplier complexColumn; private final Supplier bitmapIndex; private final Supplier spatialIndex; - private final Supplier nullValueBitmap; SimpleColumn( ColumnCapabilities capabilities, @@ -42,8 +41,7 @@ class SimpleColumn implements Column Supplier genericColumn, Supplier complexColumn, Supplier bitmapIndex, - Supplier spatialIndex, - Supplier nullValueBitmap + Supplier spatialIndex ) { this.capabilities = capabilities; @@ -53,7 +51,6 @@ class SimpleColumn implements Column this.complexColumn = complexColumn; this.bitmapIndex = bitmapIndex; this.spatialIndex = spatialIndex; - this.nullValueBitmap = nullValueBitmap; } @Override diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java index 296f5e25f5b3..e72428424961 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -101,7 +101,6 @@ public Deserializer getDeserializer() final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); - builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java index b76289cc4ae2..23c8da9e1a62 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -153,7 +153,6 @@ public Deserializer getDeserializer() final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); - builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java index ac0ede4ea5a6..2490734adbb3 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -151,7 +151,6 @@ public Deserializer getDeserializer() final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); - builder.setNullValueBitmap(Suppliers.ofInstance(bitmap)); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } From 1c8fae0f6fdd64ce47482d25a726185eaccf2e72 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 8 Nov 2017 00:38:38 +0530 Subject: [PATCH 29/50] review comments --- .../config/NullHandlingExpressionHelper.java | 77 +++++++++++ .../config}/NullValueHandlingConfig.java | 2 +- .../main/java/io/druid/math/expr/Expr.java | 13 +- .../java/io/druid/math/expr/ExprEval.java | 3 +- .../java/io/druid/math/expr/Function.java | 10 +- .../druid/java/util/common/StringUtils.java | 2 +- .../io/druid/guice/NullHandlingModule.java | 5 +- .../io/druid/segment/NullHandlingHelper.java | 1 + .../calcite/expression/DruidExpression.java | 2 +- .../druid/sql/calcite/planner/Calcites.java | 4 +- .../io/druid/sql/calcite/rel/DruidRel.java | 1 - .../druid/sql/calcite/rel/DruidSemiJoin.java | 7 +- .../druid/sql/calcite/CalciteQueryTest.java | 127 +++++------------- 13 files changed, 138 insertions(+), 116 deletions(-) create mode 100644 common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java rename {processing/src/main/java/io/druid/segment => common/src/main/java/io/druid/common/config}/NullValueHandlingConfig.java (97%) diff --git a/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java b/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java new file mode 100644 index 000000000000..20426d620879 --- /dev/null +++ b/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java @@ -0,0 +1,77 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.common.config; + +import com.google.common.base.Strings; +import com.google.inject.Inject; + +public class NullHandlingExpressionHelper +{ + private static String NULL_HANDLING_CONFIG_STRING = "druid.null.handling.useDefaultValueForNull"; + + // use these values to ensure that convertObjectToLong(), convertObjectToDouble() and convertObjectToFloat() + // return the same boxed object when returning a constant zero. + public static final Double ZERO_DOUBLE = 0.0d; + public static final Float ZERO_FLOAT = 0.0f; + public static final Long ZERO_LONG = 0L; + + // INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. + // See NullHandlingModule for details. + // The default system property is supposed to be used only in tests. + @Inject + private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( + Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) + ); + + public static boolean useDefaultValuesForNull() + { + return INSTANCE.isUseDefaultValuesForNull(); + } + + public static String nullToDefault(String value) + { + return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; + } + + public static String defaultToNull(String value) + { + return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; + } + + public static boolean isNullOrDefault(String value) + { + return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; + } + + public static Long nullToDefault(Long value) + { + return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_LONG : value; + } + + public static Double nullToDefault(Double value) + { + return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_DOUBLE : value; + } + + public static Float nullToDefault(Float value) + { + return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_FLOAT : value; + } +} diff --git a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java b/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java similarity index 97% rename from processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java rename to common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java index 7e457534bbf8..fdd1638b510e 100644 --- a/processing/src/main/java/io/druid/segment/NullValueHandlingConfig.java +++ b/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.segment; +package io.druid.common.config; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index c7d05a0bd93c..db6012767a33 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.math.LongMath; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandlingExpressionHelper; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Comparators; @@ -361,6 +362,13 @@ public ExprEval eval(ObjectBinding bindings) { ExprEval leftVal = left.eval(bindings); ExprEval rightVal = right.eval(bindings); + + // Result of any Binary expressions is null if any of the argument is null. + // e.g "select null * 2 as c;" or "select null + 1 as c;" will return null as per Standard SQL spec. + if (!NullHandlingExpressionHelper.useDefaultValuesForNull() && (leftVal.isNull() || rightVal.isNull())) { + return ExprEval.of(null); + } + if (leftVal.type() == ExprType.STRING && rightVal.type() == ExprType.STRING) { return evalString(leftVal.asString(), rightVal.asString()); } else if (leftVal.type() == ExprType.LONG && rightVal.type() == ExprType.LONG) { @@ -490,11 +498,6 @@ class BinPlusExpr extends BinaryEvalOpExprBase @Override protected ExprEval evalString(@Nullable String left, @Nullable String right) { - // Result of expression is Null if any of the argument is null. - // e.g "select null * 2 as c;" will return null as per Standard SQL spec. - if (left == null || right == null) { - return ExprEval.of(null); - } return ExprEval.of(left + right); } diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index ed8b5bfffc00..181733c9c99e 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandlingExpressionHelper; import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.IAE; @@ -232,7 +233,7 @@ private static class StringExprEval extends ExprEval { private StringExprEval(String value) { - super(value); + super(NullHandlingExpressionHelper.defaultToNull(value)); } @Override diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index e6880f1a25a3..5108138dec75 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -20,6 +20,7 @@ package io.druid.math.expr; import com.google.common.base.Strings; +import io.druid.common.config.NullHandlingExpressionHelper; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.StringUtils; @@ -74,6 +75,9 @@ abstract class SingleParamMath extends SingleParam @Override protected final ExprEval eval(ExprEval param) { + if (!NullHandlingExpressionHelper.useDefaultValuesForNull() && param.isNull()) { + return ExprEval.of(null); + } if (param.type() == ExprType.LONG) { return eval(param.asLong()); } else if (param.type() == ExprType.DOUBLE) { @@ -899,12 +903,12 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final StringBuilder builder = new StringBuilder(Strings.nullToEmpty(args.get(0).eval(bindings).asString())); for (int i = 1; i < args.size(); i++) { final String s = args.get(i).eval(bindings).asString(); - if (s != null) { - builder.append(s); - } else { + if (!NullHandlingExpressionHelper.useDefaultValuesForNull() && s == null) { // Result of concatenation is null if any of the Values is null. // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. return ExprEval.of(null); + } else { + builder.append(s); } } return ExprEval.of(builder.toString()); diff --git a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java index 72738bb4a49c..0823961f39f8 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java @@ -102,7 +102,7 @@ public static String fromUtf8Nullable(final ByteBuffer buffer, final int numByte } final byte[] bytes = new byte[numBytes]; buffer.get(bytes); - return fromUtf8(bytes); + return fromUtf8Nullable(bytes); } public static String fromUtf8(final ByteBuffer buffer) diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java index ba6ca343098d..2d70ce6ccfcc 100644 --- a/processing/src/main/java/io/druid/guice/NullHandlingModule.java +++ b/processing/src/main/java/io/druid/guice/NullHandlingModule.java @@ -21,8 +21,9 @@ import com.google.inject.Binder; import com.google.inject.Module; +import io.druid.common.config.NullHandlingExpressionHelper; import io.druid.segment.NullHandlingHelper; -import io.druid.segment.NullValueHandlingConfig; +import io.druid.common.config.NullValueHandlingConfig; /** */ @@ -33,5 +34,7 @@ public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.null.handling", NullValueHandlingConfig.class); binder.requestStaticInjection(NullHandlingHelper.class); + binder.requestStaticInjection(NullHandlingExpressionHelper.class); + } } diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index bdcdd484f24b..f1ed3a4204a9 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -22,6 +22,7 @@ import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.inject.Inject; +import io.druid.common.config.NullValueHandlingConfig; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.BufferAggregator; diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java index 8df6075aab1c..814636fa50b6 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java @@ -91,7 +91,7 @@ public static String stringLiteral(final String s) public static String nullLiteral() { - return NullHandlingHelper.useDefaultValuesForNull() ? "''" : null; + return "null"; } public static String functionCall(final String functionName, final List args) diff --git a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java index 29f71738d104..bab3b1d73140 100644 --- a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java +++ b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java @@ -19,6 +19,7 @@ package io.druid.sql.calcite.planner; +import com.google.common.base.Preconditions; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; import io.druid.java.util.common.DateTimes; @@ -105,8 +106,9 @@ public static SchemaPlus createRootSchema(final Schema druidSchema, final Author public static String escapeStringLiteral(final String s) { + Preconditions.checkNotNull(s); if (s == null) { - return DruidExpression.nullLiteral(); + return "''"; } else { boolean isPlainAscii = true; final StringBuilder builder = new StringBuilder("'"); diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/DruidRel.java b/sql/src/main/java/io/druid/sql/calcite/rel/DruidRel.java index 8056d9a7738b..4fc4212623bb 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/DruidRel.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/DruidRel.java @@ -87,7 +87,6 @@ public boolean isValidDruidQuery() * Convert this DruidRel to a DruidQuery for purposes of explaining. This must be an inexpensive operation. For * example, DruidSemiJoin will use a dummy dataSource in order to complete this method, rather than executing * the right-hand side query. - *

* This method may not return null. * * @return query diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java index 4fb12d921779..8b68c071aa48 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java @@ -309,11 +309,8 @@ public List accumulate(final List theConditions, final Object[ for (int i = 0; i < values.size(); i++) { final String value = values.get(i); - if (value == null) { - subConditions.add( - getCluster().getRexBuilder() - .makeCall(SqlStdOperatorTable.IS_NULL, leftExpressions.get(i))); - } else { + // NULLS are not supposed to match NULLs in a join. So ignore them. + if (value != null) { subConditions.add( getCluster().getRexBuilder().makeCall( SqlStdOperatorTable.EQUALS, diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index db0600b32b93..2ed7d1c6b77c 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -479,6 +479,7 @@ public void testExplainInformationSchemaColumns() throws Exception @Test public void testSelectStar() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT * FROM druid.foo", ImmutableList.of( @@ -490,22 +491,13 @@ public void testSelectStar() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", nullValue, 2f, 2.0, HLLCV1.class.getName()}, new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} - ) : - ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", null, 2f, 2.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", null, 6f, 6.0, HLLCV1.class.getName()} + new Object[]{T("2001-01-03"), 1L, "abc", nullValue, 6f, 6.0, HLLCV1.class.getName()} ) ); } @@ -574,6 +566,8 @@ public void testExplainSelectStar() throws Exception @Test public void testSelectStarWithLimit() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + testQuery( "SELECT * FROM druid.foo LIMIT 2", ImmutableList.of( @@ -586,14 +580,9 @@ public void testSelectStarWithLimit() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2.0f, 2.0, HLLCV1.class.getName()} - ) : - ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", null, 2.0f, 2.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-02"), 1L, "10.1", nullValue, 2.0f, 2.0, HLLCV1.class.getName()} ) ); } @@ -601,6 +590,7 @@ public void testSelectStarWithLimit() throws Exception @Test public void testSelectWithProjection() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT SUBSTRING(dim2, 1, 1) FROM druid.foo LIMIT 2", ImmutableList.of( @@ -616,14 +606,9 @@ public void testSelectWithProjection() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"a"}, - new Object[]{""} - ) : - ImmutableList.of( - new Object[]{"a"}, - new Object[]{null} + new Object[]{nullValue} ) ); } @@ -631,6 +616,8 @@ public void testSelectWithProjection() throws Exception @Test public void testSelectStarWithLimitTimeDescending() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + testQuery( "SELECT * FROM druid.foo ORDER BY __time DESC LIMIT 2", ImmutableList.of( @@ -645,13 +632,8 @@ public void testSelectStarWithLimitTimeDescending() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6d, HLLCV1.class.getName()}, - new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5d, HLLCV1.class.getName()} - ) : - ImmutableList.of( - new Object[]{T("2001-01-03"), 1L, "abc", null, 6f, 6d, HLLCV1.class.getName()}, + new Object[]{T("2001-01-03"), 1L, "abc", nullValue, 6f, 6d, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5d, HLLCV1.class.getName()} ) ); @@ -660,6 +642,7 @@ public void testSelectStarWithLimitTimeDescending() throws Exception @Test public void testSelectStarWithoutLimitTimeAscending() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT * FROM druid.foo ORDER BY __time", ImmutableList.of( @@ -690,22 +673,13 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? - ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} - ) : ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", null, 2f, 2.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", nullValue, 2f, 2.0, HLLCV1.class.getName()}, new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", null, 6f, 6.0, HLLCV1.class.getName()} + new Object[]{T("2001-01-03"), 1L, "abc", nullValue, 6f, 6.0, HLLCV1.class.getName()} ) ); } @@ -713,6 +687,7 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception @Test public void testSelectSingleColumnTwice() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT dim2 x, dim2 y FROM druid.foo LIMIT 2", ImmutableList.of( @@ -725,14 +700,9 @@ public void testSelectSingleColumnTwice() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"a", "a"}, - new Object[]{"", ""} - ) : - ImmutableList.of( - new Object[]{"a", "a"}, - new Object[]{null, null} + new Object[]{nullValue, nullValue} ) ); } @@ -1523,6 +1493,7 @@ public void testTopNWithSelectAndOrderByProjections() throws Exception @Test public void testGroupByCaseWhen() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" + " CASE EXTRACT(DAY FROM __time)\n" @@ -1562,14 +1533,8 @@ public void testGroupByCaseWhen() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( - new Object[]{"", 2L}, - new Object[]{"match-cnt", 1L}, - new Object[]{"match-m1 ", 3L} - ) : - ImmutableList.of( - new Object[]{null, 2L}, + new Object[]{nullValue, 2L}, new Object[]{"match-cnt", 1L}, new Object[]{"match-m1 ", 3L} ) @@ -4646,6 +4611,7 @@ public void testSillyQuarters() throws Exception @Test public void testRegexpExtract() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT DISTINCT\n" + " REGEXP_EXTRACT(dim1, '^.'),\n" @@ -4681,16 +4647,8 @@ public void testRegexpExtract() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? - ImmutableList.of( - new Object[]{"", ""}, - new Object[]{"1", "1"}, - new Object[]{"2", "2"}, - new Object[]{"a", "a"}, - new Object[]{"d", "d"} - ) : ImmutableList.of( - new Object[]{null, null}, + new Object[]{nullValue, nullValue}, new Object[]{"1", "1"}, new Object[]{"2", "2"}, new Object[]{"a", "a"}, @@ -4702,6 +4660,7 @@ public void testRegexpExtract() throws Exception @Test public void testGroupBySortPushDown() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT dim2, dim1, SUM(cnt) FROM druid.foo GROUP BY dim2, dim1 ORDER BY dim1 LIMIT 4", ImmutableList.of( @@ -4731,17 +4690,10 @@ public void testGroupBySortPushDown() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? - ImmutableList.of( - new Object[]{"a", "", 1L}, - new Object[]{"a", "1", 1L}, - new Object[]{"", "10.1", 1L}, - new Object[]{"", "2", 1L} - ) : ImmutableList.of( new Object[]{"a", "", 1L}, new Object[]{"a", "1", 1L}, - new Object[]{null, "10.1", 1L}, + new Object[]{nullValue, "10.1", 1L}, new Object[]{"", "2", 1L} ) ); @@ -4750,6 +4702,7 @@ public void testGroupBySortPushDown() throws Exception @Test public void testGroupByLimitPushDownWithHavingOnLong() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT dim1, dim2, SUM(cnt) AS thecnt " + "FROM druid.foo " @@ -4785,16 +4738,9 @@ public void testGroupByLimitPushDownWithHavingOnLong() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? - ImmutableList.of( - new Object[]{"10.1", "", 1L}, - new Object[]{"2", "", 1L}, - new Object[]{"abc", "", 1L}, - new Object[]{"", "a", 1L} - ) : ImmutableList.of( - new Object[]{"10.1", null, 1L}, - new Object[]{"abc", null, 1L}, + new Object[]{"10.1", nullValue, 1L}, + new Object[]{"abc", nullValue, 1L}, new Object[]{"2", "", 1L}, new Object[]{"", "a", 1L} ) @@ -5247,6 +5193,7 @@ public void testGroupByStringLength() throws Exception @Test public void testFilterAndGroupByLookup() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; final RegisteredLookupExtractionFn extractionFn = new RegisteredLookupExtractionFn( null, "lookyloo", @@ -5290,13 +5237,8 @@ public void testFilterAndGroupByLookup() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? - ImmutableList.of( - new Object[]{"", 5L}, - new Object[]{"xabc", 1L} - ) : ImmutableList.of( - new Object[]{null, 5L}, + new Object[]{nullValue, 5L}, new Object[]{"xabc", 1L} ) ); @@ -6244,6 +6186,8 @@ public void testUsingSubqueryAsFilterOnTwoColumns() throws Exception @Test public void testUsingSubqueryAsFilterWithInnerSort() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + // Regression test for https://github.com/druid-io/druid/issues/4208 List expectedSubqueryResult = NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of("", "a", "abc") : @@ -6285,22 +6229,13 @@ public void testUsingSubqueryAsFilterWithInnerSort() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? - ImmutableList.of( - new Object[]{"", "a"}, - new Object[]{"10.1", ""}, - new Object[]{"2", ""}, - new Object[]{"1", "a"}, - new Object[]{"def", "abc"}, - new Object[]{"abc", ""} - ) : ImmutableList.of( new Object[]{"", "a"}, - new Object[]{"10.1", null}, + new Object[]{"10.1", nullValue}, new Object[]{"2", ""}, new Object[]{"1", "a"}, new Object[]{"def", "abc"}, - new Object[]{"abc", null} + new Object[]{"abc", nullValue} ) ); } From 7a0a8a9c8dcda139516811ed0583459756cc06b4 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 8 Nov 2017 20:18:43 +0530 Subject: [PATCH 30/50] Expression support IS_NULL and IS_NOT_NULL --- .../config/NullValueHandlingConfig.java | 2 +- .../main/java/io/druid/math/expr/Expr.java | 7 +- .../java/io/druid/math/expr/Function.java | 54 +++++++++++++-- .../java/io/druid/math/expr/EvalTest.java | 9 ++- .../java/io/druid/math/expr/FunctionTest.java | 22 +++++- .../druid/java/util/common/StringUtils.java | 2 +- .../druid/segment/column/ColumnBuilder.java | 1 - .../io/druid/segment/column/SimpleColumn.java | 1 - .../serde/DoubleGenericColumnPartSerdeV2.java | 1 - .../serde/FloatGenericColumnPartSerdeV2.java | 1 - .../serde/LongGenericColumnPartSerdeV2.java | 1 - .../io/druid/query/SchemaEvolutionTest.java | 4 +- .../segment/filter/ExpressionFilterTest.java | 20 ++++-- .../virtual/ExpressionVirtualColumnTest.java | 68 +++++++++++++++---- .../druid/query/expression/ExprMacroTest.java | 16 +++-- .../calcite/expression/DruidExpression.java | 1 - .../UnaryFunctionOperatorConversion.java | 66 ++++++++++++++++++ .../druid/sql/calcite/planner/Calcites.java | 1 - .../calcite/planner/DruidOperatorTable.java | 6 +- .../druid/sql/calcite/CalciteQueryTest.java | 23 ++++--- .../sql/calcite/planner/CalcitesTest.java | 2 - 21 files changed, 244 insertions(+), 64 deletions(-) create mode 100644 sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java diff --git a/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java b/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java index fdd1638b510e..be8e7f08101b 100644 --- a/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java +++ b/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java @@ -32,7 +32,7 @@ public class NullValueHandlingConfig public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) { this.useDefaultValuesForNull = useDefaultValuesForNull == null - ? true + ? false : useDefaultValuesForNull; } diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index db6012767a33..fa2ce3cd5058 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -31,6 +31,7 @@ import javax.annotation.Nullable; import java.util.List; import java.util.Objects; +import java.util.function.UnaryOperator; /** */ @@ -124,7 +125,7 @@ class StringExpr extends ConstantExpr public StringExpr(String value) { - this.value = value; + this.value = NullHandlingExpressionHelper.defaultToNull(value); } @Nullable @@ -498,7 +499,8 @@ class BinPlusExpr extends BinaryEvalOpExprBase @Override protected ExprEval evalString(@Nullable String left, @Nullable String right) { - return ExprEval.of(left + right); + return ExprEval.of(NullHandlingExpressionHelper.nullToDefault(left) + + NullHandlingExpressionHelper.nullToDefault(right)); } @Override @@ -705,3 +707,4 @@ public ExprEval eval(ObjectBinding bindings) return leftVal.asBoolean() ? leftVal : right.eval(bindings); } } + diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index 5108138dec75..6014caff0483 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -753,7 +753,7 @@ public ExprEval apply(final List args, final Expr.ObjectBinding bindings) return ExprEval.of(null); } } - + /** * "Simple CASE" function, similar to {@code CASE expr WHEN value THEN result [ELSE else_result] END} in SQL. */ @@ -908,7 +908,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. return ExprEval.of(null); } else { - builder.append(s); + builder.append(NullHandlingExpressionHelper.nullToDefault(s)); } } return ExprEval.of(builder.toString()); @@ -992,7 +992,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } else { // If starting index of substring is greater then the lengh of string, the result will be a zero length string. // e.g. 'select substring("abc", 4,5) as c;' will return an empty string - return ExprEval.of(""); + return ExprEval.of(NullHandlingExpressionHelper.nullToDefault((String) null)); } } } @@ -1016,10 +1016,10 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String pattern = args.get(1).eval(bindings).asString(); final String replacement = args.get(2).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(null); + return ExprEval.of(NullHandlingExpressionHelper.nullToDefault((String) null)); } return ExprEval.of( - arg.replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) + NullHandlingExpressionHelper.nullToDefault(arg).replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) ); } } @@ -1041,7 +1041,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(null); + return ExprEval.of(NullHandlingExpressionHelper.nullToDefault((String) null)); } return ExprEval.of(StringUtils.toLowerCase(arg)); } @@ -1064,9 +1064,49 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(null); + return ExprEval.of(NullHandlingExpressionHelper.nullToDefault((String) null)); } return ExprEval.of(StringUtils.toUpperCase(arg)); } } + + class IsNullFunc implements Function + { + @Override + public String name() + { + return "isnull"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 1) { + throw new IAE("Function[%s] needs 1 argument", name()); + } + + final ExprEval expr = args.get(0).eval(bindings); + return ExprEval.of(expr.isNull(), ExprType.LONG); + } + } + + class IsNotNullFunc implements Function + { + @Override + public String name() + { + return "notnull"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 1) { + throw new IAE("Function[%s] needs 1 argument", name()); + } + + final ExprEval expr = args.get(0).eval(bindings); + return ExprEval.of(!expr.isNull(), ExprType.LONG); + } + } } diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java index 250eaf96d63b..456e8fe67122 100644 --- a/common/src/test/java/io/druid/math/expr/EvalTest.java +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -20,6 +20,7 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandlingExpressionHelper; import org.junit.Assert; import org.junit.Test; @@ -139,9 +140,13 @@ public void testLongEval() Assert.assertEquals(1271055781L, evalLong("unix_timestamp('2010-04-12T07:03:01')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01+09:00')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01.419+09:00')", bindings)); - - Assert.assertEquals("", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); + if (NullHandlingExpressionHelper.useDefaultValuesForNull()) { + Assert.assertEquals("NULL", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); + } else { + Assert.assertEquals("", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); + } Assert.assertEquals("x", eval("nvl(if(x == 9223372036854775806, '', 'x'), 'NULL')", bindings).asString()); + } @Test diff --git a/common/src/test/java/io/druid/math/expr/FunctionTest.java b/common/src/test/java/io/druid/math/expr/FunctionTest.java index bb0859e01ce3..37dbf214a88f 100644 --- a/common/src/test/java/io/druid/math/expr/FunctionTest.java +++ b/common/src/test/java/io/druid/math/expr/FunctionTest.java @@ -20,6 +20,7 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandlingExpressionHelper; import org.junit.Assert; import org.junit.Test; @@ -55,7 +56,12 @@ public void testCaseSearched() public void testConcat() { assertExpr("concat(x,' ',y)", "foo 2"); - assertExpr("concat(x,' ',nonexistent,' ',y)", null); + if (NullHandlingExpressionHelper.useDefaultValuesForNull()) { + assertExpr("concat(x,' ',nonexistent,' ',y)", "foo 2"); + } else { + assertExpr("concat(x,' ',nonexistent,' ',y)", null); + } + assertExpr("concat(z)", "3.1"); assertExpr("concat()", null); } @@ -110,4 +116,18 @@ private void assertExpr(final String expression, final Object expectedResult) final Expr expr = Parser.parse(expression, ExprMacroTable.nil()); Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value()); } + + @Test + public void testIsNull() + { + assertExpr("isnull(null)", 1L); + assertExpr("isnull('abc')", 0L); + } + + @Test + public void testIsNotNull() + { + assertExpr("notnull(null)", 0L); + assertExpr("notnull('abc')", 1L); + } } diff --git a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java index 0823961f39f8..e55d5260a142 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java @@ -97,7 +97,7 @@ public static String fromUtf8(final ByteBuffer buffer, final int numBytes) @Nullable public static String fromUtf8Nullable(final ByteBuffer buffer, final int numBytes) { - if(numBytes < 0){ + if (numBytes < 0) { return null; } final byte[] bytes = new byte[numBytes]; diff --git a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java index e8fa19bcc69d..5e2141cd3b93 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java @@ -21,7 +21,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; -import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; /** diff --git a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java index 3e79cbc8b2ca..188b49750e53 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java @@ -20,7 +20,6 @@ package io.druid.segment.column; import com.google.common.base.Supplier; -import io.druid.collections.bitmap.ImmutableBitmap; /** */ diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java index e72428424961..88cef6483289 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Suppliers; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.DoubleColumnSerializer; diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java index 23c8da9e1a62..bae8b8549ea7 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Suppliers; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.FloatColumnSerializer; diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java index 2490734adbb3..c9b6485d0c1b 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Suppliers; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.LongColumnSerializer; diff --git a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java index 7a8c366985b4..34ed7d59c569 100644 --- a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java +++ b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java @@ -297,8 +297,8 @@ public void testNumericEvolutionTimeseriesAggregation() Map result = Maps.newHashMap(); result.put("a", null); result.put("b", null); - result.put("c", 0L); - result.put("d", 0.0d); + result.put("c", null); + result.put("d", null); Assert.assertEquals( timeseriesResult(result), runQuery(query, factory, ImmutableList.of(index4)) diff --git a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java index 6c6bab684952..a7cf8b554705 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java @@ -135,7 +135,8 @@ public void testOneMultiValuedStringColumn() assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("0", "1", "2", "4", "5", "6", "7", "8")); } else { assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("2")); - assertFilterMatches(EDF("dim4 == null"), ImmutableList.of("0", "1", "4", "5", "6", "7", "8")); + // AS per SQL standard null == null returns false. + assertFilterMatches(EDF("dim4 == null"), ImmutableList.of()); } assertFilterMatches(EDF("dim4 == '1'"), ImmutableList.of()); assertFilterMatches(EDF("dim4 == '3'"), ImmutableList.of("3")); @@ -196,13 +197,22 @@ public void testMissingColumn() if (NullHandlingHelper.useDefaultValuesForNull()) { assertFilterMatches(EDF("missing == ''"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); } else { - assertFilterMatches(EDF("missing == null"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + // AS per SQL standard null == null returns false. + assertFilterMatches(EDF("missing == null"), ImmutableList.of()); } assertFilterMatches(EDF("missing == '1'"), ImmutableList.of()); assertFilterMatches(EDF("missing == 2"), ImmutableList.of()); - assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(EDF("missing < 2"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(EDF("missing < 2.0"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if(NullHandlingHelper.useDefaultValuesForNull()) { + // missing equivaluent to 0 + assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + assertFilterMatches(EDF("missing < 2"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + assertFilterMatches(EDF("missing < 2.0"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + } else { + // missing equivalent to null + assertFilterMatches(EDF("missing < '2'"), ImmutableList.of()); + assertFilterMatches(EDF("missing < 2"), ImmutableList.of()); + assertFilterMatches(EDF("missing < 2.0"), ImmutableList.of()); + } assertFilterMatches(EDF("missing > '2'"), ImmutableList.of()); assertFilterMatches(EDF("missing > 2"), ImmutableList.of()); assertFilterMatches(EDF("missing > 2.0"), ImmutableList.of()); diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java index 9e51d4e32828..63a64336a707 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -96,10 +96,15 @@ public void testObjectSelector() final BaseObjectColumnValueSelector selector = XPLUSY.makeColumnValueSelector("expr", COLUMN_SELECTOR_FACTORY); COLUMN_SELECTOR_FACTORY.setRow(ROW0); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? "" : null, selector.getObject()); + Assert.assertEquals(null, selector.getObject()); COLUMN_SELECTOR_FACTORY.setRow(ROW1); - Assert.assertEquals(4.0d, selector.getObject()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(4.0d, selector.getObject()); + } else { + // y is null for row1 + Assert.assertEquals(null, selector.getObject()); + } COLUMN_SELECTOR_FACTORY.setRow(ROW2); Assert.assertEquals(5.1d, selector.getObject()); @@ -117,7 +122,12 @@ public void testLongSelector() Assert.assertEquals(0L, selector.getLong()); COLUMN_SELECTOR_FACTORY.setRow(ROW1); - Assert.assertEquals(4L, selector.getLong()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(4.0d, selector.getLong()); + } else { + // y is null for row1 + Assert.assertTrue(selector.isNull()); + } COLUMN_SELECTOR_FACTORY.setRow(ROW2); Assert.assertEquals(5L, selector.getLong()); @@ -135,7 +145,12 @@ public void testLongSelectorUsingStringFunction() Assert.assertEquals(0L, selector.getLong()); COLUMN_SELECTOR_FACTORY.setRow(ROW1); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 4L : 0L, selector.getLong()); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(4L, selector.getLong()); + } else { + // y is null for row1 + Assert.assertTrue(selector.isNull()); + } COLUMN_SELECTOR_FACTORY.setRow(ROW2); Assert.assertEquals(0L, selector.getLong()); @@ -153,8 +168,12 @@ public void testFloatSelector() Assert.assertEquals(0.0f, selector.getFloat(), 0.0f); COLUMN_SELECTOR_FACTORY.setRow(ROW1); - Assert.assertEquals(4.0f, selector.getFloat(), 0.0f); - + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(4.0f, selector.getFloat(), 0.0f); + } else { + // y is null for row1 + Assert.assertTrue(selector.isNull()); + } COLUMN_SELECTOR_FACTORY.setRow(ROW2); Assert.assertEquals(5.1f, selector.getFloat(), 0.0f); @@ -181,10 +200,18 @@ public void testDimensionSelector() Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); COLUMN_SELECTOR_FACTORY.setRow(ROW1); - Assert.assertEquals(false, nullMatcher.matches()); - Assert.assertEquals(false, fiveMatcher.matches()); - Assert.assertEquals(true, nonNullMatcher.matches()); - Assert.assertEquals("4.0", selector.lookupName(selector.getRow().get(0))); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(false, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(true, nonNullMatcher.matches()); + Assert.assertEquals("4.0", selector.lookupName(selector.getRow().get(0))); + } else { + // y is null in row1 + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + } COLUMN_SELECTOR_FACTORY.setRow(ROW2); Assert.assertEquals(false, nullMatcher.matches()); @@ -215,7 +242,10 @@ public void testDimensionSelectorUsingStringFunction() COLUMN_SELECTOR_FACTORY.setRow(ROW1); Assert.assertEquals(1, selector.getRow().size()); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? "4" : null, selector.lookupName(selector.getRow().get(0))); + Assert.assertEquals( + NullHandlingHelper.useDefaultValuesForNull() ? "4" : null, + selector.lookupName(selector.getRow().get(0)) + ); COLUMN_SELECTOR_FACTORY.setRow(ROW2); Assert.assertEquals(1, selector.getRow().size()); @@ -245,10 +275,18 @@ public void testDimensionSelectorWithExtraction() Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); COLUMN_SELECTOR_FACTORY.setRow(ROW1); - Assert.assertEquals(false, nullMatcher.matches()); - Assert.assertEquals(false, fiveMatcher.matches()); - Assert.assertEquals(true, nonNullMatcher.matches()); - Assert.assertEquals("4", selector.lookupName(selector.getRow().get(0))); + if (NullHandlingHelper.useDefaultValuesForNull()) { + Assert.assertEquals(false, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(true, nonNullMatcher.matches()); + Assert.assertEquals("4.0", selector.lookupName(selector.getRow().get(0))); + } else { + // y is null in row1 + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + } COLUMN_SELECTOR_FACTORY.setRow(ROW2); Assert.assertEquals(false, nullMatcher.matches()); diff --git a/server/src/test/java/io/druid/query/expression/ExprMacroTest.java b/server/src/test/java/io/druid/query/expression/ExprMacroTest.java index 1e19ca36ec5e..dbe89f41ab9f 100644 --- a/server/src/test/java/io/druid/query/expression/ExprMacroTest.java +++ b/server/src/test/java/io/druid/query/expression/ExprMacroTest.java @@ -23,6 +23,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.math.expr.Expr; import io.druid.math.expr.Parser; +import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -43,6 +44,9 @@ public class ExprMacroTest .build() ); + public static String emptyStringResult = NullHandlingHelper.useDefaultValuesForNull() ? null : ""; + + @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -139,36 +143,36 @@ public void testTimestampFormat() @Test public void testTrim() { - assertExpr("trim('')", ""); + assertExpr("trim('')", emptyStringResult); assertExpr("trim(concat(' ',x,' '))", "foo"); assertExpr("trim(spacey)", "hey there"); assertExpr("trim(spacey, '')", " hey there "); assertExpr("trim(spacey, 'he ')", "y ther"); - assertExpr("trim(spacey, spacey)", ""); + assertExpr("trim(spacey, spacey)", emptyStringResult); assertExpr("trim(spacey, substring(spacey, 0, 4))", "y ther"); } @Test public void testLTrim() { - assertExpr("ltrim('')", ""); + assertExpr("ltrim('')", emptyStringResult); assertExpr("ltrim(concat(' ',x,' '))", "foo "); assertExpr("ltrim(spacey)", "hey there "); assertExpr("ltrim(spacey, '')", " hey there "); assertExpr("ltrim(spacey, 'he ')", "y there "); - assertExpr("ltrim(spacey, spacey)", ""); + assertExpr("ltrim(spacey, spacey)", emptyStringResult); assertExpr("ltrim(spacey, substring(spacey, 0, 4))", "y there "); } @Test public void testRTrim() { - assertExpr("rtrim('')", ""); + assertExpr("rtrim('')", emptyStringResult); assertExpr("rtrim(concat(' ',x,' '))", " foo"); assertExpr("rtrim(spacey)", " hey there"); assertExpr("rtrim(spacey, '')", " hey there "); assertExpr("rtrim(spacey, 'he ')", " hey ther"); - assertExpr("rtrim(spacey, spacey)", ""); + assertExpr("rtrim(spacey, spacey)", emptyStringResult); assertExpr("rtrim(spacey, substring(spacey, 0, 4))", " hey ther"); } diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java index 814636fa50b6..c33c2d0778a6 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java @@ -26,7 +26,6 @@ import io.druid.math.expr.Expr; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java new file mode 100644 index 000000000000..5465e311b752 --- /dev/null +++ b/sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java @@ -0,0 +1,66 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.sql.calcite.expression; + +import com.google.common.collect.Iterables; +import io.druid.java.util.common.StringUtils; +import io.druid.sql.calcite.planner.PlannerContext; +import io.druid.sql.calcite.table.RowSignature; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlOperator; + +public class UnaryFunctionOperatorConversion implements SqlOperatorConversion +{ + private final SqlOperator operator; + private final String druidOperator; + + public UnaryFunctionOperatorConversion(final SqlOperator operator, final String druidOperator) + { + this.operator = operator; + this.druidOperator = druidOperator; + } + + @Override + public SqlOperator calciteOperator() + { + return operator; + } + + @Override + public DruidExpression toDruidExpression( + final PlannerContext plannerContext, + final RowSignature rowSignature, + final RexNode rexNode + ) + { + return OperatorConversions.convertCall( + plannerContext, + rowSignature, + rexNode, + operands -> DruidExpression.fromExpression( + StringUtils.format( + "%s(%s)", + druidOperator, + Iterables.getOnlyElement(operands).getExpression() + ) + ) + ); + } +} diff --git a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java index bab3b1d73140..ed6a94376547 100644 --- a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java +++ b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java @@ -30,7 +30,6 @@ import io.druid.query.ordering.StringComparators; import io.druid.segment.column.ValueType; import io.druid.server.security.AuthorizerMapper; -import io.druid.sql.calcite.expression.DruidExpression; import io.druid.sql.calcite.schema.DruidSchema; import io.druid.sql.calcite.schema.InformationSchema; import org.apache.calcite.jdbc.CalciteSchema; diff --git a/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java index c4d807816e6f..59fab5b55201 100644 --- a/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java @@ -36,8 +36,8 @@ import io.druid.sql.calcite.expression.AliasedOperatorConversion; import io.druid.sql.calcite.expression.BinaryOperatorConversion; import io.druid.sql.calcite.expression.DirectOperatorConversion; -import io.druid.sql.calcite.expression.DruidExpression; import io.druid.sql.calcite.expression.SqlOperatorConversion; +import io.druid.sql.calcite.expression.UnaryFunctionOperatorConversion; import io.druid.sql.calcite.expression.UnaryPrefixOperatorConversion; import io.druid.sql.calcite.expression.UnarySuffixOperatorConversion; import io.druid.sql.calcite.expression.builtin.BTrimOperatorConversion; @@ -118,8 +118,8 @@ public class DruidOperatorTable implements SqlOperatorTable .add(new DirectOperatorConversion(SqlStdOperatorTable.UPPER, "upper")) .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.NOT, "!")) .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.UNARY_MINUS, "-")) - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NULL, "== " + DruidExpression.nullLiteral())) - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "!= " + DruidExpression.nullLiteral())) + .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NULL, "isnull")) + .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "notnull")) .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_FALSE, "<= 0")) // Matches Evals.asBoolean .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_TRUE, "<= 0")) // Matches Evals.asBoolean .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_TRUE, "> 0")) // Matches Evals.asBoolean diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 2ed7d1c6b77c..efd086e0793c 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -1588,9 +1588,7 @@ public void testNullEmptyStringEquality() throws Exception .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) - .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,(\"dim2\" == " - + DruidExpression.nullLiteral() - + "))")) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,isnull(\"dim2\"))")) .aggregators(AGGS(new CountAggregatorFactory("a0"))) .context(TIMESERIES_CONTEXT_DEFAULT) .build() @@ -1624,7 +1622,7 @@ public void testCoalesceColumns() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "case_searched((\"dim2\" != " + DruidExpression.nullLiteral() + "),\"dim2\",\"dim1\")", + "case_searched(notnull(\"dim2\"),\"dim2\",\"dim1\")", ValueType.STRING ) ) @@ -1916,11 +1914,9 @@ public void testCountNullableExpression() throws Exception new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), EXPRESSION_FILTER( - "(case_searched((\"dim2\" == 'abc'),'yes',(\"dim2\" == 'def'),'yes'," + "notnull(case_searched((\"dim2\" == 'abc'),'yes',(\"dim2\" == 'def'),'yes'," + DruidExpression.nullLiteral() - + ") != " - + DruidExpression.nullLiteral() - + ")" + + "))" ) ) )) @@ -4738,9 +4734,16 @@ public void testGroupByLimitPushDownWithHavingOnLong() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{"10.1", "", 1L}, + new Object[]{"2", "", 1L}, + new Object[]{"abc", "", 1L}, + new Object[]{"", "a", 1L} + ) : ImmutableList.of( - new Object[]{"10.1", nullValue, 1L}, - new Object[]{"abc", nullValue, 1L}, + new Object[]{"10.1", null, 1L}, + new Object[]{"abc", null, 1L}, new Object[]{"2", "", 1L}, new Object[]{"", "a", 1L} ) diff --git a/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java b/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java index 22c2a24bd1e9..3d2c5103acfa 100644 --- a/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java @@ -20,7 +20,6 @@ package io.druid.sql.calcite.planner; import com.google.common.collect.ImmutableSortedSet; -import io.druid.sql.calcite.expression.DruidExpression; import org.junit.Assert; import org.junit.Test; @@ -29,7 +28,6 @@ public class CalcitesTest @Test public void testEscapeStringLiteral() { - Assert.assertEquals(DruidExpression.nullLiteral(), Calcites.escapeStringLiteral(null)); Assert.assertEquals("''", Calcites.escapeStringLiteral("")); Assert.assertEquals("'foo'", Calcites.escapeStringLiteral("foo")); Assert.assertEquals("'foo bar'", Calcites.escapeStringLiteral("foo bar")); From 5e674a335e8b28f24db3201caab7818f8484a90c Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 8 Nov 2017 21:53:53 +0530 Subject: [PATCH 31/50] tests passing with null handling set to true --- .../config/NullValueHandlingConfig.java | 2 +- .../main/java/io/druid/math/expr/Expr.java | 1 - .../java/io/druid/math/expr/Function.java | 2 +- .../segment/filter/ExpressionFilterTest.java | 2 +- .../druid/sql/calcite/rel/DruidSemiJoin.java | 7 ++- .../druid/sql/calcite/CalciteQueryTest.java | 61 ++++++++++++------- 6 files changed, 47 insertions(+), 28 deletions(-) diff --git a/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java b/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java index be8e7f08101b..fdd1638b510e 100644 --- a/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java +++ b/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java @@ -32,7 +32,7 @@ public class NullValueHandlingConfig public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) { this.useDefaultValuesForNull = useDefaultValuesForNull == null - ? false + ? true : useDefaultValuesForNull; } diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index fa2ce3cd5058..04fdec166079 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -31,7 +31,6 @@ import javax.annotation.Nullable; import java.util.List; import java.util.Objects; -import java.util.function.UnaryOperator; /** */ diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index 6014caff0483..bb6122c2c674 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -753,7 +753,7 @@ public ExprEval apply(final List args, final Expr.ObjectBinding bindings) return ExprEval.of(null); } } - + /** * "Simple CASE" function, similar to {@code CASE expr WHEN value THEN result [ELSE else_result] END} in SQL. */ diff --git a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java index a7cf8b554705..ce6e789f917a 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java @@ -202,7 +202,7 @@ public void testMissingColumn() } assertFilterMatches(EDF("missing == '1'"), ImmutableList.of()); assertFilterMatches(EDF("missing == 2"), ImmutableList.of()); - if(NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandlingHelper.useDefaultValuesForNull()) { // missing equivaluent to 0 assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); assertFilterMatches(EDF("missing < 2"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java index 8b68c071aa48..f4b89e6d2c8d 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java @@ -295,6 +295,10 @@ public List accumulate(final List theConditions, final Object[ for (int i : rightKeys) { final Object value = row[i]; + if (value == null) { + // NULLS are not supposed to match NULLs in a join. So ignore them. + continue; + } final String stringValue = DimensionHandlerUtils.convertObjectToString(value); values.add(stringValue); if (values.size() > maxSemiJoinRowsInMemory) { @@ -319,9 +323,8 @@ public List accumulate(final List theConditions, final Object[ ) ); } + theConditions.add(makeAnd(subConditions)); } - - theConditions.add(makeAnd(subConditions)); } return theConditions; } diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index efd086e0793c..e90ff9a5394b 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -1340,6 +1340,7 @@ public void testHavingOnRatio() throws Exception @Test public void testGroupByWithSelectProjections() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" + " dim1," @@ -1359,10 +1360,10 @@ public void testGroupByWithSelectProjections() throws Exception .build() ), ImmutableList.of( - new Object[]{"", ""}, - new Object[]{"1", ""}, + new Object[]{"", nullValue}, + new Object[]{"1", nullValue}, new Object[]{"10.1", "0.1"}, - new Object[]{"2", ""}, + new Object[]{"2", nullValue}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"} ) @@ -1372,6 +1373,7 @@ public void testGroupByWithSelectProjections() throws Exception @Test public void testGroupByWithSelectAndOrderByProjections() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" + " dim1," @@ -1411,9 +1413,9 @@ public void testGroupByWithSelectAndOrderByProjections() throws Exception new Object[]{"10.1", "0.1"}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"}, - new Object[]{"1", ""}, - new Object[]{"2", ""}, - new Object[]{"", ""} + new Object[]{"1", nullValue}, + new Object[]{"2", nullValue}, + new Object[]{"", nullValue} ) ); } @@ -1421,6 +1423,8 @@ public void testGroupByWithSelectAndOrderByProjections() throws Exception @Test public void testTopNWithSelectProjections() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + testQuery( "SELECT\n" + " dim1," @@ -1443,10 +1447,10 @@ public void testTopNWithSelectProjections() throws Exception .build() ), ImmutableList.of( - new Object[]{"", ""}, - new Object[]{"1", ""}, + new Object[]{"", nullValue}, + new Object[]{"1", nullValue}, new Object[]{"10.1", "0.1"}, - new Object[]{"2", ""}, + new Object[]{"2", nullValue}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"} ) @@ -1456,6 +1460,8 @@ public void testTopNWithSelectProjections() throws Exception @Test public void testTopNWithSelectAndOrderByProjections() throws Exception { + String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + testQuery( "SELECT\n" + " dim1," @@ -1483,9 +1489,9 @@ public void testTopNWithSelectAndOrderByProjections() throws Exception new Object[]{"10.1", "0.1"}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"}, - new Object[]{"1", ""}, - new Object[]{"2", ""}, - new Object[]{"", ""} + new Object[]{"1", nullValue}, + new Object[]{"2", nullValue}, + new Object[]{"", nullValue} ) ); } @@ -3927,6 +3933,9 @@ public void testExactCountDistinctUsingSubquery() throws Exception @Test public void testTopNFilterJoin() throws Exception { + DimFilter filter = NullHandlingHelper.useDefaultValuesForNull() ? + IN("dim2", Arrays.asList(null, "a"), null) + : SELECTOR("dim2", "a", null); // Filters on top N values of some dimension by using an inner join. testQuery( "SELECT t1.dim1, SUM(t1.cnt)\n" @@ -3957,7 +3966,7 @@ public void testTopNFilterJoin() throws Exception .setDataSource(CalciteTests.DATASOURCE1) .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) - .setDimFilter(IN("dim2", Arrays.asList(null, "a"), null)) + .setDimFilter(filter) .setDimensions(DIMS(new DefaultDimensionSpec("dim1", "d0"))) .setAggregatorSpecs(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec( @@ -3985,9 +3994,7 @@ public void testTopNFilterJoin() throws Exception ) : ImmutableList.of( new Object[]{"", 1L}, - new Object[]{"1", 1L}, - new Object[]{"10.1", 1L}, - new Object[]{"abc", 1L} + new Object[]{"1", 1L} ) ); } @@ -6146,6 +6153,13 @@ public void testUsingSubqueryAsFilterForbiddenByConfig() throws Exception @Test public void testUsingSubqueryAsFilterOnTwoColumns() throws Exception { + DimFilter filter = NullHandlingHelper.useDefaultValuesForNull() ? + AND(SELECTOR("dim1", "def", null), SELECTOR("dim2", "abc", null)) + : OR(SELECTOR("dim1", "def", null), + AND( + SELECTOR("dim1", "def", null), + SELECTOR("dim2", "abc", null) + )); testQuery( "SELECT __time, cnt, dim1, dim2 FROM druid.foo " + " WHERE (dim1, dim2) IN (" @@ -6174,7 +6188,7 @@ public void testUsingSubqueryAsFilterOnTwoColumns() throws Exception newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) - .filters(AND(SELECTOR("dim1", "def", null), SELECTOR("dim2", "abc", null))) + .filters(filter) .columns("__time", "cnt", "dim1", "dim2") .resultFormat(ScanQuery.RESULT_FORMAT_COMPACTED_LIST) .context(QUERY_CONTEXT_DEFAULT) @@ -6192,10 +6206,6 @@ public void testUsingSubqueryAsFilterWithInnerSort() throws Exception String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; // Regression test for https://github.com/druid-io/druid/issues/4208 - List expectedSubqueryResult = NullHandlingHelper.useDefaultValuesForNull() ? - ImmutableList.of("", "a", "abc") : - Arrays.asList(null, "", "a", "abc"); - testQuery( "SELECT dim1, dim2 FROM druid.foo\n" + " WHERE dim2 IN (\n" @@ -6227,11 +6237,12 @@ public void testUsingSubqueryAsFilterWithInnerSort() throws Exception newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) - .filters(IN("dim2", expectedSubqueryResult, null)) + .filters(IN("dim2", ImmutableList.of("", "a", "abc"), null)) .columns("dim1", "dim2") .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandlingHelper.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", "a"}, new Object[]{"10.1", nullValue}, @@ -6239,6 +6250,12 @@ public void testUsingSubqueryAsFilterWithInnerSort() throws Exception new Object[]{"1", "a"}, new Object[]{"def", "abc"}, new Object[]{"abc", nullValue} + ) : + ImmutableList.of( + new Object[]{"", "a"}, + new Object[]{"2", ""}, + new Object[]{"1", "a"}, + new Object[]{"def", "abc"} ) ); } From ad8c7b978247faeaab318921921cac87eb84873c Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 8 Nov 2017 22:05:15 +0530 Subject: [PATCH 32/50] more review comments --- .travis.yml | 4 ++-- .../io/druid/common/config/NullHandlingExpressionHelper.java | 2 +- pom.xml | 4 ++-- .../src/main/java/io/druid/guice/NullHandlingModule.java | 2 +- .../src/main/java/io/druid/segment/NullHandlingHelper.java | 2 +- .../java/io/druid/guice/NullHandlingHelperInjectionTest.java | 2 +- .../java/io/druid/sql/calcite/expression/DruidExpression.java | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index f9a3ec4d2dac..f9b283c31bf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,14 +35,14 @@ matrix: install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B before_script: - unset _JAVA_OPTIONS - script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.null.handling.useDefaultValueForNull=false -pl processing + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.generic.useDefaultValueForNull=false -pl processing # non-processing modules test with SQL Compatible Null Handling - sudo: false install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B before_script: - unset _JAVA_OPTIONS - script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.null.handling.useDefaultValueForNull=false -pl '!processing' + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.generic.useDefaultValueForNull=false -pl '!processing' # run integration tests - sudo: required diff --git a/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java b/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java index 20426d620879..bf7715066bcd 100644 --- a/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java +++ b/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java @@ -24,7 +24,7 @@ public class NullHandlingExpressionHelper { - private static String NULL_HANDLING_CONFIG_STRING = "druid.null.handling.useDefaultValueForNull"; + private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; // use these values to ensure that convertObjectToLong(), convertObjectToDouble() and convertObjectToFloat() // return the same boxed object when returning a constant zero. diff --git a/pom.xml b/pom.xml index 258f4e28b4c0..ce53f7a1194f 100644 --- a/pom.xml +++ b/pom.xml @@ -80,7 +80,7 @@ Need to update Druid to use Jackson 2.6+ --> 1.10.77 2.5.5 - true + true @@ -1027,7 +1027,7 @@ -Xmx3000m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 -Duser.timezone=UTC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager - -Ddruid.null.handling.useDefaultValueForNull=${druid.null.handling.useDefaultValueForNull} + -Ddruid.generic.useDefaultValueForNull=${druid.generic.useDefaultValueForNull} -Ddruid.indexing.doubleStorage=double diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java index 2d70ce6ccfcc..8a2ff7b52e3e 100644 --- a/processing/src/main/java/io/druid/guice/NullHandlingModule.java +++ b/processing/src/main/java/io/druid/guice/NullHandlingModule.java @@ -32,7 +32,7 @@ public class NullHandlingModule implements Module @Override public void configure(Binder binder) { - JsonConfigProvider.bind(binder, "druid.null.handling", NullValueHandlingConfig.class); + JsonConfigProvider.bind(binder, "druid.generic", NullValueHandlingConfig.class); binder.requestStaticInjection(NullHandlingHelper.class); binder.requestStaticInjection(NullHandlingExpressionHelper.class); diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index f1ed3a4204a9..9cb508eed47a 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -32,7 +32,7 @@ public class NullHandlingHelper { - private static String NULL_HANDLING_CONFIG_STRING = "druid.null.handling.useDefaultValueForNull"; + private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; // use these values to ensure that convertObjectToLong(), convertObjectToDouble() and convertObjectToFloat() // return the same boxed object when returning a constant zero. diff --git a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java index 28673d0c6f51..1a1bfeac8769 100644 --- a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java +++ b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java @@ -26,7 +26,7 @@ public class NullHandlingHelperInjectionTest { - public static String NULL_HANDLING_CONFIG_STRING = ("druid.null.handling.useDefaultValueForNull"); + public static String NULL_HANDLING_CONFIG_STRING = ("druid.generic.useDefaultValueForNull"); @Test public void testNullHandlingHelperUseDefaultValues() diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java index c33c2d0778a6..d343134fad7c 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java @@ -55,7 +55,7 @@ public class DruidExpression private DruidExpression(final SimpleExtraction simpleExtraction, final String expression) { this.simpleExtraction = simpleExtraction; - this.expression = expression; + this.expression = Preconditions.checkNotNull(expression); } public static DruidExpression of(final SimpleExtraction simpleExtraction, final String expression) From 35291a6d12ec53ba09339bffae430785da6d89f9 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 8 Nov 2017 22:11:58 +0530 Subject: [PATCH 33/50] refactor method names --- .../config/NullHandlingExpressionHelper.java | 30 ++----------------- .../main/java/io/druid/math/expr/Expr.java | 6 ++-- .../java/io/druid/math/expr/ExprEval.java | 2 +- .../java/io/druid/math/expr/Function.java | 12 ++++---- .../common/task/RealtimeIndexTaskTest.java | 2 +- .../dimension/ListFilteredDimensionSpec.java | 4 +-- .../dimension/RegexFilteredDimensionSpec.java | 4 +-- .../druid/query/expression/LikeExprMacro.java | 2 +- .../expression/RegexpExtractExprMacro.java | 4 +-- .../extraction/FunctionalExtraction.java | 6 ++-- .../extraction/IdentityExtractionFn.java | 4 +-- .../extraction/JavaScriptExtractionFn.java | 2 +- .../query/extraction/LowerExtractionFn.java | 2 +- .../extraction/MatchingDimExtractionFn.java | 2 +- .../extraction/RegexDimExtractionFn.java | 4 +-- .../SearchQuerySpecDimExtractionFn.java | 2 +- .../extraction/SubstringDimExtractionFn.java | 2 +- .../query/extraction/TimeDimExtractionFn.java | 2 +- .../query/extraction/UpperExtractionFn.java | 2 +- .../io/druid/query/filter/InDimFilter.java | 4 +-- .../io/druid/query/filter/LikeDimFilter.java | 2 +- .../druid/query/filter/SelectorDimFilter.java | 2 +- .../RowBasedColumnSelectorFactory.java | 10 +++---- .../epinephelinae/RowBasedGrouperHelper.java | 8 ++--- ...ngStringGroupByColumnSelectorStrategy.java | 2 +- .../StringGroupByColumnSelectorStrategy.java | 2 +- .../query/lookup/LookupExtractionFn.java | 2 +- .../java/io/druid/query/search/SearchHit.java | 2 +- .../ColumnSelectorBitmapIndexSelector.java | 2 +- .../segment/ConstantDimensionSelector.java | 2 +- .../druid/segment/DimensionHandlerUtils.java | 2 +- .../java/io/druid/segment/IndexMerger.java | 2 +- .../druid/segment/NullDimensionSelector.java | 2 +- .../io/druid/segment/NullHandlingHelper.java | 12 ++++---- .../druid/segment/StringDimensionIndexer.java | 2 +- .../io/druid/segment/data/GenericIndexed.java | 2 +- .../io/druid/segment/filter/BoundFilter.java | 8 ++--- .../segment/filter/ExpressionFilter.java | 2 +- .../io/druid/segment/filter/LikeFilter.java | 2 +- .../virtual/ExpressionObjectSelector.java | 12 ++++---- .../segment/virtual/ExpressionSelectors.java | 4 +-- .../extraction/FunctionalExtractionTest.java | 8 ++--- .../segment/IndexMergerNullHandlingTest.java | 8 ++--- .../segment/data/GenericIndexedTest.java | 4 +-- .../incremental/IncrementalIndexTest.java | 4 +-- .../query/dimension/LookupDimensionSpec.java | 2 +- .../sql/calcite/expression/Expressions.java | 2 +- .../io/druid/sql/calcite/rel/QueryMaker.java | 2 +- 48 files changed, 93 insertions(+), 119 deletions(-) diff --git a/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java b/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java index bf7715066bcd..0bc6a453648b 100644 --- a/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java +++ b/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java @@ -26,12 +26,6 @@ public class NullHandlingExpressionHelper { private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; - // use these values to ensure that convertObjectToLong(), convertObjectToDouble() and convertObjectToFloat() - // return the same boxed object when returning a constant zero. - public static final Double ZERO_DOUBLE = 0.0d; - public static final Float ZERO_FLOAT = 0.0f; - public static final Long ZERO_LONG = 0L; - // INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. // See NullHandlingModule for details. // The default system property is supposed to be used only in tests. @@ -45,33 +39,13 @@ public static boolean useDefaultValuesForNull() return INSTANCE.isUseDefaultValuesForNull(); } - public static String nullToDefault(String value) + public static String nullToEmptyIfNeeded(String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; } - public static String defaultToNull(String value) + public static String emptyToNullIfNeeded(String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; } - - public static boolean isNullOrDefault(String value) - { - return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; - } - - public static Long nullToDefault(Long value) - { - return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_LONG : value; - } - - public static Double nullToDefault(Double value) - { - return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_DOUBLE : value; - } - - public static Float nullToDefault(Float value) - { - return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_FLOAT : value; - } } diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index 04fdec166079..aec18d677390 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -124,7 +124,7 @@ class StringExpr extends ConstantExpr public StringExpr(String value) { - this.value = NullHandlingExpressionHelper.defaultToNull(value); + this.value = NullHandlingExpressionHelper.emptyToNullIfNeeded(value); } @Nullable @@ -498,8 +498,8 @@ class BinPlusExpr extends BinaryEvalOpExprBase @Override protected ExprEval evalString(@Nullable String left, @Nullable String right) { - return ExprEval.of(NullHandlingExpressionHelper.nullToDefault(left) - + NullHandlingExpressionHelper.nullToDefault(right)); + return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded(left) + + NullHandlingExpressionHelper.nullToEmptyIfNeeded(right)); } @Override diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 181733c9c99e..7768dae39844 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -233,7 +233,7 @@ private static class StringExprEval extends ExprEval { private StringExprEval(String value) { - super(NullHandlingExpressionHelper.defaultToNull(value)); + super(NullHandlingExpressionHelper.emptyToNullIfNeeded(value)); } @Override diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index bb6122c2c674..583dfa053fed 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -908,7 +908,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. return ExprEval.of(null); } else { - builder.append(NullHandlingExpressionHelper.nullToDefault(s)); + builder.append(NullHandlingExpressionHelper.nullToEmptyIfNeeded(s)); } } return ExprEval.of(builder.toString()); @@ -992,7 +992,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } else { // If starting index of substring is greater then the lengh of string, the result will be a zero length string. // e.g. 'select substring("abc", 4,5) as c;' will return an empty string - return ExprEval.of(NullHandlingExpressionHelper.nullToDefault((String) null)); + return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded((String) null)); } } } @@ -1016,10 +1016,10 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String pattern = args.get(1).eval(bindings).asString(); final String replacement = args.get(2).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandlingExpressionHelper.nullToDefault((String) null)); + return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded((String) null)); } return ExprEval.of( - NullHandlingExpressionHelper.nullToDefault(arg).replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) + NullHandlingExpressionHelper.nullToEmptyIfNeeded(arg).replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) ); } } @@ -1041,7 +1041,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandlingExpressionHelper.nullToDefault((String) null)); + return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded((String) null)); } return ExprEval.of(StringUtils.toLowerCase(arg)); } @@ -1064,7 +1064,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandlingExpressionHelper.nullToDefault((String) null)); + return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded((String) null)); } return ExprEval.of(StringUtils.toUpperCase(arg)); } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index d0dfa1d94e3f..13ecca4ac629 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -1120,7 +1120,7 @@ public Long sumMetric(final Task task, final DimFilter filter, final String metr Lists.>newArrayList() ); if (results.isEmpty()) { - return NullHandlingHelper.nullToDefault((Long) null); + return NullHandlingHelper.nullToZeroIfNeeded((Long) null); } return results.get(0).getValue().getLongMetric(metric); } diff --git a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java index 0af2c3cbd124..fecb0e727e54 100644 --- a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java @@ -106,7 +106,7 @@ private DimensionSelector filterWhiteList(DimensionSelector selector) } } else { for (int i = 0; i < selectorCardinality; i++) { - if (values.contains(NullHandlingHelper.nullToDefault(selector.lookupName(i)))) { + if (values.contains(NullHandlingHelper.nullToEmptyIfNeeded(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } @@ -137,7 +137,7 @@ public boolean apply(@Nullable String input) forwardMapping.defaultReturnValue(-1); final int[] reverseMapping = new int[maxPossibleFilteredCardinality]; for (int i = 0; i < selectorCardinality; i++) { - if (!values.contains(NullHandlingHelper.nullToDefault(selector.lookupName(i)))) { + if (!values.contains(NullHandlingHelper.nullToEmptyIfNeeded(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } diff --git a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java index cee0d2ba5f78..f80faaed39f6 100644 --- a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -76,7 +76,7 @@ public DimensionSelector decorate(final DimensionSelector selector) @Override public boolean apply(@Nullable String input) { - return compiledRegex.matcher(NullHandlingHelper.nullToDefault(input)).matches(); + return compiledRegex.matcher(NullHandlingHelper.nullToEmptyIfNeeded(input)).matches(); } } ); @@ -86,7 +86,7 @@ public boolean apply(@Nullable String input) final Int2IntOpenHashMap forwardMapping = new Int2IntOpenHashMap(); forwardMapping.defaultReturnValue(-1); for (int i = 0; i < selectorCardinality; i++) { - String val = NullHandlingHelper.nullToDefault(selector.lookupName(i)); + String val = NullHandlingHelper.nullToEmptyIfNeeded(selector.lookupName(i)); if (val != null && compiledRegex.matcher(val).matches()) { forwardMapping.put(i, count++); } diff --git a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java index ebdb15465c9e..d848b0a83e31 100644 --- a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java @@ -63,7 +63,7 @@ public Expr apply(final List args) } final LikeDimFilter.LikeMatcher likeMatcher = LikeDimFilter.LikeMatcher.from( - NullHandlingHelper.nullToDefault((String) patternExpr.getLiteralValue()), + NullHandlingHelper.nullToEmptyIfNeeded((String) patternExpr.getLiteralValue()), escapeChar ); diff --git a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java index 7d48546ea32d..0516dc6a77d9 100644 --- a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java @@ -64,9 +64,9 @@ class RegexpExtractExpr implements Expr public ExprEval eval(final ObjectBinding bindings) { String s = arg.eval(bindings).asString(); - final Matcher matcher = pattern.matcher(NullHandlingHelper.nullToDefault(s)); + final Matcher matcher = pattern.matcher(NullHandlingHelper.nullToEmptyIfNeeded(s)); final String retVal = matcher.find() ? matcher.group(index) : null; - return ExprEval.of(NullHandlingHelper.defaultToNull(retVal)); + return ExprEval.of(NullHandlingHelper.emptyToNullIfNeeded(retVal)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java index 9ad5d0656ed5..5b5931e8f1e4 100644 --- a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java +++ b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java @@ -52,7 +52,7 @@ public FunctionalExtraction( ) { this.retainMissingValue = retainMissingValue; - this.replaceMissingValueWith = NullHandlingHelper.defaultToNull(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandlingHelper.emptyToNullIfNeeded(replaceMissingValueWith); Preconditions.checkArgument( !(this.retainMissingValue && !(this.replaceMissingValueWith == null)), "Cannot specify a [replaceMissingValueWith] and set [retainMissingValue] to true" @@ -69,7 +69,7 @@ public FunctionalExtraction( public String apply(@Nullable String dimValue) { final String retval = extractionFunction.apply(dimValue); - return NullHandlingHelper.isNullOrDefault(retval) ? NullHandlingHelper.defaultToNull(dimValue) : retval; + return NullHandlingHelper.isNullOrEquivalent(retval) ? NullHandlingHelper.emptyToNullIfNeeded(dimValue) : retval; } }; } else { @@ -79,7 +79,7 @@ public String apply(@Nullable String dimValue) @Override public String apply(@Nullable String dimValue) { - final String retval = NullHandlingHelper.defaultToNull(extractionFunction.apply(dimValue)); + final String retval = NullHandlingHelper.emptyToNullIfNeeded(extractionFunction.apply(dimValue)); return retval == null ? FunctionalExtraction.this.replaceMissingValueWith : retval; diff --git a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java index 39fb96285e7f..efe31a3040bc 100644 --- a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java @@ -42,14 +42,14 @@ public byte[] getCacheKey() @Nullable public String apply(@Nullable Object value) { - return value == null ? null : NullHandlingHelper.defaultToNull(value.toString()); + return value == null ? null : NullHandlingHelper.emptyToNullIfNeeded(value.toString()); } @Override @Nullable public String apply(@Nullable String value) { - return NullHandlingHelper.defaultToNull(value); + return NullHandlingHelper.emptyToNullIfNeeded(value); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java index dcf2c66e9819..4f641da6435e 100644 --- a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java @@ -139,7 +139,7 @@ private void checkAndCompileScript() @Nullable public String apply(@Nullable String value) { - return this.apply((Object) NullHandlingHelper.defaultToNull(value)); + return this.apply((Object) NullHandlingHelper.emptyToNullIfNeeded(value)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java index f3366ac7e80d..59b9ff9c4f6c 100644 --- a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java @@ -53,7 +53,7 @@ public LowerExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (NullHandlingHelper.isNullOrDefault(key)) { + if (NullHandlingHelper.isNullOrEquivalent(key)) { return null; } return key.toLowerCase(locale); diff --git a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java index 851fe2290aa1..de07a1fc04e2 100644 --- a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java @@ -62,7 +62,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (NullHandlingHelper.isNullOrDefault(dimValue)) { + if (NullHandlingHelper.isNullOrEquivalent(dimValue)) { // We'd return null whether or not the pattern matched return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java index 1f5aceb3fc8f..446d5773f4e8 100644 --- a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java @@ -106,14 +106,14 @@ public byte[] getCacheKey() public String apply(@Nullable String dimValue) { final String retVal; - String val = NullHandlingHelper.nullToDefault(dimValue); + String val = NullHandlingHelper.nullToEmptyIfNeeded(dimValue); final Matcher matcher = val == null ? null : pattern.matcher(val); if (matcher != null && matcher.find()) { retVal = matcher.group(index); } else { retVal = replaceMissingValue ? replaceMissingValueWith : dimValue; } - return NullHandlingHelper.defaultToNull(retVal); + return NullHandlingHelper.emptyToNullIfNeeded(retVal); } @JsonProperty("expr") diff --git a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java index 0217b8a33b8d..ee9a04b47815 100644 --- a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java @@ -64,7 +64,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - return searchQuerySpec.accept(dimValue) ? NullHandlingHelper.defaultToNull(dimValue) : null; + return searchQuerySpec.accept(dimValue) ? NullHandlingHelper.emptyToNullIfNeeded(dimValue) : null; } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java index 2b71984726e0..e0031d2817f4 100644 --- a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java @@ -65,7 +65,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (NullHandlingHelper.isNullOrDefault(dimValue)) { + if (NullHandlingHelper.isNullOrEquivalent(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java index 39f8bfe44f7b..3c40596e7d2e 100644 --- a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java @@ -77,7 +77,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (NullHandlingHelper.isNullOrDefault(dimValue)) { + if (NullHandlingHelper.isNullOrEquivalent(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java index 8c6ddd9f3c1f..6ab50683dbfb 100644 --- a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java @@ -52,7 +52,7 @@ public UpperExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (NullHandlingHelper.isNullOrDefault(key)) { + if (NullHandlingHelper.isNullOrEquivalent(key)) { return null; } return key.toUpperCase(locale); diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index a7c10b6ceba2..94f6635061cb 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -79,7 +79,7 @@ public InDimFilter( this.values = new TreeSet<>(Comparators.naturalNullsFirst()); for (String value : values) { - this.values.add(NullHandlingHelper.defaultToNull(value)); + this.values.add(NullHandlingHelper.emptyToNullIfNeeded(value)); } this.dimension = dimension; this.extractionFn = extractionFn; @@ -175,7 +175,7 @@ private InDimFilter optimizeLookup() // there may be row values that match the selector value but are not included // in the lookup map. Match on the selector value as well. // If the selector value is overwritten in the lookup map, don't add selector value to keys. - if (exFn.isRetainMissingValue() && NullHandlingHelper.isNullOrDefault(lookup.apply(convertedValue))) { + if (exFn.isRetainMissingValue() && NullHandlingHelper.isNullOrEquivalent(lookup.apply(convertedValue))) { keys.add(convertedValue); } } diff --git a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java index ad76da563771..256fdf2b399c 100644 --- a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java @@ -152,7 +152,7 @@ private static void addPatternCharacter(final StringBuilder patternBuilder, fina public boolean matches(@Nullable final String s) { - String val = NullHandlingHelper.nullToDefault(s); + String val = NullHandlingHelper.nullToEmptyIfNeeded(s); return val != null && pattern.matcher(val).matches(); } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 56d86497d9b6..25b8a2b82f5c 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -65,7 +65,7 @@ public SelectorDimFilter( Preconditions.checkArgument(dimension != null, "dimension must not be null"); this.dimension = dimension; - this.value = NullHandlingHelper.defaultToNull(value); + this.value = NullHandlingHelper.emptyToNullIfNeeded(value); this.extractionFn = extractionFn; } diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java index 5cc41de6e395..c239f02ea3ad 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java @@ -234,7 +234,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (Objects.equals(NullHandlingHelper.defaultToNull(dimensionValue), value)) { + if (Objects.equals(NullHandlingHelper.emptyToNullIfNeeded(dimensionValue), value)) { return true; } } @@ -259,7 +259,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (Objects.equals(extractionFn.apply(NullHandlingHelper.defaultToNull(dimensionValue)), value)) { + if (Objects.equals(extractionFn.apply(NullHandlingHelper.emptyToNullIfNeeded(dimensionValue)), value)) { return true; } } @@ -292,7 +292,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (predicate.apply(NullHandlingHelper.defaultToNull(dimensionValue))) { + if (predicate.apply(NullHandlingHelper.emptyToNullIfNeeded(dimensionValue))) { return true; } } @@ -318,7 +318,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (predicate.apply(extractionFn.apply(NullHandlingHelper.defaultToNull(dimensionValue)))) { + if (predicate.apply(extractionFn.apply(NullHandlingHelper.emptyToNullIfNeeded(dimensionValue)))) { return true; } } @@ -344,7 +344,7 @@ public int getValueCardinality() @Override public String lookupName(int id) { - final String value = NullHandlingHelper.defaultToNull(row.get().getDimension(dimension).get(id)); + final String value = NullHandlingHelper.emptyToNullIfNeeded(row.get().getDimension(dimension).get(id)); return extractionFn == null ? value : extractionFn.apply(value); } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index f86f05f55391..ddfbb84028a9 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -442,7 +442,7 @@ public Row apply(Grouper.Entry entry) Object dimVal = entry.getKey().getKey()[i]; theMap.put( query.getDimensions().get(i - dimStart).getOutputName(), - dimVal instanceof String ? NullHandlingHelper.defaultToNull((String) dimVal) : dimVal + dimVal instanceof String ? NullHandlingHelper.emptyToNullIfNeeded((String) dimVal) : dimVal ); } @@ -1588,7 +1588,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) protected Long readFromBuffer(ByteBuffer buffer, int initialOffset) { if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandlingHelper.nullToDefault((Long) null); + return NullHandlingHelper.nullToZeroIfNeeded((Long) null); } else { return buffer.getLong(initialOffset + keyBufferPosition + Byte.BYTES); } @@ -1656,7 +1656,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) protected Float readFromBuffer(ByteBuffer buffer, int initialOffset) { if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandlingHelper.nullToDefault((Float) null); + return NullHandlingHelper.nullToZeroIfNeeded((Float) null); } else { return buffer.getFloat(initialOffset + keyBufferPosition + Byte.BYTES); } @@ -1724,7 +1724,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) protected Double readFromBuffer(ByteBuffer buffer, int initialOffset) { if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandlingHelper.nullToDefault((Double) null); + return NullHandlingHelper.nullToZeroIfNeeded((Double) null); } else { return buffer.getDouble(initialOffset + keyBufferPosition + Byte.BYTES); } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java index 91818ee5a0f8..2e26d480ca6e 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java @@ -60,7 +60,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, value ); } else { - resultMap.put(selectorPlus.getOutputName(), NullHandlingHelper.nullToDefault((String) null)); + resultMap.put(selectorPlus.getOutputName(), NullHandlingHelper.nullToEmptyIfNeeded((String) null)); } } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java index 43cedcecc613..646167464fef 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java @@ -49,7 +49,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, ((DimensionSelector) selectorPlus.getSelector()).lookupName(id) ); } else { - resultMap.put(selectorPlus.getOutputName(), NullHandlingHelper.nullToDefault((String) null)); + resultMap.put(selectorPlus.getOutputName(), NullHandlingHelper.nullToEmptyIfNeeded((String) null)); } } diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java index ae4752f56bf0..15de6c5cb3f5 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java @@ -58,7 +58,7 @@ public LookupExtractionFn( @Override public String apply(@Nullable String input) { - return NullHandlingHelper.defaultToNull(lookup.apply(NullHandlingHelper.nullToDefault(input))); + return NullHandlingHelper.emptyToNullIfNeeded(lookup.apply(NullHandlingHelper.nullToEmptyIfNeeded(input))); } }, retainMissingValue, diff --git a/processing/src/main/java/io/druid/query/search/SearchHit.java b/processing/src/main/java/io/druid/query/search/SearchHit.java index a9aae2dbe663..314a5a3513be 100644 --- a/processing/src/main/java/io/druid/query/search/SearchHit.java +++ b/processing/src/main/java/io/druid/query/search/SearchHit.java @@ -40,7 +40,7 @@ public SearchHit( ) { this.dimension = Preconditions.checkNotNull(dimension); - this.value = NullHandlingHelper.nullToDefault(value); + this.value = NullHandlingHelper.nullToEmptyIfNeeded(value); this.count = count; } diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index 0ed41dac64cc..b1641c5ed0f9 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -209,7 +209,7 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) final Column column = index.getColumn(dimension); if (column == null || !columnSupportsFiltering(column)) { - if (NullHandlingHelper.isNullOrDefault(value)) { + if (NullHandlingHelper.isNullOrEquivalent(value)) { return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows()); } else { return bitmapFactory.makeEmptyImmutableBitmap(); diff --git a/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java b/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java index 9b15814865f8..b41278381b6b 100644 --- a/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java @@ -36,7 +36,7 @@ public class ConstantDimensionSelector implements SingleValueHistoricalDimension public ConstantDimensionSelector(final String value) { - if (NullHandlingHelper.isNullOrDefault(value)) { + if (NullHandlingHelper.isNullOrEquivalent(value)) { // There's an optimized implementation for nulls that callers should use instead. throw new IllegalArgumentException("Use NullDimensionSelector or DimensionSelectorUtils.constantSelector"); } diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java index be42946430ac..50edf32dd747 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java @@ -296,7 +296,7 @@ public static Double convertObjectToDouble(@Nullable Object valObj) return ((Number) valObj).doubleValue(); } else if (valObj instanceof String) { Double doubleVal = Doubles.tryParse((String) valObj); - return NullHandlingHelper.nullToDefault(doubleVal); + return NullHandlingHelper.nullToZeroIfNeeded(doubleVal); } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index bd8abba39cf9..520592f46227 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -454,7 +454,7 @@ class DictionaryMergeIterator implements CloseableIterator final PeekingIterator iter = Iterators.peekingIterator( Iterators.transform( indexed.iterator(), - input -> NullHandlingHelper.nullToDefault(input) + input -> NullHandlingHelper.nullToEmptyIfNeeded(input) ) ); if (iter.hasNext()) { diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index 7f00bb0136c9..0cb20bc74e19 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -108,7 +108,7 @@ public IdLookup idLookup() @Override public int lookupId(String name) { - return NullHandlingHelper.isNullOrDefault(name) ? 0 : -1; + return NullHandlingHelper.isNullOrEquivalent(name) ? 0 : -1; } @Nullable diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index 9cb508eed47a..d1b557ef075c 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -53,32 +53,32 @@ public static boolean useDefaultValuesForNull() return INSTANCE.isUseDefaultValuesForNull(); } - public static String nullToDefault(String value) + public static String nullToEmptyIfNeeded(String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; } - public static String defaultToNull(String value) + public static String emptyToNullIfNeeded(String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; } - public static boolean isNullOrDefault(String value) + public static boolean isNullOrEquivalent(String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; } - public static Long nullToDefault(Long value) + public static Long nullToZeroIfNeeded(Long value) { return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_LONG : value; } - public static Double nullToDefault(Double value) + public static Double nullToZeroIfNeeded(Double value) { return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_DOUBLE : value; } - public static Float nullToDefault(Float value) + public static Float nullToZeroIfNeeded(Float value) { return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_FLOAT : value; } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 8086f5ed0753..b47d47eee743 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -59,7 +59,7 @@ public class StringDimensionIndexer implements DimensionIndexer { private static final Function STRING_TRANSFORMER = o -> o != null - ? NullHandlingHelper.defaultToNull(o.toString()) + ? NullHandlingHelper.emptyToNullIfNeeded(o.toString()) : null; private static final int ABSENT_VALUE_ID = -1; diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index e4a432d3c81a..f6768227b851 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -93,7 +93,7 @@ public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) // nulBytes will be -1 for null values. return null; } - return NullHandlingHelper.defaultToNull(StringUtils.fromUtf8Nullable(buffer, numBytes)); + return NullHandlingHelper.emptyToNullIfNeeded(StringUtils.fromUtf8Nullable(buffer, numBytes)); } @Override diff --git a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java index bdc134d46f0e..62cceaa0e032 100644 --- a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java @@ -149,7 +149,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasLowerBound()) { startIndex = 0; } else { - final int found = bitmapIndex.getIndex(NullHandlingHelper.defaultToNull(boundDimFilter.getLower())); + final int found = bitmapIndex.getIndex(NullHandlingHelper.emptyToNullIfNeeded(boundDimFilter.getLower())); if (found >= 0) { startIndex = boundDimFilter.isLowerStrict() ? found + 1 : found; } else { @@ -160,7 +160,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasUpperBound()) { endIndex = bitmapIndex.getCardinality(); } else { - final int found = bitmapIndex.getIndex(NullHandlingHelper.defaultToNull(boundDimFilter.getUpper())); + final int found = bitmapIndex.getIndex(NullHandlingHelper.emptyToNullIfNeeded(boundDimFilter.getUpper())); if (found >= 0) { endIndex = boundDimFilter.isUpperStrict() ? found : found + 1; } else { @@ -250,10 +250,10 @@ private boolean doesMatch(String input) { if (input == null) { return (!boundDimFilter.hasLowerBound() - || (NullHandlingHelper.isNullOrDefault(boundDimFilter.getLower()) && !boundDimFilter.isLowerStrict())) + || (NullHandlingHelper.isNullOrEquivalent(boundDimFilter.getLower()) && !boundDimFilter.isLowerStrict())) // lower bound allows null && (!boundDimFilter.hasUpperBound() - || !NullHandlingHelper.isNullOrDefault(boundDimFilter.getUpper()) + || !NullHandlingHelper.isNullOrEquivalent(boundDimFilter.getUpper()) || !boundDimFilter.isUpperStrict()); // upper bound allows null } int lowerComparing = 1; diff --git a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java index 554f420f57c7..b388ccf8e714 100644 --- a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java @@ -109,7 +109,7 @@ public T getBitmapResult(final BitmapIndexSelector selector, final BitmapRes // There's only one binding, and it must be the single column, so it can safely be ignored in production. assert column.equals(identifierName); // convert null to Empty before passing to expressions if needed. - return NullHandlingHelper.nullToDefault(value); + return NullHandlingHelper.nullToEmptyIfNeeded(value); }).asBoolean() ); } diff --git a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java index 8b65d15d9e03..1b92cb660fb1 100644 --- a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java @@ -93,7 +93,7 @@ private Iterable getBitmapIterable(final BitmapIndexSelector se // Verify that dimension equals prefix. return ImmutableList.of(selector.getBitmapIndex( dimension, - NullHandlingHelper.defaultToNull(likeMatcher.getPrefix()) + NullHandlingHelper.emptyToNullIfNeeded(likeMatcher.getPrefix()) )); } else if (isSimplePrefix()) { // Verify that dimension startsWith prefix, and is accepted by likeMatcher.matchesSuffixOnly. diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java index 132e0f020c02..ca97f69e23be 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java @@ -104,13 +104,13 @@ static Supplier supplierFromDimensionSelector(final DimensionSelector se final IndexedInts row = selector.getRow(); if (row.size() == 0) { // Treat empty multi-value rows as nulls. - return NullHandlingHelper.nullToDefault((String) null); + return NullHandlingHelper.nullToEmptyIfNeeded((String) null); } else if (row.size() == 1) { - return NullHandlingHelper.nullToDefault(selector.lookupName(row.get(0))); + return NullHandlingHelper.nullToEmptyIfNeeded(selector.lookupName(row.get(0))); } else { // Can't handle multi-value rows in expressions. // Treat them as nulls until we think of something better to do. - return NullHandlingHelper.nullToDefault((String) null); + return NullHandlingHelper.nullToEmptyIfNeeded((String) null); } }; } @@ -121,7 +121,7 @@ static Supplier supplierFromObjectSelector(final BaseObjectColumnValueSe { if (selector == null) { // Missing column. - return Suppliers.ofInstance(NullHandlingHelper.nullToDefault((String) null)); + return Suppliers.ofInstance(NullHandlingHelper.nullToEmptyIfNeeded((String) null)); } final Class clazz = selector.classOfObject(); @@ -133,11 +133,11 @@ static Supplier supplierFromObjectSelector(final BaseObjectColumnValueSe return () -> { final Object val = selector.getObject(); if (val instanceof String) { - return NullHandlingHelper.nullToDefault((String) val); + return NullHandlingHelper.nullToEmptyIfNeeded((String) val); } else if (val instanceof Number) { return val; } else { - return NullHandlingHelper.nullToDefault((String) null); + return NullHandlingHelper.nullToEmptyIfNeeded((String) null); } }; } else { diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index a3eca30f06e9..ba46c6edc528 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -107,7 +107,7 @@ class DefaultExpressionDimensionSelector extends BaseSingleValueDimensionSelecto @Override protected String getValue() { - return NullHandlingHelper.defaultToNull(baseSelector.getObject().asString()); + return NullHandlingHelper.emptyToNullIfNeeded(baseSelector.getObject().asString()); } @Override @@ -123,7 +123,7 @@ class ExtractionExpressionDimensionSelector extends BaseSingleValueDimensionSele @Override protected String getValue() { - return extractionFn.apply(NullHandlingHelper.defaultToNull(baseSelector.getObject().asString())); + return extractionFn.apply(NullHandlingHelper.emptyToNullIfNeeded(baseSelector.getObject().asString())); } @Override diff --git a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java index 147e208d8f83..8147a3517c54 100644 --- a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java +++ b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java @@ -136,7 +136,7 @@ public void testRetainMissing() false ); final String out = fn.apply(in); - Assert.assertEquals(NullHandlingHelper.isNullOrDefault(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandlingHelper.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); } @Test @@ -150,7 +150,7 @@ public void testRetainMissingButFound() false ); final String out = fn.apply(in); - Assert.assertEquals(NullHandlingHelper.isNullOrDefault(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandlingHelper.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); } @Test @@ -165,7 +165,7 @@ public void testReplaceMissing() ); final String out = fn.apply(in); if (NullHandlingHelper.useDefaultValuesForNull()) { - Assert.assertEquals(NullHandlingHelper.isNullOrDefault(out) ? MISSING : out, exFn.apply(in)); + Assert.assertEquals(NullHandlingHelper.isNullOrEquivalent(out) ? MISSING : out, exFn.apply(in)); } else { Assert.assertEquals(out == null ? MISSING : out, exFn.apply(in)); } @@ -217,7 +217,7 @@ public void testNullInputs() null, false ); - if (NullHandlingHelper.isNullOrDefault(fn.apply(null))) { + if (NullHandlingHelper.isNullOrEquivalent(fn.apply(null))) { Assert.assertEquals(null, exFn.apply(null)); } } diff --git a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java index f4bac53b8fc5..c38f31f9e033 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java @@ -184,7 +184,7 @@ public void testStringColumnNullHandling() throws Exception final List expectedNullRows = new ArrayList<>(); for (int i = 0; i < index.getNumRows(); i++) { final List row = getRow(dictionaryColumn, i); - if (row.isEmpty() || row.stream().anyMatch(NullHandlingHelper::isNullOrDefault)) { + if (row.isEmpty() || row.stream().anyMatch(NullHandlingHelper::isNullOrEquivalent)) { expectedNullRows.add(i); } } @@ -221,14 +221,14 @@ private static List normalize(final Object value, final boolean hasMulti if (value == null) { retVal.add(null); } else if (value instanceof String) { - retVal.add(NullHandlingHelper.defaultToNull(((String) value))); + retVal.add(NullHandlingHelper.emptyToNullIfNeeded(((String) value))); } else if (value instanceof List) { final List list = (List) value; if (list.isEmpty() && !hasMultipleValues) { // empty lists become nulls in single valued columns - retVal.add(NullHandlingHelper.defaultToNull(null)); + retVal.add(NullHandlingHelper.emptyToNullIfNeeded(null)); } else { - retVal.addAll(list.stream().map(NullHandlingHelper::defaultToNull).collect(Collectors.toList())); + retVal.addAll(list.stream().map(NullHandlingHelper::emptyToNullIfNeeded).collect(Collectors.toList())); } } else { throw new ISE("didn't expect class[%s]", value.getClass()); diff --git a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java index 300b33ffc6cb..48d5899afdb0 100644 --- a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java @@ -149,7 +149,7 @@ public void testNullSerializationWithoutCache() throws Exception ) ); Assert.assertNull(deserialized.get(0)); - Assert.assertEquals(deserialized.get(1), NullHandlingHelper.defaultToNull("")); + Assert.assertEquals(deserialized.get(1), NullHandlingHelper.emptyToNullIfNeeded("")); } @Test @@ -166,7 +166,7 @@ public void testNullSerialization() throws Exception unCached, CachingIndexed.INITIAL_CACHE_CAPACITY); Assert.assertNull(deserialized.get(0)); - Assert.assertEquals(deserialized.get(1), NullHandlingHelper.defaultToNull("")); + Assert.assertEquals(deserialized.get(1), NullHandlingHelper.emptyToNullIfNeeded("")); } diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java index dfda52ecccf6..766ecdeb19e1 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java @@ -232,7 +232,7 @@ public void testNullDimensionTransform() throws IndexSizeExceededException row.getRaw("float") ); Assert.assertEquals(Arrays.asList(null, null, String.valueOf(Long.MIN_VALUE)), row.getRaw("long")); - Assert.assertEquals(NullHandlingHelper.nullToDefault((Double) null), row.getRaw("double")); + Assert.assertEquals(NullHandlingHelper.nullToZeroIfNeeded((Double) null), row.getRaw("double")); } else { Assert.assertEquals(Arrays.asList(null, "", "A"), row.getRaw("string")); Assert.assertEquals( @@ -240,7 +240,7 @@ public void testNullDimensionTransform() throws IndexSizeExceededException row.getRaw("float") ); Assert.assertEquals(Arrays.asList(null, "", String.valueOf(Long.MIN_VALUE)), row.getRaw("long")); - Assert.assertEquals(NullHandlingHelper.nullToDefault((Double) null), row.getRaw("double")); + Assert.assertEquals(NullHandlingHelper.nullToZeroIfNeeded((Double) null), row.getRaw("double")); } } diff --git a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java index d03ddb741688..637fa77d51a0 100644 --- a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java +++ b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -78,7 +78,7 @@ public LookupDimensionSpec( { this.retainMissingValue = retainMissingValue; this.optimize = optimize == null ? true : optimize; - this.replaceMissingValueWith = NullHandlingHelper.defaultToNull(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandlingHelper.emptyToNullIfNeeded(replaceMissingValueWith); this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be Null"); this.outputName = Preconditions.checkNotNull(outputName, "outputName can not be Null"); this.lookupReferencesManager = lookupReferencesManager; diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java index 7b9f3c3940e3..9484ab68983c 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java @@ -305,7 +305,7 @@ private static DimFilter toSimpleLeafFilter( final DimFilter equalFilter = new SelectorDimFilter( druidExpression.getSimpleExtraction().getColumn(), - NullHandlingHelper.nullToDefault((String) null), + NullHandlingHelper.nullToEmptyIfNeeded((String) null), druidExpression.getSimpleExtraction().getExtractionFn() ); diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java index 9c0d46731e52..6260b6bc719a 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java @@ -398,7 +398,7 @@ private Object coerce(final Object value, final SqlTypeName sqlType) if (SqlTypeName.CHAR_TYPES.contains(sqlType)) { if (value == null || value instanceof String) { - coercedValue = NullHandlingHelper.nullToDefault((String) value); + coercedValue = NullHandlingHelper.nullToEmptyIfNeeded((String) value); } else if (value instanceof NlsString) { coercedValue = ((NlsString) value).getValue(); } else if (value instanceof Number) { From 6b28b982c03a149d989ab7b735a24e984e86b90c Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 22 Nov 2017 20:32:41 +0530 Subject: [PATCH 34/50] review comments --- .../common/config/NullHandlingExpressionHelper.java | 4 +++- common/src/main/java/io/druid/math/expr/Function.java | 2 +- ...leCardinalityAggregatorColumnSelectorStrategy.java | 9 +++++++-- ...atCardinalityAggregatorColumnSelectorStrategy.java | 9 +++++++-- ...ngCardinalityAggregatorColumnSelectorStrategy.java | 9 +++++++-- .../io/druid/query/filter/BitmapIndexSelector.java | 6 ------ .../java/io/druid/query/filter/SelectorDimFilter.java | 3 ++- .../java/io/druid/segment/DimensionHandlerUtils.java | 2 +- .../io/druid/segment/DoubleDimensionMergerV9.java | 2 +- .../java/io/druid/segment/NullHandlingHelper.java | 3 ++- .../java/io/druid/segment/StringDimensionIndexer.java | 11 +++++------ .../io/druid/segment/transform/TransformSpec.java | 2 +- .../src/test/java/io/druid/segment/TestHelper.java | 2 +- .../segment/incremental/IncrementalIndexTest.java | 4 ++-- sql/src/main/java/io/druid/sql/avatica/DruidMeta.java | 3 +-- 15 files changed, 41 insertions(+), 30 deletions(-) diff --git a/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java b/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java index 0bc6a453648b..7f888e006197 100644 --- a/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java +++ b/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java @@ -28,7 +28,9 @@ public class NullHandlingExpressionHelper // INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. // See NullHandlingModule for details. - // The default system property is supposed to be used only in tests. + // It does not take effect in all unit tests since we don't use Guice Injection. + // For tests default system property is supposed to be used only in tests + @Inject private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index 583dfa053fed..e70a26995a7b 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -990,7 +990,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) return ExprEval.of(arg.substring(index)); } } else { - // If starting index of substring is greater then the lengh of string, the result will be a zero length string. + // If starting index of substring is greater then the length of string, the result will be a zero length string. // e.g. 'select substring("abc", 4,5) as c;' will return an empty string return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded((String) null)); } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java index 5fd15ae30c1b..11bec1edf3e7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java @@ -23,6 +23,7 @@ import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.NullHandlingHelper; public class DoubleCardinalityAggregatorColumnSelectorStrategy @@ -31,12 +32,16 @@ public class DoubleCardinalityAggregatorColumnSelectorStrategy @Override public void hashRow(BaseDoubleColumnValueSelector dimSelector, Hasher hasher) { - hasher.putDouble(dimSelector.getDouble()); + if (NullHandlingHelper.useDefaultValuesForNull() || !dimSelector.isNull()) { + hasher.putDouble(dimSelector.getDouble()); + } } @Override public void hashValues(BaseDoubleColumnValueSelector dimSelector, HyperLogLogCollector collector) { - collector.add(CardinalityAggregator.hashFn.hashLong(Double.doubleToLongBits(dimSelector.getDouble())).asBytes()); + if (NullHandlingHelper.useDefaultValuesForNull() || !dimSelector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashLong(Double.doubleToLongBits(dimSelector.getDouble())).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java index b46261c7b15e..c22b1477eacb 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java @@ -23,6 +23,7 @@ import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.NullHandlingHelper; public class FloatCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy @@ -30,12 +31,16 @@ public class FloatCardinalityAggregatorColumnSelectorStrategy @Override public void hashRow(BaseFloatColumnValueSelector selector, Hasher hasher) { - hasher.putFloat(selector.getFloat()); + if (NullHandlingHelper.useDefaultValuesForNull() || !selector.isNull()) { + hasher.putFloat(selector.getFloat()); + } } @Override public void hashValues(BaseFloatColumnValueSelector selector, HyperLogLogCollector collector) { - collector.add(CardinalityAggregator.hashFn.hashInt(Float.floatToIntBits(selector.getFloat())).asBytes()); + if (NullHandlingHelper.useDefaultValuesForNull() || !selector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashInt(Float.floatToIntBits(selector.getFloat())).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java index a666ed64e1a7..bdfda2f6a8f5 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java @@ -23,6 +23,7 @@ import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.NullHandlingHelper; public class LongCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy @@ -30,12 +31,16 @@ public class LongCardinalityAggregatorColumnSelectorStrategy @Override public void hashRow(BaseLongColumnValueSelector dimSelector, Hasher hasher) { - hasher.putLong(dimSelector.getLong()); + if (NullHandlingHelper.useDefaultValuesForNull() || !dimSelector.isNull()) { + hasher.putLong(dimSelector.getLong()); + } } @Override public void hashValues(BaseLongColumnValueSelector dimSelector, HyperLogLogCollector collector) { - collector.add(CardinalityAggregator.hashFn.hashLong(dimSelector.getLong()).asBytes()); + if (NullHandlingHelper.useDefaultValuesForNull() || !dimSelector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashLong(dimSelector.getLong()).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java index cdcfa7665f8e..776480068c3e 100644 --- a/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java @@ -30,16 +30,10 @@ public interface BitmapIndexSelector { Indexed getDimensionValues(String dimension); - boolean hasMultipleValues(String dimension); - int getNumRows(); - BitmapFactory getBitmapFactory(); - BitmapIndex getBitmapIndex(String dimension); - ImmutableBitmap getBitmapIndex(String dimension, String value); - ImmutableRTree getSpatialIndex(String dimension); } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 25b8a2b82f5c..2749467877cc 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -76,9 +76,10 @@ public byte[] getCacheKey() byte[] valueBytes = (value == null) ? new byte[]{} : StringUtils.toUtf8(value); byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - return ByteBuffer.allocate(5 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) + return ByteBuffer.allocate(6 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) .put(DimFilterUtils.SELECTOR_CACHE_ID) .put(value == null ? (byte) 1 : (byte) 0) + .put(DimFilterUtils.STRING_SEPARATOR) .put(dimensionBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(valueBytes) diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java index 50edf32dd747..a7403be88d3b 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java @@ -296,7 +296,7 @@ public static Double convertObjectToDouble(@Nullable Object valObj) return ((Number) valObj).doubleValue(); } else if (valObj instanceof String) { Double doubleVal = Doubles.tryParse((String) valObj); - return NullHandlingHelper.nullToZeroIfNeeded(doubleVal); + return doubleVal; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java index 3a92bbd3c8e5..c1aecc2ce319 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java @@ -103,7 +103,7 @@ public void processMergedRow(Double rowValues) throws IOException @Override public void writeIndexes(List segmentRowNumConversions) throws IOException { - // doubless have no indices to write + // doubles have no indices to write } @Override diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index d1b557ef075c..1e762843af58 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -42,7 +42,8 @@ public class NullHandlingHelper // INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. // See NullHandlingModule for details. - // The default system property is supposed to be used only in tests. + // It does not take effect in all unit tests since we don't use Guice Injection. + // For tests default system property is supposed to be used only in tests @Inject private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 57c6190a531b..fe18cdb7eafb 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -57,7 +57,7 @@ public class StringDimensionIndexer implements DimensionIndexer { - private static final Function STRING_TRANSFORMER = o -> o != null + private static final Function EMPTY_TO_NULL_IF_NEEDED = o -> o != null ? NullHandlingHelper.emptyToNullIfNeeded(o.toString()) : null; @@ -87,8 +87,7 @@ public int getId(String value) if (value == null) { return idForNull; } - final int id = valueToId.getInt(value); - return id < 0 ? ABSENT_VALUE_ID : id; + return valueToId.getInt(value); } } @@ -235,11 +234,11 @@ public int[] processRowValsToUnsortedEncodedKeyComponent(Object dimValues) dimLookup.add(null); encodedDimensionValues = EMPTY_INT_ARRAY; } else if (dimValuesList.size() == 1) { - encodedDimensionValues = new int[]{dimLookup.add(STRING_TRANSFORMER.apply(dimValuesList.get(0)))}; + encodedDimensionValues = new int[]{dimLookup.add(EMPTY_TO_NULL_IF_NEEDED.apply(dimValuesList.get(0)))}; } else { final String[] dimensionValues = new String[dimValuesList.size()]; for (int i = 0; i < dimValuesList.size(); i++) { - dimensionValues[i] = STRING_TRANSFORMER.apply(dimValuesList.get(i)); + dimensionValues[i] = EMPTY_TO_NULL_IF_NEEDED.apply(dimValuesList.get(i)); } if (multiValueHandling.needSorting()) { // Sort multival row by their unencoded values first. @@ -264,7 +263,7 @@ public int[] processRowValsToUnsortedEncodedKeyComponent(Object dimValues) encodedDimensionValues = pos == retVal.length ? retVal : Arrays.copyOf(retVal, pos); } } else { - encodedDimensionValues = new int[]{dimLookup.add(STRING_TRANSFORMER.apply(dimValues))}; + encodedDimensionValues = new int[]{dimLookup.add(EMPTY_TO_NULL_IF_NEEDED.apply(dimValues))}; } // If dictionary size has changed, the sorted lookup is no longer valid. diff --git a/processing/src/main/java/io/druid/segment/transform/TransformSpec.java b/processing/src/main/java/io/druid/segment/transform/TransformSpec.java index d01880d28963..706b022c7f24 100644 --- a/processing/src/main/java/io/druid/segment/transform/TransformSpec.java +++ b/processing/src/main/java/io/druid/segment/transform/TransformSpec.java @@ -39,7 +39,7 @@ * Specifies how input rows should be filtered and transforms. There are two parts: a "filter" (which can filter out * input rows) and "transforms" (which can add fields to input rows). Filters may refer to fields generated by * a transform. - *

+ * * See {@link Transform} for details on how each transform works. */ public class TransformSpec diff --git a/processing/src/test/java/io/druid/segment/TestHelper.java b/processing/src/test/java/io/druid/segment/TestHelper.java index 1dcb914b98c9..69fd684b4bdf 100644 --- a/processing/src/test/java/io/druid/segment/TestHelper.java +++ b/processing/src/test/java/io/druid/segment/TestHelper.java @@ -328,7 +328,7 @@ private static void assertRow(String msg, Row expected, Row actual) StringUtils.format("%s: key[%s]", msg, key), ((Number) expectedValue).doubleValue(), ((Number) actualValue).doubleValue(), - ((Number) expectedValue).doubleValue() * 1e-6 + Math.abs(((Number) expectedValue).doubleValue() * 1e-6) ); } } else { diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java index 766ecdeb19e1..4ceab59b4c28 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java @@ -232,7 +232,6 @@ public void testNullDimensionTransform() throws IndexSizeExceededException row.getRaw("float") ); Assert.assertEquals(Arrays.asList(null, null, String.valueOf(Long.MIN_VALUE)), row.getRaw("long")); - Assert.assertEquals(NullHandlingHelper.nullToZeroIfNeeded((Double) null), row.getRaw("double")); } else { Assert.assertEquals(Arrays.asList(null, "", "A"), row.getRaw("string")); Assert.assertEquals( @@ -240,8 +239,9 @@ public void testNullDimensionTransform() throws IndexSizeExceededException row.getRaw("float") ); Assert.assertEquals(Arrays.asList(null, "", String.valueOf(Long.MIN_VALUE)), row.getRaw("long")); - Assert.assertEquals(NullHandlingHelper.nullToZeroIfNeeded((Double) null), row.getRaw("double")); } + Assert.assertEquals(null, row.getRaw("double")); + } @Test diff --git a/sql/src/main/java/io/druid/sql/avatica/DruidMeta.java b/sql/src/main/java/io/druid/sql/avatica/DruidMeta.java index 5b04525bf1f6..eb1a32961565 100644 --- a/sql/src/main/java/io/druid/sql/avatica/DruidMeta.java +++ b/sql/src/main/java/io/druid/sql/avatica/DruidMeta.java @@ -183,8 +183,7 @@ public ExecuteResult prepareAndExecute( if (authenticationResult == null) { throw new ForbiddenException("Authentication failed."); } - final Signature signature = druidStatement.prepare(plannerFactory, sql, maxRowCount, authenticationResult) - .getSignature(); + final Signature signature = druidStatement.prepare(plannerFactory, sql, maxRowCount, authenticationResult).getSignature(); final Frame firstFrame = druidStatement.execute() .nextFrame( DruidStatement.START_OFFSET, From eb24122d51930fa401f1f0051640a8102ab78db0 Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 7 Dec 2017 22:39:21 +0530 Subject: [PATCH 35/50] review comments --- .travis.yml | 6 +- .../main/java/io/druid/data/input/Row.java | 9 ++- .../main/java/io/druid/data/input/Rows.java | 2 + ...ssionHelper.java => NullHandlingUtil.java} | 8 +- .../main/java/io/druid/math/expr/Expr.java | 10 +-- .../java/io/druid/math/expr/ExprEval.java | 4 +- .../java/io/druid/math/expr/Function.java | 18 ++--- .../java/io/druid/math/expr/EvalTest.java | 4 +- .../java/io/druid/math/expr/FunctionTest.java | 4 +- .../java/io/druid/indexer/InputRowSerde.java | 8 +- .../io/druid/guice/NullHandlingModule.java | 4 +- .../io/druid/query/filter/ValueGetter.java | 3 + .../druid/sql/calcite/CalciteQueryTest.java | 79 ++++++------------- 13 files changed, 71 insertions(+), 88 deletions(-) rename common/src/main/java/io/druid/common/config/{NullHandlingExpressionHelper.java => NullHandlingUtil.java} (89%) diff --git a/.travis.yml b/.travis.yml index 4402776c406c..0ec078d6b2dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,7 @@ matrix: # processing module test with SQL Compatible Null Handling - sudo: false env: - - NAME="processing module test" + - NAME="processing module test with SQL Compatible Null Handling" install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B before_script: - unset _JAVA_OPTIONS @@ -60,7 +60,7 @@ matrix: # server module test with SQL Compatible Null Handling - sudo: false env: - - NAME="server module test" + - NAME="server module test with SQL Compatible Null Handling" install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -Ddruid.generic.useDefaultValueForNull=false -q -ff -DskipTests -B before_script: - unset _JAVA_OPTIONS @@ -70,7 +70,7 @@ matrix: # other modules test with SQL Compatible Null Handling - sudo: false env: - - NAME="other modules test" + - NAME="other modules test with SQL Compatible Null Handling" install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B before_script: - unset _JAVA_OPTIONS diff --git a/api/src/main/java/io/druid/data/input/Row.java b/api/src/main/java/io/druid/data/input/Row.java index f6f9974d4dd6..d63730b023b0 100644 --- a/api/src/main/java/io/druid/data/input/Row.java +++ b/api/src/main/java/io/druid/data/input/Row.java @@ -24,6 +24,7 @@ import io.druid.guice.annotations.PublicApi; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.List; /** @@ -71,13 +72,13 @@ public interface Row extends Comparable * * @return the value of the provided column name */ + @Nullable Object getRaw(String dimension); /** - * Returns the metric column value for the given column name. This method is different from {@link #getRaw} in two - * aspects: - * 1. If the column is absent in the row, null is returned. - * 2. If the column has string value, an attempt is made to parse this value as a number. + * Returns the metric column value for the given column name. This method is different from {@link #getRaw} + * when column has string value, an attempt is made to parse this value as a number. */ + @Nullable Number getMetric(String metric); } diff --git a/api/src/main/java/io/druid/data/input/Rows.java b/api/src/main/java/io/druid/data/input/Rows.java index 3e7510c22aae..6d72093a8211 100644 --- a/api/src/main/java/io/druid/data/input/Rows.java +++ b/api/src/main/java/io/druid/data/input/Rows.java @@ -26,6 +26,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.parsers.ParseException; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -92,6 +93,7 @@ public static List objectToStrings(final Object inputValue) * @throws NullPointerException if the string is null * @throws ParseException if the column cannot be converted to a number */ + @Nullable public static Number objectToNumber(final String name, final Object inputValue) { if (inputValue == null) { diff --git a/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java b/common/src/main/java/io/druid/common/config/NullHandlingUtil.java similarity index 89% rename from common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java rename to common/src/main/java/io/druid/common/config/NullHandlingUtil.java index 7f888e006197..e053c0947ba8 100644 --- a/common/src/main/java/io/druid/common/config/NullHandlingExpressionHelper.java +++ b/common/src/main/java/io/druid/common/config/NullHandlingUtil.java @@ -22,7 +22,9 @@ import com.google.common.base.Strings; import com.google.inject.Inject; -public class NullHandlingExpressionHelper +import javax.annotation.Nullable; + +public class NullHandlingUtil { private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; @@ -41,12 +43,12 @@ public static boolean useDefaultValuesForNull() return INSTANCE.isUseDefaultValuesForNull(); } - public static String nullToEmptyIfNeeded(String value) + public static @Nullable String nullToEmptyIfNeeded(@Nullable String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; } - public static String emptyToNullIfNeeded(String value) + public static @Nullable String emptyToNullIfNeeded(@Nullable String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; } diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index aec18d677390..b11c619fc423 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -22,7 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.math.LongMath; import com.google.common.primitives.Ints; -import io.druid.common.config.NullHandlingExpressionHelper; +import io.druid.common.config.NullHandlingUtil; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Comparators; @@ -124,7 +124,7 @@ class StringExpr extends ConstantExpr public StringExpr(String value) { - this.value = NullHandlingExpressionHelper.emptyToNullIfNeeded(value); + this.value = NullHandlingUtil.emptyToNullIfNeeded(value); } @Nullable @@ -365,7 +365,7 @@ public ExprEval eval(ObjectBinding bindings) // Result of any Binary expressions is null if any of the argument is null. // e.g "select null * 2 as c;" or "select null + 1 as c;" will return null as per Standard SQL spec. - if (!NullHandlingExpressionHelper.useDefaultValuesForNull() && (leftVal.isNull() || rightVal.isNull())) { + if (!NullHandlingUtil.useDefaultValuesForNull() && (leftVal.isNull() || rightVal.isNull())) { return ExprEval.of(null); } @@ -498,8 +498,8 @@ class BinPlusExpr extends BinaryEvalOpExprBase @Override protected ExprEval evalString(@Nullable String left, @Nullable String right) { - return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded(left) - + NullHandlingExpressionHelper.nullToEmptyIfNeeded(right)); + return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded(left) + + NullHandlingUtil.nullToEmptyIfNeeded(right)); } @Override diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 7768dae39844..650fccb5d1cd 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -22,7 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; import com.google.common.primitives.Ints; -import io.druid.common.config.NullHandlingExpressionHelper; +import io.druid.common.config.NullHandlingUtil; import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.IAE; @@ -233,7 +233,7 @@ private static class StringExprEval extends ExprEval { private StringExprEval(String value) { - super(NullHandlingExpressionHelper.emptyToNullIfNeeded(value)); + super(NullHandlingUtil.emptyToNullIfNeeded(value)); } @Override diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index e70a26995a7b..b326d4bbc3c0 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -20,7 +20,7 @@ package io.druid.math.expr; import com.google.common.base.Strings; -import io.druid.common.config.NullHandlingExpressionHelper; +import io.druid.common.config.NullHandlingUtil; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.StringUtils; @@ -75,7 +75,7 @@ abstract class SingleParamMath extends SingleParam @Override protected final ExprEval eval(ExprEval param) { - if (!NullHandlingExpressionHelper.useDefaultValuesForNull() && param.isNull()) { + if (!NullHandlingUtil.useDefaultValuesForNull() && param.isNull()) { return ExprEval.of(null); } if (param.type() == ExprType.LONG) { @@ -903,12 +903,12 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final StringBuilder builder = new StringBuilder(Strings.nullToEmpty(args.get(0).eval(bindings).asString())); for (int i = 1; i < args.size(); i++) { final String s = args.get(i).eval(bindings).asString(); - if (!NullHandlingExpressionHelper.useDefaultValuesForNull() && s == null) { + if (!NullHandlingUtil.useDefaultValuesForNull() && s == null) { // Result of concatenation is null if any of the Values is null. // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. return ExprEval.of(null); } else { - builder.append(NullHandlingExpressionHelper.nullToEmptyIfNeeded(s)); + builder.append(NullHandlingUtil.nullToEmptyIfNeeded(s)); } } return ExprEval.of(builder.toString()); @@ -992,7 +992,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } else { // If starting index of substring is greater then the length of string, the result will be a zero length string. // e.g. 'select substring("abc", 4,5) as c;' will return an empty string - return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded((String) null)); + return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded((String) null)); } } } @@ -1016,10 +1016,10 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String pattern = args.get(1).eval(bindings).asString(); final String replacement = args.get(2).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded((String) null)); + return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded(null)); } return ExprEval.of( - NullHandlingExpressionHelper.nullToEmptyIfNeeded(arg).replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) + arg.replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) ); } } @@ -1041,7 +1041,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded((String) null)); + return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded((String) null)); } return ExprEval.of(StringUtils.toLowerCase(arg)); } @@ -1064,7 +1064,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandlingExpressionHelper.nullToEmptyIfNeeded((String) null)); + return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded((String) null)); } return ExprEval.of(StringUtils.toUpperCase(arg)); } diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java index 456e8fe67122..c3458bb8ac00 100644 --- a/common/src/test/java/io/druid/math/expr/EvalTest.java +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -20,7 +20,7 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; -import io.druid.common.config.NullHandlingExpressionHelper; +import io.druid.common.config.NullHandlingUtil; import org.junit.Assert; import org.junit.Test; @@ -140,7 +140,7 @@ public void testLongEval() Assert.assertEquals(1271055781L, evalLong("unix_timestamp('2010-04-12T07:03:01')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01+09:00')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01.419+09:00')", bindings)); - if (NullHandlingExpressionHelper.useDefaultValuesForNull()) { + if (NullHandlingUtil.useDefaultValuesForNull()) { Assert.assertEquals("NULL", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); } else { Assert.assertEquals("", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); diff --git a/common/src/test/java/io/druid/math/expr/FunctionTest.java b/common/src/test/java/io/druid/math/expr/FunctionTest.java index 37dbf214a88f..27a98ef1982c 100644 --- a/common/src/test/java/io/druid/math/expr/FunctionTest.java +++ b/common/src/test/java/io/druid/math/expr/FunctionTest.java @@ -20,7 +20,7 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; -import io.druid.common.config.NullHandlingExpressionHelper; +import io.druid.common.config.NullHandlingUtil; import org.junit.Assert; import org.junit.Test; @@ -56,7 +56,7 @@ public void testCaseSearched() public void testConcat() { assertExpr("concat(x,' ',y)", "foo 2"); - if (NullHandlingExpressionHelper.useDefaultValuesForNull()) { + if (NullHandlingUtil.useDefaultValuesForNull()) { assertExpr("concat(x,' ',nonexistent,' ',y)", "foo 2"); } else { assertExpr("concat(x,' ',nonexistent,' ',y)", null); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index 29f8d4de8b13..86f002f9b0b9 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -49,6 +49,8 @@ public class InputRowSerde { private static final Logger log = new Logger(InputRowSerde.class); + private static final byte TRUE_BYTE = (byte) 1; + private static final byte FALSE_BYTE = (byte) 0; public static final byte[] toBytes(final InputRow row, AggregatorFactory[] aggs, boolean reportParseExceptions) { @@ -105,9 +107,9 @@ public InputRow get() String t = aggFactory.getTypeName(); if (agg.isNull()) { - out.writeByte((byte) 1); + out.writeByte(TRUE_BYTE); } else { - out.writeByte((byte) 0); + out.writeByte(FALSE_BYTE); if (t.equals("float")) { out.writeFloat(agg.getFloat()); } else if (t.equals("long")) { @@ -222,7 +224,7 @@ public static final InputRow fromBytes(byte[] data, AggregatorFactory[] aggs) String metric = readString(in); String type = getType(metric, aggs, i); byte metricNullability = in.readByte(); - if (metricNullability == (byte) 1) { + if (metricNullability == TRUE_BYTE) { // metric value is null. continue; } diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java index 8a2ff7b52e3e..b4f448031e07 100644 --- a/processing/src/main/java/io/druid/guice/NullHandlingModule.java +++ b/processing/src/main/java/io/druid/guice/NullHandlingModule.java @@ -21,7 +21,7 @@ import com.google.inject.Binder; import com.google.inject.Module; -import io.druid.common.config.NullHandlingExpressionHelper; +import io.druid.common.config.NullHandlingUtil; import io.druid.segment.NullHandlingHelper; import io.druid.common.config.NullValueHandlingConfig; @@ -34,7 +34,7 @@ public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.generic", NullValueHandlingConfig.class); binder.requestStaticInjection(NullHandlingHelper.class); - binder.requestStaticInjection(NullHandlingExpressionHelper.class); + binder.requestStaticInjection(NullHandlingUtil.class); } } diff --git a/processing/src/main/java/io/druid/query/filter/ValueGetter.java b/processing/src/main/java/io/druid/query/filter/ValueGetter.java index 72000a4756da..c478ce971b37 100644 --- a/processing/src/main/java/io/druid/query/filter/ValueGetter.java +++ b/processing/src/main/java/io/druid/query/filter/ValueGetter.java @@ -19,6 +19,8 @@ package io.druid.query.filter; +import javax.annotation.Nullable; + /** */ public interface ValueGetter @@ -27,5 +29,6 @@ public interface ValueGetter // converted to strings. We should also add functions // for these and modify ColumnComparisonFilter to handle // comparing Long and Float columns to eachother. + @Nullable String[] get(); } diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 56badc541a5b..93d44fccadef 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -1631,8 +1631,12 @@ public void testGroupByCaseWhen() throws Exception } @Test - public void testNullEmptyStringEquality() throws Exception + public void testNullEmptyStringEqualityBackwardsCompatibility() throws Exception { + if(!NullHandlingHelper.useDefaultValuesForNull()){ + // This test only makes sense when nulls and empty strings are considered equal. + return; + } // Doesn't conform to the SQL standard, but it's how we do it. // This example is used in the sql.md doc. @@ -1641,58 +1645,27 @@ public void testNullEmptyStringEquality() throws Exception "NULLIF(dim2, 'a') IS NULL" ); - testQuery( - "SELECT COUNT(*)\n" - + "FROM druid.foo\n" - + "WHERE " + wheres.get(0), - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," - + (NullHandlingHelper.useDefaultValuesForNull() ? "1" : "0") - + ",(\"dim2\" == ''))")) - .aggregators(AGGS(new CountAggregatorFactory("a0"))) - .context(TIMESERIES_CONTEXT_DEFAULT) - .build() - ), - NullHandlingHelper.useDefaultValuesForNull() ? - ImmutableList.of( - // Matches everything but "abc" - new Object[]{5L} - ) : - ImmutableList.of( - // Matches only null value - new Object[]{1L} - ) - ); - - testQuery( - "SELECT COUNT(*)\n" - + "FROM druid.foo\n" - + "WHERE " + wheres.get(1), - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,isnull(\"dim2\"))")) - .aggregators(AGGS(new CountAggregatorFactory("a0"))) - .context(TIMESERIES_CONTEXT_DEFAULT) - .build() - ), - NullHandlingHelper.useDefaultValuesForNull() ? - ImmutableList.of( - // Matches everything but "abc" - new Object[]{5L} - ) : - ImmutableList.of( - // Matches everything but "abc" and ""(Empty String) - new Object[]{4L} - ) - ); - + for (String where : wheres) { + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE " + where, + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,(\"dim2\" == ''))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + // Matches everything but "abc" + new Object[]{5L} + ) + ); + } } @Test From 9362286b253899e9b8a299190d8ad9991d2503d5 Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 8 Dec 2017 03:00:16 +0530 Subject: [PATCH 36/50] merge master and resolve conflicts --- .../druid/data/input/impl/InputRowParser.java | 20 +- .../data/input/impl/MapInputRowParser.java | 5 +- .../data/input/impl/NoopInputRowParser.java | 7 +- .../data/input/impl/StringInputRowParser.java | 11 +- .../data/input/impl/TimeAndDimsParseSpec.java | 2 +- .../java/io/druid/js/JavaScriptConfig.java | 8 +- .../input/impl/InputRowParserSerdeTest.java | 16 +- .../impl/JSONLowercaseParseSpecTest.java | 2 +- .../data/input/impl/JSONParseSpecTest.java | 2 +- .../input/impl/JavaScriptParseSpecTest.java | 2 +- .../CompressedIndexedIntsBenchmark.java | 24 +- .../CompressedVSizeIndexedBenchmark.java | 14 +- .../benchmark/FilterPartitionBenchmark.java | 7 +- .../FilteredAggregatorBenchmark.java | 7 +- .../druid/benchmark/FlattenJSONBenchmark.java | 10 +- .../benchmark/FloatCompressionBenchmark.java | 2 +- ...loatCompressionBenchmarkFileGenerator.java | 48 +- .../benchmark/GenericIndexedBenchmark.java | 7 +- .../GroupByTypeInterfaceBenchmark.java | 7 +- .../benchmark/LongCompressionBenchmark.java | 2 +- ...LongCompressionBenchmarkFileGenerator.java | 49 +- .../benchmark/TopNTypeInterfaceBenchmark.java | 7 +- .../benchmark/datagen/SegmentGenerator.java | 7 +- .../indexing/IndexMergeBenchmark.java | 15 +- .../indexing/IndexPersistBenchmark.java | 7 +- .../benchmark/query/GroupByBenchmark.java | 7 +- .../benchmark/query/SearchBenchmark.java | 7 +- .../benchmark/query/SelectBenchmark.java | 7 +- .../benchmark/query/TimeseriesBenchmark.java | 7 +- .../druid/benchmark/query/TopNBenchmark.java | 7 +- .../FlattenJSONBenchmarkUtilTest.java | 6 +- .../java/io/druid/common/utils/ByteUtils.java | 35 ++ .../druid/common/utils/SerializerUtils.java | 92 +--- .../io/druid/io/ByteBufferInputStream.java | 71 +++ .../src/main/java/io/druid/io/Channels.java | 25 +- .../main/java/io/druid/math/expr/Parser.java | 11 + .../content/configuration/indexing-service.md | 23 + .../extensions-core/kafka-ingestion.md | 1 + docs/content/ingestion/stream-pull.md | 4 +- docs/content/ingestion/tasks.md | 3 +- .../rocketmq/RocketMQFirehoseFactory.java | 13 +- ...fkaEightSimpleConsumerFirehoseFactory.java | 23 +- .../input/orc/OrcHadoopInputRowParser.java | 5 +- .../input/orc/DruidOrcInputFormatTest.java | 2 +- .../orc/OrcHadoopInputRowParserTest.java | 2 +- .../parquet/ParquetHadoopInputRowParser.java | 5 +- .../input/parquet/DruidParquetInputTest.java | 9 +- .../rabbitmq/RabbitMQFirehoseFactory.java | 32 +- .../input/thrift/ThriftInputRowParser.java | 13 +- .../thrift/ThriftInputRowParserTest.java | 6 +- .../data/input/AvroHadoopInputRowParser.java | 4 +- .../data/input/AvroStreamInputRowParser.java | 3 +- .../io/druid/data/input/avro/AvroParsers.java | 6 +- .../input/AvroHadoopInputRowParserTest.java | 2 +- .../input/AvroStreamInputRowParserTest.java | 4 +- extensions-core/datasketches/pom.xml | 2 +- .../DoublesSketchAggregatorFactory.java | 268 ++++++++++ .../DoublesSketchBuildAggregator.java | 78 +++ .../DoublesSketchBuildBufferAggregator.java | 131 +++++ .../DoublesSketchComplexMetricSerde.java | 116 +++++ .../DoublesSketchJsonSerializer.java | 39 ++ .../DoublesSketchMergeAggregator.java | 80 +++ .../DoublesSketchMergeAggregatorFactory.java | 63 +++ .../DoublesSketchMergeBufferAggregator.java | 137 +++++ .../quantiles/DoublesSketchModule.java | 65 +++ .../DoublesSketchNoOpAggregator.java | 60 +++ .../DoublesSketchNoOpBufferAggregator.java | 68 +++ .../DoublesSketchObjectStrategy.java | 64 +++ .../quantiles/DoublesSketchOperations.java | 59 +++ ...oublesSketchToHistogramPostAggregator.java | 153 ++++++ ...DoublesSketchToQuantilePostAggregator.java | 149 ++++++ ...oublesSketchToQuantilesPostAggregator.java | 147 ++++++ .../DoublesSketchToStringPostAggregator.java | 130 +++++ .../theta/SketchMergeComplexMetricSerde.java | 6 +- .../datasketches/theta/SynchronizedUnion.java | 13 +- .../io.druid.initialization.DruidModule | 1 + .../DoublesSketchAggregatorTest.java | 419 +++++++++++++++ .../quantiles/GenerateTestData.java | 72 +++ .../quantiles/doubles_build_data.tsv | 400 +++++++++++++++ .../quantiles/doubles_sketch_data.tsv | 20 + .../ApproximateHistogramFoldingSerde.java | 6 +- .../sql/QuantileSqlAggregatorTest.java | 4 +- .../kafka/KafkaEightFirehoseFactory.java | 17 +- .../druid/indexing/kafka/KafkaIndexTask.java | 177 ++++--- .../indexing/kafka/KafkaTuningConfig.java | 106 ++-- .../kafka/supervisor/KafkaSupervisorSpec.java | 1 + .../KafkaSupervisorTuningConfig.java | 7 +- .../indexing/kafka/KafkaIndexTaskTest.java | 3 +- .../indexing/kafka/KafkaTuningConfigTest.java | 1 + .../kafka/supervisor/KafkaSupervisorTest.java | 1 + .../io/druid/data/input/MapPopulator.java | 2 +- .../namespace/UriExtractionNamespace.java | 6 +- .../namespace/UriExtractionNamespaceTest.java | 40 +- .../protobuf/ProtobufInputRowParser.java | 10 +- .../protobuf/ProtobufInputRowParserTest.java | 4 +- .../aggregation/variance/VarianceSerde.java | 6 +- .../indexer/HadoopDruidIndexerMapper.java | 40 +- .../indexer/HadoopyStringInputRowParser.java | 9 +- .../io/druid/indexer/IndexGeneratorJob.java | 4 +- .../indexer/updater/HadoopConverterJob.java | 3 +- .../updater/HadoopConverterJobTest.java | 10 +- .../common/index/YeOldePlumberSchool.java | 14 +- .../indexing/common/task/AppendTask.java | 7 +- ...ConvertSegmentBackwardsCompatibleTask.java | 12 +- .../common/task/ConvertSegmentTask.java | 142 ++++-- .../common/task/HadoopConverterTask.java | 16 +- .../druid/indexing/common/task/IndexTask.java | 124 ++--- .../druid/indexing/common/task/MergeTask.java | 7 +- .../indexing/common/task/MergeTaskBase.java | 18 +- .../common/task/SameIntervalMergeTask.java | 11 +- .../indexing/overlord/RemoteTaskRunner.java | 2 +- .../io/druid/indexing/common/TestUtils.java | 4 +- .../common/task/CompactionTaskTest.java | 10 +- .../common/task/ConvertSegmentTaskTest.java | 4 +- .../task/HadoopConverterTaskSerDeTest.java | 16 +- .../indexing/common/task/IndexTaskTest.java | 1 + .../common/task/MergeTaskBaseTest.java | 2 +- .../common/task/RealtimeIndexTaskTest.java | 3 +- .../task/SameIntervalMergeTaskTest.java | 1 + .../indexing/common/task/TaskSerdeTest.java | 37 +- .../IngestSegmentFirehoseFactoryTest.java | 4 +- ...estSegmentFirehoseFactoryTimelineTest.java | 2 +- .../indexing/overlord/TaskLifecycleTest.java | 7 +- .../druid/java/util/common/collect/Utils.java | 43 +- .../util/common/io/smoosh/FileSmoosher.java | 19 +- .../parsers/AbstractFlatTextFormatParser.java | 2 +- .../util/common/parsers/JSONPathParser.java | 2 +- .../common/parsers/JSONToLowerParser.java | 2 +- .../util/common/parsers/JavaScriptParser.java | 2 +- .../java/util/common/parsers/Parser.java | 2 +- .../java/util/common/parsers/Parsers.java | 2 +- .../java/util/common/parsers/RegexParser.java | 2 +- .../common/parsers/ToLowerCaseParser.java | 4 +- .../parsers/FlatTextFormatParserTest.java | 26 +- .../common/parsers/JSONPathParserTest.java | 16 +- .../common/parsers/JavaScriptParserTest.java | 4 +- .../util/common/parsers/RegexParserTest.java | 10 +- pom.xml | 4 +- processing/pom.xml | 5 + .../query/aggregation/AggregatorUtil.java | 8 + .../hyperloglog/HyperUniquesSerde.java | 6 +- .../io/druid/query/expression/ExprUtils.java | 6 +- .../expression/TimestampFloorExprMacro.java | 29 +- ...Generic1AggPooledTopNScannerPrototype.java | 12 + ...Generic2AggPooledTopNScannerPrototype.java | 12 + ...leDoubleAggPooledTopNScannerPrototype.java | 12 + ...leDoubleAggPooledTopNScannerPrototype.java | 13 + .../druid/query/topn/PooledTopNAlgorithm.java | 53 +- .../java/io/druid/query/topn/TopNUtils.java | 4 + .../CompressedVSizeIndexedSupplier.java | 49 +- .../CompressedVSizeIndexedV3Supplier.java | 52 +- .../io/druid/segment/DimensionHandler.java | 9 +- .../druid/segment/DoubleColumnSelector.java | 3 - .../druid/segment/DoubleColumnSerializer.java | 50 +- .../druid/segment/DoubleDimensionHandler.java | 10 +- .../segment/DoubleDimensionMergerV9.java | 47 +- .../java/io/druid/segment/FilteredOffset.java | 12 +- .../io/druid/segment/FloatColumnSelector.java | 3 - .../druid/segment/FloatColumnSerializer.java | 45 +- .../druid/segment/FloatDimensionHandler.java | 10 +- .../druid/segment/FloatDimensionMergerV9.java | 18 +- .../segment/GenericColumnSerializer.java | 13 +- .../main/java/io/druid/segment/IndexIO.java | 17 +- .../java/io/druid/segment/IndexMerger.java | 48 +- .../java/io/druid/segment/IndexMergerV9.java | 191 +++---- .../main/java/io/druid/segment/IndexSpec.java | 30 +- .../io/druid/segment/LongColumnSelector.java | 3 - .../druid/segment/LongColumnSerializer.java | 43 +- .../druid/segment/LongDimensionHandler.java | 10 +- .../druid/segment/LongDimensionMergerV9.java | 19 +- .../java/io/druid/segment/MetricHolder.java | 3 +- .../main/java/io/druid/segment/Rowboat.java | 2 +- .../druid/segment/StringDimensionHandler.java | 8 +- .../segment/StringDimensionMergerV9.java | 243 ++++----- .../segment/column/ColumnDescriptor.java | 19 +- .../BlockLayoutDoubleSupplierSerializer.java | 99 ++-- .../BlockLayoutFloatSupplierSerializer.java | 113 ++--- .../BlockLayoutIndexedDoubleSupplier.java | 11 +- .../data/BlockLayoutIndexedFloatSupplier.java | 12 +- .../data/BlockLayoutIndexedLongSupplier.java | 14 +- .../BlockLayoutLongSupplierSerializer.java | 119 ++--- .../segment/data/ByteBufferSerializer.java | 3 +- .../druid/segment/data/ByteBufferWriter.java | 98 +--- .../CompressedByteBufferObjectStrategy.java | 70 --- .../CompressedDoubleBufferObjectStrategy.java | 73 --- .../CompressedDoublesIndexedSupplier.java | 18 +- .../CompressedFloatBufferObjectStrategy.java | 72 --- .../data/CompressedFloatsIndexedSupplier.java | 52 +- .../CompressedIntBufferObjectStrategy.java | 71 --- .../data/CompressedIntsIndexedSupplier.java | 155 +++--- .../data/CompressedIntsIndexedWriter.java | 103 ++-- .../data/CompressedLongsIndexedSupplier.java | 66 +-- .../data/CompressedObjectStrategy.java | 402 --------------- .../data/CompressedVSizeIndexedV3Writer.java | 72 +-- .../CompressedVSizeIntsIndexedSupplier.java | 123 +++-- .../CompressedVSizeIntsIndexedWriter.java | 138 +++-- .../segment/data/CompressionFactory.java | 106 ++-- .../segment/data/CompressionStrategy.java | 338 +++++++++++++ .../data/ConciseBitmapSerdeFactory.java | 3 +- ...DecompressingByteBufferObjectStrategy.java | 84 ++++ .../segment/data/DeltaLongEncodingReader.java | 6 - .../segment/data/DeltaLongEncodingWriter.java | 22 +- .../data/DoubleSupplierSerializer.java | 9 +- .../EntireLayoutDoubleSupplierSerializer.java | 61 +-- .../EntireLayoutFloatSupplierSerializer.java | 88 +--- .../EntireLayoutLongSupplierSerializer.java | 73 +-- .../segment/data/FloatSupplierSerializer.java | 10 +- .../io/druid/segment/data/GenericIndexed.java | 102 ++-- .../segment/data/GenericIndexedWriter.java | 476 +++++++++--------- .../data/ImmutableRTreeObjectStrategy.java | 3 +- .../druid/segment/data/IndexedIntsWriter.java | 10 +- .../IntermediateLongSupplierSerializer.java | 80 ++- .../segment/data/LongSupplierSerializer.java | 10 +- .../segment/data/LongsLongEncodingReader.java | 8 - .../segment/data/LongsLongEncodingWriter.java | 13 +- .../data/MultiValueIndexedIntsWriter.java | 12 +- .../io/druid/segment/data/ObjectStrategy.java | 17 +- .../data/RoaringBitmapSerdeFactory.java | 3 +- .../segment/data/TableLongEncodingReader.java | 6 - .../segment/data/TableLongEncodingWriter.java | 24 +- .../io/druid/segment/data/TmpFileIOPeon.java | 95 ---- .../data/VSizeCompressedObjectStrategy.java | 88 ---- .../io/druid/segment/data/VSizeIndexed.java | 95 ++-- .../druid/segment/data/VSizeIndexedInts.java | 115 ++--- .../segment/data/VSizeIndexedIntsWriter.java | 61 ++- .../segment/data/VSizeIndexedWriter.java | 193 ++++--- .../io/druid/segment/data/VSizeLongSerde.java | 20 + .../druid/segment/data/WritableSupplier.java | 8 +- .../segment/incremental/IncrementalIndex.java | 13 +- .../incremental/OffheapIncrementalIndex.java | 3 +- .../incremental/OnheapIncrementalIndex.java | 7 +- .../druid/segment/serde/ColumnPartSerde.java | 10 - .../segment/serde/ComplexColumnPartSerde.java | 21 +- .../serde/ComplexColumnSerializer.java | 53 +- .../segment/serde/ComplexMetricSerde.java | 8 +- .../DictionaryEncodedColumnPartSerde.java | 41 +- .../serde/DoubleGenericColumnPartSerde.java | 93 ++-- .../serde/DoubleGenericColumnPartSerdeV2.java | 9 +- .../serde/FloatGenericColumnPartSerde.java | 26 +- .../serde/FloatGenericColumnPartSerdeV2.java | 9 +- ...olumnSupportedComplexColumnSerializer.java | 65 ++- .../serde/LongGenericColumnPartSerde.java | 26 +- .../serde/LongGenericColumnPartSerdeV2.java | 9 +- .../druid/segment/serde/MetaSerdeHelper.java | 158 ++++++ .../io/druid/segment/serde/Serializer.java | 43 ++ .../transform/TransformingInputRowParser.java | 7 +- .../TransformingStringInputRowParser.java | 6 +- .../writeout/ByteBufferWriteOutBytes.java | 265 ++++++++++ .../DirectByteBufferWriteOutBytes.java | 52 ++ .../segment/writeout/FileWriteOutBytes.java | 150 ++++++ .../writeout/HeapByteBufferWriteOutBytes.java | 31 ++ .../OffHeapMemorySegmentWriteOutMedium.java | 49 ++ ...eapMemorySegmentWriteOutMediumFactory.java | 45 ++ .../OnHeapMemorySegmentWriteOutMedium.java | 49 ++ .../writeout/SegmentWriteOutMedium.java | 47 ++ .../SegmentWriteOutMediumFactory.java | 51 ++ .../writeout/SegmentWriteOutMediumModule.java | 37 ++ .../TmpFileSegmentWriteOutMedium.java | 68 +++ .../TmpFileSegmentWriteOutMediumFactory.java} | 39 +- .../druid/segment/writeout/WriteOutBytes.java | 80 +++ .../io/druid/query/DoubleStorageTest.java | 10 +- .../druid/query/MultiValuedDimensionTest.java | 57 ++- .../io/druid/query/SchemaEvolutionTest.java | 12 +- .../aggregation/AggregationTestHelper.java | 24 +- .../aggregation/FilteredAggregatorTest.java | 6 +- ...ByLimitPushDownInsufficientBufferTest.java | 10 +- ...roupByLimitPushDownMultiNodeMergeTest.java | 10 +- .../groupby/GroupByMultiSegmentTest.java | 10 +- .../druid/query/topn/TopNQueryRunnerTest.java | 32 ++ .../java/io/druid/segment/AppendTest.java | 8 +- .../CompressedVSizeIndexedV3SupplierTest.java | 18 +- .../ConciseBitmapIndexMergerV9Test.java | 16 +- .../java/io/druid/segment/EmptyIndexTest.java | 30 +- .../java/io/druid/segment/IndexBuilder.java | 24 +- .../java/io/druid/segment/IndexIOTest.java | 9 +- .../segment/IndexMergerNullHandlingTest.java | 7 +- .../io/druid/segment/IndexMergerTestBase.java | 347 +++++++------ .../IndexMergerV9CompatibilityTest.java | 48 +- .../IndexMergerV9WithSpatialIndexTest.java | 67 +-- .../java/io/druid/segment/IndexSpecTest.java | 12 +- .../QueryableIndexIndexableAdapterTest.java | 45 +- .../RoaringBitmapIndexMergerV9Test.java | 16 +- .../io/druid/segment/SchemalessIndexTest.java | 77 +-- .../druid/segment/SchemalessTestFullTest.java | 36 +- .../segment/SchemalessTestSimpleTest.java | 34 +- .../segment/StringDimensionHandlerTest.java | 6 +- .../java/io/druid/segment/TestHelper.java | 29 +- .../test/java/io/druid/segment/TestIndex.java | 15 +- .../data/CompressedFloatsSerdeTest.java | 32 +- .../CompressedIntsIndexedSupplierTest.java | 15 +- .../data/CompressedIntsIndexedWriterTest.java | 115 ++--- .../data/CompressedLongsSerdeTest.java | 35 +- .../CompressedVSizeIndexedSupplierTest.java | 18 +- .../CompressedVSizeIndexedV3WriterTest.java | 109 ++-- ...ompressedVSizeIntsIndexedSupplierTest.java | 26 +- .../CompressedVSizeIntsIndexedWriterTest.java | 65 +-- .../segment/data/CompressionStrategyTest.java | 88 ++-- .../data/GenericIndexedStringWriterTest.java | 59 +++ .../segment/data/GenericIndexedTest.java | 2 +- .../druid/segment/data/IOPeonForTesting.java | 76 --- .../segment/data/VSizeIndexedIntsTest.java | 16 +- .../data/VSizeIndexedIntsWriterTest.java | 24 +- .../druid/segment/data/VSizeIndexedTest.java | 2 +- .../druid/segment/filter/AndFilterTest.java | 12 +- .../druid/segment/filter/BaseFilterTest.java | 30 +- .../druid/segment/filter/BoundFilterTest.java | 16 +- .../filter/ColumnComparisonFilterTest.java | 26 +- .../segment/filter/ExpressionFilterTest.java | 2 +- .../segment/filter/FilterPartitionTest.java | 20 +- .../filter/FloatAndDoubleFilteringTest.java | 12 +- .../io/druid/segment/filter/InFilterTest.java | 12 +- .../segment/filter/InvalidFilteringTest.java | 12 +- .../segment/filter/JavaScriptFilterTest.java | 12 +- .../druid/segment/filter/LikeFilterTest.java | 12 +- .../segment/filter/LongFilteringTest.java | 20 +- .../druid/segment/filter/NotFilterTest.java | 12 +- .../druid/segment/filter/RegexFilterTest.java | 12 +- .../segment/filter/SearchQueryFilterTest.java | 12 +- .../segment/filter/SelectorFilterTest.java | 12 +- .../filter/SpatialFilterBonusTest.java | 66 +-- .../segment/filter/SpatialFilterTest.java | 16 +- .../segment/filter/TimeFilteringTest.java | 12 +- .../IncrementalIndexAdapterTest.java | 6 +- .../OnheapIncrementalIndexBenchmark.java | 3 +- .../loading/SegmentizerFactoryTest.java | 21 +- .../serde/HyperUniquesSerdeForTest.java | 6 +- ...nSupportedComplexColumnSerializerTest.java | 11 +- .../segment/writeout/WriteOutBytesTest.java | 122 +++++ .../io/druid/client/DirectDruidClient.java | 2 +- .../IndexerMetadataStorageCoordinator.java | 22 +- .../druid/initialization/Initialization.java | 2 + .../metadata/SQLMetadataSegmentManager.java | 2 +- .../indexing/RealtimeTuningConfig.java | 27 +- .../segment/realtime/RealtimeManager.java | 3 +- .../realtime/appenderator/Appenderator.java | 67 ++- .../appenderator/AppenderatorConfig.java | 5 + .../appenderator/AppenderatorDriver.java | 76 ++- .../AppenderatorDriverAddResult.java | 22 +- .../appenderator/AppenderatorImpl.java | 24 +- .../appenderator/AppenderatorPlumber.java | 2 +- .../appenderator/SegmentAllocator.java | 8 +- .../EventReceiverFirehoseFactory.java | 2 +- .../realtime/firehose/IrcFirehoseFactory.java | 26 +- .../realtime/firehose/IrcInputRowParser.java | 9 +- .../realtime/plumber/FlushingPlumber.java | 2 +- .../realtime/plumber/RealtimePlumber.java | 9 +- .../druid/segment/realtime/plumber/Sink.java | 5 +- .../coordination/SegmentLoadDropHandler.java | 5 +- .../io/druid/server/security/Escalator.java | 8 +- .../segment/indexing/DataSchemaTest.java | 8 +- .../segment/indexing/TransformSpecTest.java | 12 +- .../SegmentLoaderLocalCacheManagerTest.java | 31 +- .../segment/realtime/FireDepartmentTest.java | 5 +- .../segment/realtime/RealtimeManagerTest.java | 5 +- .../AppenderatorDriverFailTest.java | 7 +- .../appenderator/AppenderatorPlumberTest.java | 1 + .../appenderator/AppenderatorTest.java | 61 ++- .../appenderator/AppenderatorTester.java | 5 +- ...DefaultOfflineAppenderatorFactoryTest.java | 1 + .../firehose/IngestSegmentFirehoseTest.java | 29 +- .../plumber/RealtimePlumberSchoolTest.java | 43 +- .../segment/realtime/plumber/SinkTest.java | 7 +- .../cli/validate/DruidJsonValidatorTest.java | 1 + .../sql/calcite/expression/Expressions.java | 211 ++++++-- .../builtin/CastOperatorConversion.java | 6 +- .../builtin/FloorOperatorConversion.java | 6 +- .../builtin/TimeFloorOperatorConversion.java | 60 ++- .../io/druid/sql/calcite/rel/DruidQuery.java | 30 +- .../druid/sql/calcite/CalciteQueryTest.java | 239 +++++---- .../calcite/expression/ExpressionsTest.java | 41 +- .../sql/calcite/schema/DruidSchemaTest.java | 6 +- .../druid/sql/calcite/util/CalciteTests.java | 50 +- 372 files changed, 9462 insertions(+), 5239 deletions(-) create mode 100644 common/src/main/java/io/druid/common/utils/ByteUtils.java create mode 100644 common/src/main/java/io/druid/io/ByteBufferInputStream.java rename processing/src/main/java/io/druid/segment/data/IOPeon.java => common/src/main/java/io/druid/io/Channels.java (70%) create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpBufferAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java create mode 100644 extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java create mode 100644 extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java create mode 100644 extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java create mode 100644 extensions-core/datasketches/src/test/resources/quantiles/doubles_build_data.tsv create mode 100644 extensions-core/datasketches/src/test/resources/quantiles/doubles_sketch_data.tsv delete mode 100644 processing/src/main/java/io/druid/segment/data/CompressedByteBufferObjectStrategy.java delete mode 100644 processing/src/main/java/io/druid/segment/data/CompressedDoubleBufferObjectStrategy.java delete mode 100644 processing/src/main/java/io/druid/segment/data/CompressedFloatBufferObjectStrategy.java delete mode 100644 processing/src/main/java/io/druid/segment/data/CompressedIntBufferObjectStrategy.java delete mode 100644 processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java create mode 100644 processing/src/main/java/io/druid/segment/data/CompressionStrategy.java create mode 100644 processing/src/main/java/io/druid/segment/data/DecompressingByteBufferObjectStrategy.java delete mode 100644 processing/src/main/java/io/druid/segment/data/TmpFileIOPeon.java delete mode 100644 processing/src/main/java/io/druid/segment/data/VSizeCompressedObjectStrategy.java create mode 100644 processing/src/main/java/io/druid/segment/serde/MetaSerdeHelper.java create mode 100644 processing/src/main/java/io/druid/segment/serde/Serializer.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/ByteBufferWriteOutBytes.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/DirectByteBufferWriteOutBytes.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/FileWriteOutBytes.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/HeapByteBufferWriteOutBytes.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/OffHeapMemorySegmentWriteOutMedium.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/OffHeapMemorySegmentWriteOutMediumFactory.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/OnHeapMemorySegmentWriteOutMedium.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMedium.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMediumFactory.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMediumModule.java create mode 100644 processing/src/main/java/io/druid/segment/writeout/TmpFileSegmentWriteOutMedium.java rename processing/src/main/java/io/druid/segment/{data/FixedSizeCompressedObjectStrategy.java => writeout/TmpFileSegmentWriteOutMediumFactory.java} (54%) create mode 100644 processing/src/main/java/io/druid/segment/writeout/WriteOutBytes.java create mode 100644 processing/src/test/java/io/druid/segment/data/GenericIndexedStringWriterTest.java delete mode 100644 processing/src/test/java/io/druid/segment/data/IOPeonForTesting.java create mode 100644 processing/src/test/java/io/druid/segment/writeout/WriteOutBytesTest.java diff --git a/api/src/main/java/io/druid/data/input/impl/InputRowParser.java b/api/src/main/java/io/druid/data/input/impl/InputRowParser.java index 9b6ca0bb979e..c3b8fba6e4fd 100644 --- a/api/src/main/java/io/druid/data/input/impl/InputRowParser.java +++ b/api/src/main/java/io/druid/data/input/impl/InputRowParser.java @@ -23,8 +23,11 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.druid.data.input.InputRow; import io.druid.guice.annotations.ExtensionPoint; +import io.druid.java.util.common.collect.Utils; import javax.annotation.Nullable; +import javax.validation.constraints.NotNull; +import java.util.List; @ExtensionPoint @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = StringInputRowParser.class) @@ -35,12 +38,27 @@ }) public interface InputRowParser { + /** + * Parse an input into list of {@link InputRow}. List can contains null for rows that should be thrown away, + * or throws {@code ParseException} if the input is unparseable. This method should never return null otherwise + * lots of things will break. + */ + @NotNull + default List parseBatch(T input) + { + return Utils.nullableListOf(parse(input)); + } + /** * Parse an input into an {@link InputRow}. Return null if this input should be thrown away, or throws * {@code ParseException} if the input is unparseable. */ + @Deprecated @Nullable - InputRow parse(T input); + default InputRow parse(T input) + { + return null; + } ParseSpec getParseSpec(); diff --git a/api/src/main/java/io/druid/data/input/impl/MapInputRowParser.java b/api/src/main/java/io/druid/data/input/impl/MapInputRowParser.java index acfa0e3e3d8d..3fa2305a7007 100644 --- a/api/src/main/java/io/druid/data/input/impl/MapInputRowParser.java +++ b/api/src/main/java/io/druid/data/input/impl/MapInputRowParser.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.druid.data.input.InputRow; @@ -45,7 +46,7 @@ public MapInputRowParser( } @Override - public InputRow parse(Map theMap) + public List parseBatch(Map theMap) { final List dimensions = parseSpec.getDimensionsSpec().hasCustomDimensions() ? parseSpec.getDimensionsSpec().getDimensionNames() @@ -74,7 +75,7 @@ public InputRow parse(Map theMap) throw new ParseException(e, "Unparseable timestamp found!"); } - return new MapBasedInputRow(timestamp.getMillis(), dimensions, theMap); + return ImmutableList.of(new MapBasedInputRow(timestamp.getMillis(), dimensions, theMap)); } @JsonProperty diff --git a/api/src/main/java/io/druid/data/input/impl/NoopInputRowParser.java b/api/src/main/java/io/druid/data/input/impl/NoopInputRowParser.java index 7b41391309a9..19459dfc4d45 100644 --- a/api/src/main/java/io/druid/data/input/impl/NoopInputRowParser.java +++ b/api/src/main/java/io/druid/data/input/impl/NoopInputRowParser.java @@ -21,8 +21,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; import io.druid.data.input.InputRow; +import java.util.List; + /** */ public class NoopInputRowParser implements InputRowParser @@ -38,9 +41,9 @@ public NoopInputRowParser( } @Override - public InputRow parse(InputRow input) + public List parseBatch(InputRow input) { - return input; + return ImmutableList.of(input); } @JsonProperty diff --git a/api/src/main/java/io/druid/data/input/impl/StringInputRowParser.java b/api/src/main/java/io/druid/data/input/impl/StringInputRowParser.java index 74467ccbf483..cefe4706e2f4 100644 --- a/api/src/main/java/io/druid/data/input/impl/StringInputRowParser.java +++ b/api/src/main/java/io/druid/data/input/impl/StringInputRowParser.java @@ -23,8 +23,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; import io.druid.data.input.ByteBufferInputRowParser; import io.druid.data.input.InputRow; +import io.druid.java.util.common.collect.Utils; import io.druid.java.util.common.parsers.ParseException; import io.druid.java.util.common.parsers.Parser; @@ -34,6 +36,7 @@ import java.nio.charset.Charset; import java.nio.charset.CoderResult; import java.nio.charset.CodingErrorAction; +import java.util.List; import java.util.Map; /** @@ -72,9 +75,9 @@ public StringInputRowParser(ParseSpec parseSpec) } @Override - public InputRow parse(ByteBuffer input) + public List parseBatch(ByteBuffer input) { - return parseMap(buildStringKeyMap(input)); + return Utils.nullableListOf(parseMap(buildStringKeyMap(input))); } @JsonProperty @@ -149,7 +152,7 @@ public InputRow parse(@Nullable String input) private Map parseString(@Nullable String inputString) { initializeParser(); - return parser.parse(inputString); + return parser.parseToMap(inputString); } @Nullable @@ -159,6 +162,6 @@ private InputRow parseMap(@Nullable Map theMap) if (theMap == null) { return null; } - return mapParser.parse(theMap); + return Iterators.getOnlyElement(mapParser.parseBatch(theMap).iterator()); } } diff --git a/api/src/main/java/io/druid/data/input/impl/TimeAndDimsParseSpec.java b/api/src/main/java/io/druid/data/input/impl/TimeAndDimsParseSpec.java index 9f81a18fc578..4b962b99227d 100644 --- a/api/src/main/java/io/druid/data/input/impl/TimeAndDimsParseSpec.java +++ b/api/src/main/java/io/druid/data/input/impl/TimeAndDimsParseSpec.java @@ -48,7 +48,7 @@ public Parser makeParser() return new Parser() { @Override - public Map parse(String input) + public Map parseToMap(String input) { throw new UnsupportedOperationException("not supported"); } diff --git a/api/src/main/java/io/druid/js/JavaScriptConfig.java b/api/src/main/java/io/druid/js/JavaScriptConfig.java index 7dc6bb1b2fb1..e97f19b1a8ae 100644 --- a/api/src/main/java/io/druid/js/JavaScriptConfig.java +++ b/api/src/main/java/io/druid/js/JavaScriptConfig.java @@ -35,16 +35,14 @@ public class JavaScriptConfig private static final JavaScriptConfig ENABLED_INSTANCE = new JavaScriptConfig(true); @JsonProperty - private boolean enabled = false; + private final boolean enabled; @JsonCreator public JavaScriptConfig( - @JsonProperty("enabled") Boolean enabled + @JsonProperty("enabled") boolean enabled ) { - if (enabled != null) { - this.enabled = enabled.booleanValue(); - } + this.enabled = enabled; } public boolean isEnabled() diff --git a/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java b/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java index ad24dab287fb..1a5d59f2f612 100644 --- a/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java +++ b/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java @@ -60,9 +60,9 @@ public void testStringInputRowParserSerde() throws Exception jsonMapper.writeValueAsBytes(parser), ByteBufferInputRowParser.class ); - final InputRow parsed = parser2.parse( + final InputRow parsed = parser2.parseBatch( ByteBuffer.wrap(StringUtils.toUtf8("{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"2000\"}")) - ); + ).get(0); Assert.assertEquals(ImmutableList.of("foo", "bar"), parsed.getDimensions()); Assert.assertEquals(ImmutableList.of("x"), parsed.getDimension("foo")); Assert.assertEquals(ImmutableList.of("y"), parsed.getDimension("bar")); @@ -101,14 +101,14 @@ public void testMapInputRowParserSerde() throws Exception jsonMapper.writeValueAsBytes(parser), MapInputRowParser.class ); - final InputRow parsed = parser2.parse( + final InputRow parsed = parser2.parseBatch( ImmutableMap.of( "foo", "x", "bar", "y", "qux", "z", "timeposix", "1" ) - ); + ).get(0); Assert.assertEquals(ImmutableList.of("foo", "bar"), parsed.getDimensions()); Assert.assertEquals(ImmutableList.of("x"), parsed.getDimension("foo")); Assert.assertEquals(ImmutableList.of("y"), parsed.getDimension("bar")); @@ -130,7 +130,7 @@ public void testMapInputRowParserNumbersSerde() throws Exception jsonMapper.writeValueAsBytes(parser), MapInputRowParser.class ); - final InputRow parsed = parser2.parse( + final InputRow parsed = parser2.parseBatch( ImmutableMap.of( "timemillis", 1412705931123L, "toobig", 123E64, @@ -138,7 +138,7 @@ public void testMapInputRowParserNumbersSerde() throws Exception "long", 123456789000L, "values", Lists.newArrayList(1412705931123L, 123.456, 123E45, "hello") ) - ); + ).get(0); Assert.assertEquals(ImmutableList.of("foo", "values"), parsed.getDimensions()); Assert.assertEquals(ImmutableList.of(), parsed.getDimension("foo")); Assert.assertEquals( @@ -170,11 +170,11 @@ private InputRow testCharsetParseHelper(Charset charset) throws Exception ByteBufferInputRowParser.class ); - final InputRow parsed = parser2.parse( + final InputRow parsed = parser2.parseBatch( ByteBuffer.wrap( "{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"3000\"}".getBytes(charset) ) - ); + ).get(0); return parsed; } diff --git a/api/src/test/java/io/druid/data/input/impl/JSONLowercaseParseSpecTest.java b/api/src/test/java/io/druid/data/input/impl/JSONLowercaseParseSpecTest.java index d8fc9fde77e3..9d6b6180b404 100644 --- a/api/src/test/java/io/druid/data/input/impl/JSONLowercaseParseSpecTest.java +++ b/api/src/test/java/io/druid/data/input/impl/JSONLowercaseParseSpecTest.java @@ -45,7 +45,7 @@ public void testLowercasing() throws Exception ) ); Parser parser = spec.makeParser(); - Map event = parser.parse("{\"timestamp\":\"2015-01-01\",\"A\":\"foo\"}"); + Map event = parser.parseToMap("{\"timestamp\":\"2015-01-01\",\"A\":\"foo\"}"); Assert.assertEquals("foo", event.get("a")); } } diff --git a/api/src/test/java/io/druid/data/input/impl/JSONParseSpecTest.java b/api/src/test/java/io/druid/data/input/impl/JSONParseSpecTest.java index 26812b459bae..de2814eda9a7 100644 --- a/api/src/test/java/io/druid/data/input/impl/JSONParseSpecTest.java +++ b/api/src/test/java/io/druid/data/input/impl/JSONParseSpecTest.java @@ -69,7 +69,7 @@ public void testParseRow() expected.put("jq_omg2", null); final Parser parser = parseSpec.makeParser(); - final Map parsedRow = parser.parse("{\"bar\":null,\"foo\":\"x\",\"baz\":4,\"o\":{\"mg\":1}}"); + final Map parsedRow = parser.parseToMap("{\"bar\":null,\"foo\":\"x\",\"baz\":4,\"o\":{\"mg\":1}}"); Assert.assertNotNull(parsedRow); Assert.assertEquals(expected, parsedRow); Assert.assertNull(parsedRow.get("bar")); diff --git a/api/src/test/java/io/druid/data/input/impl/JavaScriptParseSpecTest.java b/api/src/test/java/io/druid/data/input/impl/JavaScriptParseSpecTest.java index e9dcc196f3a0..805e019b1b08 100644 --- a/api/src/test/java/io/druid/data/input/impl/JavaScriptParseSpecTest.java +++ b/api/src/test/java/io/druid/data/input/impl/JavaScriptParseSpecTest.java @@ -81,7 +81,7 @@ public void testMakeParser() ); final Parser parser = spec.makeParser(); - final Map obj = parser.parse("x-y"); + final Map obj = parser.parseToMap("x-y"); Assert.assertEquals(ImmutableMap.of("one", "x", "two", "y"), obj); } diff --git a/benchmarks/src/main/java/io/druid/benchmark/CompressedIndexedIntsBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/CompressedIndexedIntsBenchmark.java index 307ef5d67d4a..23c4adb57ddb 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/CompressedIndexedIntsBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/CompressedIndexedIntsBenchmark.java @@ -19,12 +19,13 @@ package io.druid.benchmark; -import com.google.common.primitives.Ints; -import io.druid.segment.data.CompressedObjectStrategy; +import io.druid.java.util.common.io.Closer; import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.VSizeIndexedInts; import io.druid.segment.data.WritableSupplier; +import it.unimi.dsi.fastutil.ints.IntArrayList; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; @@ -69,25 +70,20 @@ public void setup() throws IOException } final ByteBuffer bufferCompressed = serialize( CompressedVSizeIntsIndexedSupplier.fromList( - Ints.asList(vals), + IntArrayList.wrap(vals), bound - 1, CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForBytes(bytes), - ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4 + ByteOrder.nativeOrder(), + CompressionStrategy.LZ4, + Closer.create() ) ); this.compressed = CompressedVSizeIntsIndexedSupplier.fromByteBuffer( bufferCompressed, - ByteOrder.nativeOrder(), - null + ByteOrder.nativeOrder() ).get(); - final ByteBuffer bufferUncompressed = serialize( - new VSizeIndexedInts.VSizeIndexedIntsSupplier( - VSizeIndexedInts.fromArray( - vals - ) - ) - ); + final ByteBuffer bufferUncompressed = serialize(VSizeIndexedInts.fromArray(vals)); this.uncompressed = VSizeIndexedInts.readFromByteBuffer(bufferUncompressed); filter = new BitSet(); @@ -128,7 +124,7 @@ public void close() throws IOException } }; - writableSupplier.writeToChannel(channel); + writableSupplier.writeTo(channel, null); buffer.rewind(); return buffer; } diff --git a/benchmarks/src/main/java/io/druid/benchmark/CompressedVSizeIndexedBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/CompressedVSizeIndexedBenchmark.java index c0b1c7f10973..f829bccf9399 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/CompressedVSizeIndexedBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/CompressedVSizeIndexedBenchmark.java @@ -22,8 +22,9 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.druid.java.util.common.io.Closer; import io.druid.segment.CompressedVSizeIndexedSupplier; -import io.druid.segment.data.CompressedObjectStrategy; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.IndexedMultivalue; import io.druid.segment.data.VSizeIndexed; @@ -95,13 +96,14 @@ public IndexedInts apply(int[] input) } ), bound - 1, - ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4 + ByteOrder.nativeOrder(), + CompressionStrategy.LZ4, + Closer.create() ) ); this.compressed = CompressedVSizeIndexedSupplier.fromByteBuffer( bufferCompressed, - ByteOrder.nativeOrder(), - null + ByteOrder.nativeOrder() ).get(); final ByteBuffer bufferUncompressed = serialize( @@ -117,7 +119,7 @@ public VSizeIndexedInts apply(int[] input) } } ) - ).asWritableSupplier() + ) ); this.uncompressed = VSizeIndexed.readFromByteBuffer(bufferUncompressed); @@ -159,7 +161,7 @@ public void close() throws IOException } }; - writableSupplier.writeToChannel(channel); + writableSupplier.writeTo(channel, null); buffer.rewind(); return buffer; } diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java index ca011055bb94..b409b9cb6d5b 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java @@ -36,6 +36,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.js.JavaScriptConfig; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesSerde; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.extraction.ExtractionFn; @@ -132,6 +133,7 @@ public class FilterPartitionBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -141,7 +143,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } @Setup @@ -178,7 +180,8 @@ public void setup() throws IOException indexFile = INDEX_MERGER_V9.persist( incIndex, tmpDir, - new IndexSpec() + new IndexSpec(), + null ); qIndex = INDEX_IO.loadIndex(indexFile); diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilteredAggregatorBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilteredAggregatorBenchmark.java index 5763fa9b12bd..4b9422ae6cca 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FilteredAggregatorBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FilteredAggregatorBenchmark.java @@ -35,6 +35,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.js.JavaScriptConfig; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; @@ -128,6 +129,7 @@ public class FilteredAggregatorBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -137,7 +139,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } @Setup @@ -194,7 +196,8 @@ public void setup() throws IOException indexFile = INDEX_MERGER_V9.persist( incIndex, tmpDir, - new IndexSpec() + new IndexSpec(), + null ); qIndex = INDEX_IO.loadIndex(indexFile); diff --git a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java index 03a54b795ad4..e241295582bd 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java @@ -91,7 +91,7 @@ public void prepare() throws Exception @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map baseline(final Blackhole blackhole) { - Map parsed = flatParser.parse(flatInputs.get(flatCounter)); + Map parsed = flatParser.parseToMap(flatInputs.get(flatCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } @@ -104,7 +104,7 @@ public Map baseline(final Blackhole blackhole) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map flatten(final Blackhole blackhole) { - Map parsed = nestedParser.parse(nestedInputs.get(nestedCounter)); + Map parsed = nestedParser.parseToMap(nestedInputs.get(nestedCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } @@ -117,7 +117,7 @@ public Map flatten(final Blackhole blackhole) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map jqflatten(final Blackhole blackhole) { - Map parsed = jqParser.parse(jqInputs.get(jqCounter)); + Map parsed = jqParser.parseToMap(jqInputs.get(jqCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } @@ -130,7 +130,7 @@ public Map jqflatten(final Blackhole blackhole) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map preflattenNestedParser(final Blackhole blackhole) { - Map parsed = fieldDiscoveryParser.parse(flatInputs.get(nestedCounter)); + Map parsed = fieldDiscoveryParser.parseToMap(flatInputs.get(nestedCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } @@ -143,7 +143,7 @@ public Map preflattenNestedParser(final Blackhole blackhole) @OutputTimeUnit(TimeUnit.MICROSECONDS) public Map forcedRootPaths(final Blackhole blackhole) { - Map parsed = forcedPathParser.parse(flatInputs.get(nestedCounter)); + Map parsed = forcedPathParser.parseToMap(flatInputs.get(nestedCounter)); for (String s : parsed.keySet()) { blackhole.consume(parsed.get(s)); } diff --git a/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmark.java index b06ecb0657a2..ee46f8384f79 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmark.java @@ -72,7 +72,7 @@ public void setup() throws Exception File compFile = new File(dir, file + "-" + strategy); rand = new Random(); ByteBuffer buffer = Files.map(compFile); - supplier = CompressedFloatsIndexedSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder(), null); + supplier = CompressedFloatsIndexedSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder()); } @Benchmark diff --git a/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmarkFileGenerator.java b/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmarkFileGenerator.java index f66a14f420ba..0f1eaf76d349 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmarkFileGenerator.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmarkFileGenerator.java @@ -20,24 +20,20 @@ package io.druid.benchmark; import com.google.common.collect.ImmutableList; -import com.google.common.io.ByteSink; import io.druid.benchmark.datagen.BenchmarkColumnSchema; import io.druid.benchmark.datagen.BenchmarkColumnValueGenerator; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; import io.druid.segment.column.ValueType; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.FloatSupplierSerializer; -import io.druid.segment.data.TmpFileIOPeon; import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.io.OutputStream; import java.io.Writer; import java.net.URISyntaxException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; @@ -51,10 +47,10 @@ public class FloatCompressionBenchmarkFileGenerator { private static final Logger log = new Logger(FloatCompressionBenchmarkFileGenerator.class); public static final int ROW_NUM = 5000000; - public static final List compressions = + public static final List compressions = ImmutableList.of( - CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.NONE + CompressionStrategy.LZ4, + CompressionStrategy.NONE ); private static String dirPath = "floatCompress/"; @@ -143,48 +139,30 @@ public static void main(String[] args) throws IOException, URISyntaxException // create compressed files using all combinations of CompressionStrategy and FloatEncoding provided for (Map.Entry entry : generators.entrySet()) { - for (CompressedObjectStrategy.CompressionStrategy compression : compressions) { + for (CompressionStrategy compression : compressions) { String name = entry.getKey() + "-" + compression.toString(); log.info("%s: ", name); File compFile = new File(dir, name); compFile.delete(); File dataFile = new File(dir, entry.getKey()); - TmpFileIOPeon iopeon = new TmpFileIOPeon(true); FloatSupplierSerializer writer = CompressionFactory.getFloatSerializer( - iopeon, + new OffHeapMemorySegmentWriteOutMedium(), "float", ByteOrder.nativeOrder(), compression ); - BufferedReader br = Files.newBufferedReader(dataFile.toPath(), StandardCharsets.UTF_8); - - try (FileChannel output = FileChannel.open( - compFile.toPath(), - StandardOpenOption.CREATE_NEW, - StandardOpenOption.WRITE - )) { + try ( + BufferedReader br = Files.newBufferedReader(dataFile.toPath(), StandardCharsets.UTF_8); + FileChannel output = + FileChannel.open(compFile.toPath(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE) + ) { writer.open(); String line; while ((line = br.readLine()) != null) { writer.add(Float.parseFloat(line)); } - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - writer.closeAndConsolidate( - new ByteSink() - { - @Override - public OutputStream openStream() throws IOException - { - return baos; - } - } - ); - output.write(ByteBuffer.wrap(baos.toByteArray())); - } - finally { - iopeon.close(); - br.close(); + writer.writeTo(output, null); } log.info("%d", compFile.length() / 1024); } diff --git a/benchmarks/src/main/java/io/druid/benchmark/GenericIndexedBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/GenericIndexedBenchmark.java index 4b5fb28574d5..2d9a60c559e2 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/GenericIndexedBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/GenericIndexedBenchmark.java @@ -23,10 +23,10 @@ import com.google.common.primitives.Ints; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.GenericIndexedWriter; import io.druid.segment.data.ObjectStrategy; -import io.druid.segment.data.TmpFileIOPeon; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -106,7 +106,7 @@ public int compare(byte[] o1, byte[] o2) public void createGenericIndexed() throws IOException { GenericIndexedWriter genericIndexedWriter = new GenericIndexedWriter<>( - new TmpFileIOPeon(), + new OffHeapMemorySegmentWriteOutMedium(), "genericIndexedBenchmark", byteArrayStrategy ); @@ -121,14 +121,13 @@ public void createGenericIndexed() throws IOException element.putInt(0, i); genericIndexedWriter.write(element.array()); } - genericIndexedWriter.close(); smooshDir = Files.createTempDir(); file = File.createTempFile("genericIndexedBenchmark", "meta"); try (FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); FileSmoosher fileSmoosher = new FileSmoosher(smooshDir)) { - genericIndexedWriter.writeToChannel(fileChannel, fileSmoosher); + genericIndexedWriter.writeTo(fileChannel, fileSmoosher); } FileChannel fileChannel = FileChannel.open(file.toPath()); diff --git a/benchmarks/src/main/java/io/druid/benchmark/GroupByTypeInterfaceBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/GroupByTypeInterfaceBenchmark.java index 4cdacaf3bc57..e8160672d7cd 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/GroupByTypeInterfaceBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/GroupByTypeInterfaceBenchmark.java @@ -46,6 +46,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.offheap.OffheapBufferGenerator; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.DruidProcessingConfig; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; @@ -153,6 +154,7 @@ public class GroupByTypeInterfaceBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -162,7 +164,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } private static final Map> SCHEMA_QUERY_MAP = new LinkedHashMap<>(); @@ -339,7 +341,8 @@ public void setup() throws IOException final File file = INDEX_MERGER_V9.persist( index, new File(tmpDir, String.valueOf(i)), - new IndexSpec() + new IndexSpec(), + null ); queryableIndexes.add(INDEX_IO.loadIndex(file)); diff --git a/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmark.java index ac41c6874571..cfb40ceabd5f 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmark.java @@ -75,7 +75,7 @@ public void setup() throws Exception File compFile = new File(dir, file + "-" + strategy + "-" + format); rand = new Random(); ByteBuffer buffer = Files.map(compFile); - supplier = CompressedLongsIndexedSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder(), null); + supplier = CompressedLongsIndexedSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder()); } @Benchmark diff --git a/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmarkFileGenerator.java b/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmarkFileGenerator.java index bd00dfa62c00..2b654e72599c 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmarkFileGenerator.java +++ b/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmarkFileGenerator.java @@ -20,24 +20,20 @@ package io.druid.benchmark; import com.google.common.collect.ImmutableList; -import com.google.common.io.ByteSink; import io.druid.benchmark.datagen.BenchmarkColumnSchema; import io.druid.benchmark.datagen.BenchmarkColumnValueGenerator; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; import io.druid.segment.column.ValueType; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.LongSupplierSerializer; -import io.druid.segment.data.TmpFileIOPeon; import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.io.OutputStream; import java.io.Writer; import java.net.URISyntaxException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; @@ -51,9 +47,10 @@ public class LongCompressionBenchmarkFileGenerator { private static final Logger log = new Logger(LongCompressionBenchmarkFileGenerator.class); public static final int ROW_NUM = 5000000; - public static final List compressions = - ImmutableList.of(CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.NONE); + public static final List compressions = + ImmutableList.of( + CompressionStrategy.LZ4, + CompressionStrategy.NONE); public static final List encodings = ImmutableList.of(CompressionFactory.LongEncodingStrategy.AUTO, CompressionFactory.LongEncodingStrategy.LONGS); @@ -134,7 +131,7 @@ public static void main(String[] args) throws IOException, URISyntaxException // create compressed files using all combinations of CompressionStrategy and LongEncoding provided for (Map.Entry entry : generators.entrySet()) { - for (CompressedObjectStrategy.CompressionStrategy compression : compressions) { + for (CompressionStrategy compression : compressions) { for (CompressionFactory.LongEncodingStrategy encoding : encodings) { String name = entry.getKey() + "-" + compression.toString() + "-" + encoding.toString(); log.info("%s: ", name); @@ -142,42 +139,24 @@ public static void main(String[] args) throws IOException, URISyntaxException compFile.delete(); File dataFile = new File(dir, entry.getKey()); - TmpFileIOPeon iopeon = new TmpFileIOPeon(true); LongSupplierSerializer writer = CompressionFactory.getLongSerializer( - iopeon, + new OffHeapMemorySegmentWriteOutMedium(), "long", ByteOrder.nativeOrder(), encoding, compression ); - BufferedReader br = Files.newBufferedReader(dataFile.toPath(), StandardCharsets.UTF_8); - - try (FileChannel output = FileChannel.open( - compFile.toPath(), - StandardOpenOption.CREATE_NEW, - StandardOpenOption.WRITE - )) { + try ( + BufferedReader br = Files.newBufferedReader(dataFile.toPath(), StandardCharsets.UTF_8); + FileChannel output = + FileChannel.open(compFile.toPath(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE) + ) { writer.open(); String line; while ((line = br.readLine()) != null) { writer.add(Long.parseLong(line)); } - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - writer.closeAndConsolidate( - new ByteSink() - { - @Override - public OutputStream openStream() throws IOException - { - return baos; - } - } - ); - output.write(ByteBuffer.wrap(baos.toByteArray())); - } - finally { - iopeon.close(); - br.close(); + writer.writeTo(output, null); } log.info("%d", compFile.length() / 1024); } diff --git a/benchmarks/src/main/java/io/druid/benchmark/TopNTypeInterfaceBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/TopNTypeInterfaceBenchmark.java index 52b6e468d55b..71645ddcb2eb 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/TopNTypeInterfaceBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/TopNTypeInterfaceBenchmark.java @@ -37,6 +37,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.offheap.OffheapBufferGenerator; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; import io.druid.query.QueryPlus; @@ -136,6 +137,7 @@ public class TopNTypeInterfaceBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -145,7 +147,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } private static final Map> SCHEMA_QUERY_MAP = new LinkedHashMap<>(); @@ -291,7 +293,8 @@ public void setup() throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndexes.get(i), tmpFile, - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile); diff --git a/benchmarks/src/main/java/io/druid/benchmark/datagen/SegmentGenerator.java b/benchmarks/src/main/java/io/druid/benchmark/datagen/SegmentGenerator.java index 62e87cb851fe..e6bca29f0f68 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/datagen/SegmentGenerator.java +++ b/benchmarks/src/main/java/io/druid/benchmark/datagen/SegmentGenerator.java @@ -33,6 +33,7 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.granularity.Granularity; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesSerde; import io.druid.segment.IndexBuilder; @@ -144,8 +145,8 @@ public QueryableIndex generate( return Iterables.getOnlyElement(indexes); } else { try { - final QueryableIndex merged = TestHelper.getTestIndexIO().loadIndex( - TestHelper.getTestIndexMergerV9().merge( + final QueryableIndex merged = TestHelper.getTestIndexIO(OffHeapMemorySegmentWriteOutMediumFactory.instance()).loadIndex( + TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()).merge( indexes.stream().map(QueryableIndexIndexableAdapter::new).collect(Collectors.toList()), false, schemaInfo.getAggs() @@ -186,7 +187,7 @@ private QueryableIndex makeIndex( .create() .schema(indexSchema) .tmpDir(new File(new File(tempDir, identifier), String.valueOf(indexNumber))) - .indexMerger(TestHelper.getTestIndexMergerV9()) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .rows(rows) .buildMMappedIndex(); } diff --git a/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexMergeBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexMergeBenchmark.java index 842c7245e3df..17ebafff4095 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexMergeBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexMergeBenchmark.java @@ -19,6 +19,7 @@ package io.druid.benchmark.indexing; +import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.io.Files; import io.druid.benchmark.datagen.BenchmarkDataGenerator; @@ -28,6 +29,7 @@ import io.druid.hll.HyperLogLogHash; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.logger.Logger; +import io.druid.math.expr.ExprMacroTable; import io.druid.query.aggregation.hyperloglog.HyperUniquesSerde; import io.druid.segment.IndexIO; import io.druid.segment.IndexMergerV9; @@ -37,6 +39,7 @@ import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.serde.ComplexMetrics; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.commons.io.FileUtils; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -88,8 +91,12 @@ public class IndexMergeBenchmark static { JSON_MAPPER = new DefaultObjectMapper(); + InjectableValues.Std injectableValues = new InjectableValues.Std(); + injectableValues.addValue(ExprMacroTable.class, ExprMacroTable.nil()); + JSON_MAPPER.setInjectableValues(injectableValues); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -99,7 +106,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } @Setup @@ -139,7 +146,8 @@ public void setup() throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndex, tmpDir, - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile); @@ -183,7 +191,8 @@ public void mergeV9(Blackhole blackhole) throws Exception rollup, schemaInfo.getAggsArray(), tmpFile, - new IndexSpec() + new IndexSpec(), + null ); blackhole.consume(mergedFile); diff --git a/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexPersistBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexPersistBenchmark.java index bde2eda5ef5c..1208b7864cc2 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexPersistBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexPersistBenchmark.java @@ -28,6 +28,7 @@ import io.druid.hll.HyperLogLogHash; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesSerde; import io.druid.segment.IndexIO; import io.druid.segment.IndexMergerV9; @@ -88,6 +89,7 @@ public class IndexPersistBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -97,7 +99,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } @Setup @@ -172,7 +174,8 @@ public void persistV9(Blackhole blackhole) throws Exception File indexFile = INDEX_MERGER_V9.persist( incIndex, tmpDir, - new IndexSpec() + new IndexSpec(), + null ); blackhole.consume(indexFile); diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/GroupByBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/GroupByBenchmark.java index 94c576eae9c1..ad8041d3dfa2 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/GroupByBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/GroupByBenchmark.java @@ -47,6 +47,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.offheap.OffheapBufferGenerator; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.DruidProcessingConfig; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; @@ -158,6 +159,7 @@ public class GroupByBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -167,7 +169,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } private static final Map> SCHEMA_QUERY_MAP = new LinkedHashMap<>(); @@ -434,7 +436,8 @@ public void setup() throws IOException final File file = INDEX_MERGER_V9.persist( index, new File(tmpDir, String.valueOf(i)), - new IndexSpec() + new IndexSpec(), + null ); queryableIndexes.add(INDEX_IO.loadIndex(file)); diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java index be7476eba84b..8b071f94bd25 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java @@ -38,6 +38,7 @@ import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.Druids.SearchQueryBuilder; import io.druid.query.FinalizeResultsQueryRunner; @@ -140,6 +141,7 @@ public class SearchBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -149,7 +151,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } private static final Map> SCHEMA_QUERY_MAP = new LinkedHashMap<>(); @@ -360,7 +362,8 @@ public void setup() throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndexes.get(i), tmpDir, - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile); diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/SelectBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/SelectBenchmark.java index 271d6c45a959..410c3f5d83a9 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/SelectBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/SelectBenchmark.java @@ -37,6 +37,7 @@ import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; @@ -134,6 +135,7 @@ public class SelectBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -143,7 +145,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } private static final Map> SCHEMA_QUERY_MAP = new LinkedHashMap<>(); @@ -223,7 +225,8 @@ public void setup() throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndexes.get(i), tmpDir, - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile); qIndexes.add(qIndex); diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/TimeseriesBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/TimeseriesBenchmark.java index 57242d28771c..a7aa4eb2ea65 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/TimeseriesBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/TimeseriesBenchmark.java @@ -35,6 +35,7 @@ import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; @@ -132,6 +133,7 @@ public class TimeseriesBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -141,7 +143,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } private static final Map> SCHEMA_QUERY_MAP = new LinkedHashMap<>(); @@ -286,7 +288,8 @@ public void setup() throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndexes.get(i), tmpDir, - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile); diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/TopNBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/TopNBenchmark.java index 7c57715f9c4a..f213c91808ac 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/query/TopNBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/query/TopNBenchmark.java @@ -36,6 +36,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.offheap.OffheapBufferGenerator; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; import io.druid.query.QueryPlus; @@ -133,6 +134,7 @@ public class TopNBenchmark JSON_MAPPER = new DefaultObjectMapper(); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -142,7 +144,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } private static final Map> SCHEMA_QUERY_MAP = new LinkedHashMap<>(); @@ -263,7 +265,8 @@ public void setup() throws IOException File indexFile = INDEX_MERGER_V9.persist( incIndexes.get(i), tmpDir, - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile); diff --git a/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java b/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java index 21c4abaad047..3ff146e6a07d 100644 --- a/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java +++ b/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java @@ -40,9 +40,9 @@ public void testOne() throws Exception Parser nestedParser = eventGen.getNestedParser(); Parser jqParser = eventGen.getJqParser(); - Map event = flatParser.parse(newEvent); - Map event2 = nestedParser.parse(newEvent2); - Map event3 = jqParser.parse(newEvent2); // reuse the same event as "nested" + Map event = flatParser.parseToMap(newEvent); + Map event2 = nestedParser.parseToMap(newEvent2); + Map event3 = jqParser.parseToMap(newEvent2); // reuse the same event as "nested" checkEvent1(event); checkEvent2(event2); diff --git a/common/src/main/java/io/druid/common/utils/ByteUtils.java b/common/src/main/java/io/druid/common/utils/ByteUtils.java new file mode 100644 index 000000000000..f377a628e2bf --- /dev/null +++ b/common/src/main/java/io/druid/common/utils/ByteUtils.java @@ -0,0 +1,35 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.common.utils; + +public final class ByteUtils +{ + + public static byte checkedCast(int value) + { + byte result = (byte) value; + if (result != value) { + throw new IllegalArgumentException("Out of range: " + value); + } + return result; + } + + private ByteUtils() {} +} diff --git a/common/src/main/java/io/druid/common/utils/SerializerUtils.java b/common/src/main/java/io/druid/common/utils/SerializerUtils.java index 9191391f44c4..0ead88af0769 100644 --- a/common/src/main/java/io/druid/common/utils/SerializerUtils.java +++ b/common/src/main/java/io/druid/common/utils/SerializerUtils.java @@ -24,6 +24,7 @@ import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; +import io.druid.io.Channels; import io.druid.java.util.common.StringUtils; import java.io.IOException; @@ -38,38 +39,6 @@ public class SerializerUtils { - /** - * Writes the given long value into the given OutputStream in big-endian byte order, using the helperBuffer. Faster - * alternative to out.write(Longs.toByteArray(value)), more convenient (sometimes) than wrapping the OutputStream into - * {@link java.io.DataOutputStream}. - * - * @param helperBuffer a big-endian heap ByteBuffer with capacity of at least 8 - */ - public static void writeBigEndianLongToOutputStream(OutputStream out, long value, ByteBuffer helperBuffer) - throws IOException - { - if (helperBuffer.order() != ByteOrder.BIG_ENDIAN || !helperBuffer.hasArray()) { - throw new IllegalArgumentException("Expected writable, big-endian, heap byteBuffer"); - } - helperBuffer.putLong(0, value); - out.write(helperBuffer.array(), helperBuffer.arrayOffset(), Longs.BYTES); - } - - /** - * Writes the given long value into the given OutputStream in the native byte order, using the helperBuffer. - * - * @param helperBuffer a heap ByteBuffer with capacity of at least 8, with the native byte order - */ - public static void writeNativeOrderedLongToOutputStream(OutputStream out, long value, ByteBuffer helperBuffer) - throws IOException - { - if (helperBuffer.order() != ByteOrder.nativeOrder() || !helperBuffer.hasArray()) { - throw new IllegalArgumentException("Expected writable heap byteBuffer with the native byte order"); - } - helperBuffer.putLong(0, value); - out.write(helperBuffer.array(), helperBuffer.arrayOffset(), Longs.BYTES); - } - /** * Writes the given int value into the given OutputStream in big-endian byte order, using the helperBuffer. Faster * alternative to out.write(Ints.toByteArray(value)), more convenient (sometimes) than wrapping the OutputStream into @@ -87,21 +56,6 @@ public static void writeBigEndianIntToOutputStream(OutputStream out, int value, out.write(helperBuffer.array(), helperBuffer.arrayOffset(), Ints.BYTES); } - /** - * Writes the given int value into the given OutputStream in the native byte order, using the given helperBuffer. - * - * @param helperBuffer a heap ByteBuffer with capacity of at least 4, with the native byte order - */ - public static void writeNativeOrderedIntToOutputStream(OutputStream out, int value, ByteBuffer helperBuffer) - throws IOException - { - if (helperBuffer.order() != ByteOrder.nativeOrder() || !helperBuffer.hasArray()) { - throw new IllegalArgumentException("Expected writable heap byteBuffer with the native byte order"); - } - helperBuffer.putInt(0, value); - out.write(helperBuffer.array(), helperBuffer.arrayOffset(), Ints.BYTES); - } - public void writeString(T out, String name) throws IOException { byte[] nameBytes = StringUtils.toUtf8(name); @@ -120,10 +74,10 @@ public void writeString(WritableByteChannel out, String name) throws IOException { byte[] nameBytes = StringUtils.toUtf8(name); writeInt(out, nameBytes.length); - out.write(ByteBuffer.wrap(nameBytes)); + Channels.writeFully(out, ByteBuffer.wrap(nameBytes)); } - public String readString(InputStream in) throws IOException + String readString(InputStream in) throws IOException { final int length = readInt(in); byte[] stringBytes = new byte[length]; @@ -144,12 +98,12 @@ public byte[] readBytes(ByteBuffer in, int length) throws IOException return bytes; } - public void writeStrings(OutputStream out, String[] names) throws IOException + void writeStrings(OutputStream out, String[] names) throws IOException { writeStrings(out, Arrays.asList(names)); } - public void writeStrings(OutputStream out, List names) throws IOException + private void writeStrings(OutputStream out, List names) throws IOException { writeInt(out, names.size()); @@ -158,7 +112,7 @@ public void writeStrings(OutputStream out, List names) throws IOExceptio } } - public String[] readStrings(InputStream in) throws IOException + String[] readStrings(InputStream in) throws IOException { int length = readInt(in); @@ -171,7 +125,7 @@ public String[] readStrings(InputStream in) throws IOException return retVal; } - public String[] readStrings(ByteBuffer in) throws IOException + String[] readStrings(ByteBuffer in) throws IOException { int length = in.getInt(); @@ -184,20 +138,20 @@ public String[] readStrings(ByteBuffer in) throws IOException return retVal; } - public void writeInt(OutputStream out, int intValue) throws IOException + private void writeInt(OutputStream out, int intValue) throws IOException { out.write(Ints.toByteArray(intValue)); } - public void writeInt(WritableByteChannel out, int intValue) throws IOException + private void writeInt(WritableByteChannel out, int intValue) throws IOException { final ByteBuffer buffer = ByteBuffer.allocate(Ints.BYTES); buffer.putInt(intValue); buffer.flip(); - out.write(buffer); + Channels.writeFully(out, buffer); } - public int readInt(InputStream in) throws IOException + private int readInt(InputStream in) throws IOException { byte[] intBytes = new byte[Ints.BYTES]; @@ -206,7 +160,7 @@ public int readInt(InputStream in) throws IOException return Ints.fromByteArray(intBytes); } - public void writeInts(OutputStream out, int[] ints) throws IOException + void writeInts(OutputStream out, int[] ints) throws IOException { writeInt(out, ints.length); @@ -215,7 +169,7 @@ public void writeInts(OutputStream out, int[] ints) throws IOException } } - public int[] readInts(InputStream in) throws IOException + int[] readInts(InputStream in) throws IOException { int size = readInt(in); @@ -227,7 +181,7 @@ public int[] readInts(InputStream in) throws IOException return retVal; } - public void writeLong(OutputStream out, long longValue) throws IOException + private void writeLong(OutputStream out, long longValue) throws IOException { out.write(Longs.toByteArray(longValue)); } @@ -237,10 +191,10 @@ public void writeLong(WritableByteChannel out, long longValue) throws IOExceptio final ByteBuffer buffer = ByteBuffer.allocate(Longs.BYTES); buffer.putLong(longValue); buffer.flip(); - out.write(buffer); + Channels.writeFully(out, buffer); } - public long readLong(InputStream in) throws IOException + long readLong(InputStream in) throws IOException { byte[] longBytes = new byte[Longs.BYTES]; @@ -249,7 +203,7 @@ public long readLong(InputStream in) throws IOException return Longs.fromByteArray(longBytes); } - public void writeLongs(OutputStream out, long[] longs) throws IOException + void writeLongs(OutputStream out, long[] longs) throws IOException { writeInt(out, longs.length); @@ -258,7 +212,7 @@ public void writeLongs(OutputStream out, long[] longs) throws IOException } } - public long[] readLongs(InputStream in) throws IOException + long[] readLongs(InputStream in) throws IOException { int size = readInt(in); @@ -275,20 +229,20 @@ public void writeFloat(OutputStream out, float floatValue) throws IOException writeInt(out, Float.floatToRawIntBits(floatValue)); } - public void writeFloat(WritableByteChannel out, float floatValue) throws IOException + void writeFloat(WritableByteChannel out, float floatValue) throws IOException { final ByteBuffer buffer = ByteBuffer.allocate(Floats.BYTES); buffer.putFloat(floatValue); buffer.flip(); - out.write(buffer); + Channels.writeFully(out, buffer); } - public float readFloat(InputStream in) throws IOException + float readFloat(InputStream in) throws IOException { return Float.intBitsToFloat(readInt(in)); } - public void writeFloats(OutputStream out, float[] floats) throws IOException + void writeFloats(OutputStream out, float[] floats) throws IOException { writeInt(out, floats.length); @@ -297,7 +251,7 @@ public void writeFloats(OutputStream out, float[] floats) throws IOException } } - public float[] readFloats(InputStream in) throws IOException + float[] readFloats(InputStream in) throws IOException { int size = readInt(in); diff --git a/common/src/main/java/io/druid/io/ByteBufferInputStream.java b/common/src/main/java/io/druid/io/ByteBufferInputStream.java new file mode 100644 index 000000000000..189bf777e512 --- /dev/null +++ b/common/src/main/java/io/druid/io/ByteBufferInputStream.java @@ -0,0 +1,71 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.io; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Streams bytes from the {@link ByteBuffer}'s position to the limit as {@link InputStream}. + */ +public final class ByteBufferInputStream extends InputStream +{ + private final ByteBuffer buffer; + + /** + * Does *not* make a copy of the given buffer, so the position of the given buffer is incremented, as the created + * InputStream is used. + */ + public ByteBufferInputStream(ByteBuffer buffer) + { + this.buffer = buffer; + } + + @Override + public int read() + { + if (!buffer.hasRemaining()) { + return -1; + } + return buffer.get() & 0xFF; + } + + @Override + public int read(byte[] bytes, int off, int len) + { + if (len == 0) { + return 0; + } + if (!buffer.hasRemaining()) { + return -1; + } + + len = Math.min(len, buffer.remaining()); + buffer.get(bytes, off, len); + return len; + } + + @Override + public int available() throws IOException + { + return buffer.remaining(); + } +} diff --git a/processing/src/main/java/io/druid/segment/data/IOPeon.java b/common/src/main/java/io/druid/io/Channels.java similarity index 70% rename from processing/src/main/java/io/druid/segment/data/IOPeon.java rename to common/src/main/java/io/druid/io/Channels.java index 09bfa090d8b5..876fcd898586 100644 --- a/processing/src/main/java/io/druid/segment/data/IOPeon.java +++ b/common/src/main/java/io/druid/io/Channels.java @@ -17,21 +17,22 @@ * under the License. */ -package io.druid.segment.data; +package io.druid.io; -import java.io.Closeable; -import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; -/** - */ -public interface IOPeon extends Closeable +public final class Channels { - OutputStream makeOutputStream(String filename) throws IOException; - - InputStream makeInputStream(String filename) throws IOException; + public static void writeFully(WritableByteChannel channel, ByteBuffer src) throws IOException + { + while (src.remaining() > 0) { + channel.write(src); + } + } - File getFile(String filename); + private Channels() + { + } } diff --git a/common/src/main/java/io/druid/math/expr/Parser.java b/common/src/main/java/io/druid/math/expr/Parser.java index 72e2608a9dc0..a05b73ac5aaf 100644 --- a/common/src/main/java/io/druid/math/expr/Parser.java +++ b/common/src/main/java/io/druid/math/expr/Parser.java @@ -35,6 +35,7 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTreeWalker; +import javax.annotation.Nullable; import java.lang.reflect.Modifier; import java.util.List; import java.util.Map; @@ -147,6 +148,16 @@ public void visit(Expr expr) return Lists.newArrayList(found); } + @Nullable + public static String getIdentifierIfIdentifier(Expr expr) + { + if (expr instanceof IdentifierExpr) { + return expr.toString(); + } else { + return null; + } + } + public static Expr.ObjectBinding withMap(final Map bindings) { return bindings::get; diff --git a/docs/content/configuration/indexing-service.md b/docs/content/configuration/indexing-service.md index 4181fecfaafb..1ce3152ef582 100644 --- a/docs/content/configuration/indexing-service.md +++ b/docs/content/configuration/indexing-service.md @@ -343,3 +343,26 @@ If the peon is running in remote mode, there must be an overlord up and running. |`druid.peon.taskActionClient.retry.minWait`|The minimum retry time to communicate with overlord.|PT5S| |`druid.peon.taskActionClient.retry.maxWait`|The maximum retry time to communicate with overlord.|PT1M| |`druid.peon.taskActionClient.retry.maxRetryCount`|The maximum number of retries to communicate with overlord.|60| + +##### SegmentWriteOutMediumFactory + +When new segments are created, Druid temporarily stores some pre-processed data in some buffers. Currently two types of +*medium* exist for those buffers: *temporary files* and *off-heap memory*. + +*Temporary files* (`tmpFile`) are stored under the task working directory (see `druid.indexer.task.baseTaskDir` +configuration above) and thus share it's mounting properies, e. g. they could be backed by HDD, SSD or memory (tmpfs). +This type of medium may do unnecessary disk I/O and requires some disk space to be available. + +*Off-heap memory medium* (`offHeapMemory`) creates buffers in off-heap memory of a JVM process that is running a task. +This type of medium is preferred, but it may require to allow the JVM to have more off-heap memory, by changing +`-XX:MaxDirectMemorySize` configuration. It is not yet understood how does the required off-heap memory size relates +to the size of the segments being created. But definitely it doesn't make sense to add more extra off-heap memory, +than the configured maximum *heap* size (`-Xmx`) for the same JVM. + +For most types of tasks SegmentWriteOutMediumFactory could be configured per-task (see [Tasks](../ingestion/tasks.html) +page, "TuningConfig" section), but if it's not specified for a task, or it's not supported for a particular task type, +then the value from the configuration below is used: + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.peon.defaultSegmentWriteOutMediumFactory`|`tmpFile` or `offHeapMemory`, see explanation above|`tmpFile`| diff --git a/docs/content/development/extensions-core/kafka-ingestion.md b/docs/content/development/extensions-core/kafka-ingestion.md index f8c64ef823af..b32c479fe7ba 100644 --- a/docs/content/development/extensions-core/kafka-ingestion.md +++ b/docs/content/development/extensions-core/kafka-ingestion.md @@ -129,6 +129,7 @@ The tuningConfig is optional and default parameters will be used if no tuningCon |`httpTimeout`|ISO8601 Period|How long to wait for a HTTP response from an indexing task.|no (default == PT10S)| |`shutdownTimeout`|ISO8601 Period|How long to wait for the supervisor to attempt a graceful shutdown of tasks before exiting.|no (default == PT80S)| |`offsetFetchPeriod`|ISO8601 Period|How often the supervisor queries Kafka and the indexing tasks to fetch current offsets and calculate lag.|no (default == PT30S, min == PT5S)| +|`segmentWriteOutMediumFactory`|String|Segment write-out medium to use when creating segments. See [Indexing Service Configuration](../configuration/indexing-service.html) page, "SegmentWriteOutMediumFactory" section for explanation and available options.|no (not specified by default, the value from `druid.peon.defaultSegmentWriteOutMediumFactory` is used)| #### IndexSpec diff --git a/docs/content/ingestion/stream-pull.md b/docs/content/ingestion/stream-pull.md index 7cd73c441af6..48e011ec0466 100644 --- a/docs/content/ingestion/stream-pull.md +++ b/docs/content/ingestion/stream-pull.md @@ -12,8 +12,7 @@ the [stream push method](stream-push.html). Another option is *stream pull*. With this approach, a Druid Realtime Node ingests data from a [Firehose](../ingestion/firehose.html) connected to the data you want to read. The Druid quickstart and tutorials do not include information about how to set up standalone realtime nodes, but -they can be used in place for Tranquility server and the indexing service. Please note that Realtime nodes have very properties than -the indexing service. +they can be used in place for Tranquility server and the indexing service. Please note that Realtime nodes have different properties and roles than the indexing service. ## Realtime Node Ingestion @@ -155,6 +154,7 @@ The tuningConfig is optional and default parameters will be used if no tuningCon |reportParseExceptions|Boolean|If true, exceptions encountered during parsing will be thrown and will halt ingestion. If false, unparseable rows and fields will be skipped. If an entire row is skipped, the "unparseable" counter will be incremented. If some fields in a row were parseable and some were not, the parseable fields will be indexed and the "unparseable" counter will not be incremented.|no (default == false)| |handoffConditionTimeout|long|Milliseconds to wait for segment handoff. It must be >= 0, where 0 means to wait forever.|no (default == 0)| |alertTimeout|long|Milliseconds timeout after which an alert is created if the task isn't finished by then. This allows users to monitor tasks that are failing to finish and give up the worker slot for any unexpected errors.|no (default == 0)| +|segmentWriteOutMediumFactory|String|Segment write-out medium to use when creating segments. See [Indexing Service Configuration](../configuration/indexing-service.html) page, "SegmentWriteOutMediumFactory" section for explanation and available options.|no (not specified by default, the value from `druid.peon.defaultSegmentWriteOutMediumFactory` is used)| |indexSpec|Object|Tune how data is indexed. See below for more information.|no| Before enabling thread priority settings, users are highly encouraged to read the [original pull request](https://github.com/druid-io/druid/pull/984) and other documentation about proper use of `-XX:+UseThreadPriorities`. diff --git a/docs/content/ingestion/tasks.md b/docs/content/ingestion/tasks.md index 628e2c9b9762..b5b1bc1c543b 100644 --- a/docs/content/ingestion/tasks.md +++ b/docs/content/ingestion/tasks.md @@ -143,9 +143,10 @@ The tuningConfig is optional and default parameters will be used if no tuningCon |indexSpec|defines segment storage format options to be used at indexing time, see [IndexSpec](#indexspec)|null|no| |maxPendingPersists|Maximum number of persists that can be pending but not started. If this limit would be exceeded by a new intermediate persist, ingestion will block until the currently-running persist finishes. Maximum heap memory usage for indexing scales with maxRowsInMemory * (2 + maxPendingPersists).|0 (meaning one persist can be running concurrently with ingestion, and none can be queued up)|no| |forceExtendableShardSpecs|Forces use of extendable shardSpecs. Experimental feature intended for use with the [Kafka indexing service extension](../development/extensions-core/kafka-ingestion.html).|false|no| -|forceGuaranteedRollup|Forces guaranteeing the [perfect rollup](./design/index.html). The perfect rollup optimizes the total size of generated segments and querying time while indexing time will be increased. This flag cannot be used with either `appendToExisting` of IOConfig or `forceExtendableShardSpecs`. For more details, see the below __Segment publishing modes__ section.|false|no| +|forceGuaranteedRollup|Forces guaranteeing the [perfect rollup](../design/index.html). The perfect rollup optimizes the total size of generated segments and querying time while indexing time will be increased. This flag cannot be used with either `appendToExisting` of IOConfig or `forceExtendableShardSpecs`. For more details, see the below __Segment publishing modes__ section.|false|no| |reportParseExceptions|If true, exceptions encountered during parsing will be thrown and will halt ingestion; if false, unparseable rows and fields will be skipped.|false|no| |publishTimeout|Milliseconds to wait for publishing segments. It must be >= 0, where 0 means to wait forever.|0|no| +|segmentWriteOutMediumFactory|Segment write-out medium to use when creating segments. See [Indexing Service Configuration](../configuration/indexing-service.html) page, "SegmentWriteOutMediumFactory" section for explanation and available options.|Not specified, the value from `druid.peon.defaultSegmentWriteOutMediumFactory` is used|no| #### IndexSpec diff --git a/extensions-contrib/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQFirehoseFactory.java b/extensions-contrib/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQFirehoseFactory.java index 60c194ca1e6b..1f4f03cb9594 100644 --- a/extensions-contrib/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQFirehoseFactory.java +++ b/extensions-contrib/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQFirehoseFactory.java @@ -33,6 +33,7 @@ import com.alibaba.rocketmq.remoting.exception.RemotingException; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Iterators; import com.google.common.collect.Sets; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; @@ -198,10 +199,14 @@ public Firehose connect( return new Firehose() { + private Iterator nextIterator = Iterators.emptyIterator(); @Override public boolean hasMore() { + if (nextIterator.hasNext()) { + return true; + } boolean hasMore = false; DruidPullRequest earliestPullRequest = null; @@ -252,15 +257,19 @@ public boolean hasMore() @Override public InputRow nextRow() { + if (nextIterator.hasNext()) { + return nextIterator.next(); + } + for (Map.Entry> entry : messageQueueTreeSetMap.entrySet()) { if (!entry.getValue().isEmpty()) { MessageExt message = entry.getValue().pollFirst(); - InputRow inputRow = theParser.parse(ByteBuffer.wrap(message.getBody())); + nextIterator = theParser.parseBatch(ByteBuffer.wrap(message.getBody())).iterator(); windows .computeIfAbsent(entry.getKey(), k -> new ConcurrentSkipListSet<>()) .add(message.getQueueOffset()); - return inputRow; + return nextIterator.next(); } } diff --git a/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java b/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java index fff7eaeeb57c..0c53007311d8 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java +++ b/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; import com.google.common.collect.Maps; import com.google.common.io.Closeables; import com.metamx.common.parsers.ParseException; @@ -37,6 +38,7 @@ import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; @@ -139,7 +141,7 @@ private Map loadOffsetFromPreviousMetaData(Object lastCommit) } log.info("Loaded offset map[%s]", offsetMap); } else { - log.makeAlert("Unable to cast lastCommit to Map for feed [%s]", feed); + log.makeAlert("Unable to cast lastCommit to Map for feed [%s]", feed).emit(); } return offsetMap; } @@ -175,6 +177,7 @@ public FirehoseV2 connect(final ByteBufferInputRowParser firehoseParser, Object private volatile boolean stopped; private volatile BytesMessageWithOffset msg = null; private volatile InputRow row = null; + private volatile Iterator nextIterator = Iterators.emptyIterator(); { lastOffsetPartitions = Maps.newHashMap(); @@ -202,14 +205,18 @@ private void nextMessage() try { row = null; while (row == null) { - if (msg != null) { - lastOffsetPartitions.put(msg.getPartition(), msg.offset()); + if (!nextIterator.hasNext()) { + if (msg != null) { + lastOffsetPartitions.put(msg.getPartition(), msg.offset()); + } + msg = messageQueue.take(); + final byte[] message = msg.message(); + nextIterator = message == null + ? Iterators.emptyIterator() + : firehoseParser.parseBatch(ByteBuffer.wrap(message)).iterator(); + continue; } - - msg = messageQueue.take(); - - final byte[] message = msg.message(); - row = message == null ? null : firehoseParser.parse(ByteBuffer.wrap(message)); + row = nextIterator.next(); } } catch (InterruptedException e) { diff --git a/extensions-contrib/orc-extensions/src/main/java/io/druid/data/input/orc/OrcHadoopInputRowParser.java b/extensions-contrib/orc-extensions/src/main/java/io/druid/data/input/orc/OrcHadoopInputRowParser.java index 4e249f1b97d2..ad08cf36c793 100644 --- a/extensions-contrib/orc-extensions/src/main/java/io/druid/data/input/orc/OrcHadoopInputRowParser.java +++ b/extensions-contrib/orc-extensions/src/main/java/io/druid/data/input/orc/OrcHadoopInputRowParser.java @@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import io.druid.data.input.InputRow; @@ -74,7 +75,7 @@ public OrcHadoopInputRowParser( @SuppressWarnings("ArgumentParameterSwap") @Override - public InputRow parse(OrcStruct input) + public List parseBatch(OrcStruct input) { Map map = Maps.newHashMap(); List fields = oip.getAllStructFieldRefs(); @@ -106,7 +107,7 @@ public InputRow parse(OrcStruct input) TimestampSpec timestampSpec = parseSpec.getTimestampSpec(); DateTime dateTime = timestampSpec.extractTimestamp(map); - return new MapBasedInputRow(dateTime, dimensions, map); + return ImmutableList.of(new MapBasedInputRow(dateTime, dimensions, map)); } private List getListObject(ListObjectInspector listObjectInspector, Object listObject) diff --git a/extensions-contrib/orc-extensions/src/test/java/io/druid/data/input/orc/DruidOrcInputFormatTest.java b/extensions-contrib/orc-extensions/src/test/java/io/druid/data/input/orc/DruidOrcInputFormatTest.java index 9736b10b40d0..80c6046d1b38 100644 --- a/extensions-contrib/orc-extensions/src/test/java/io/druid/data/input/orc/DruidOrcInputFormatTest.java +++ b/extensions-contrib/orc-extensions/src/test/java/io/druid/data/input/orc/DruidOrcInputFormatTest.java @@ -99,7 +99,7 @@ public void testRead() throws IOException, InterruptedException OrcStruct data = (OrcStruct) reader.getCurrentValue(); - MapBasedInputRow row = (MapBasedInputRow) parser.parse(data); + MapBasedInputRow row = (MapBasedInputRow) parser.parseBatch(data).get(0); Assert.assertTrue(row.getEvent().keySet().size() == 4); Assert.assertEquals(DateTimes.of(timestamp), row.getTimestamp()); diff --git a/extensions-contrib/orc-extensions/src/test/java/io/druid/data/input/orc/OrcHadoopInputRowParserTest.java b/extensions-contrib/orc-extensions/src/test/java/io/druid/data/input/orc/OrcHadoopInputRowParserTest.java index 455e432ea62b..c70376e83297 100644 --- a/extensions-contrib/orc-extensions/src/test/java/io/druid/data/input/orc/OrcHadoopInputRowParserTest.java +++ b/extensions-contrib/orc-extensions/src/test/java/io/druid/data/input/orc/OrcHadoopInputRowParserTest.java @@ -172,7 +172,7 @@ public void testParse() ); oi.setStructFieldData(struct, oi.getStructFieldRef("col6"), null); - final InputRow row = parser.parse(struct); + final InputRow row = parser.parseBatch(struct).get(0); Assert.assertEquals("timestamp", DateTimes.of("2000-01-01"), row.getTimestamp()); Assert.assertEquals("col1", "foo", row.getRaw("col1")); Assert.assertEquals("col2", ImmutableList.of("foo", "bar"), row.getRaw("col2")); diff --git a/extensions-contrib/parquet-extensions/src/main/java/io/druid/data/input/parquet/ParquetHadoopInputRowParser.java b/extensions-contrib/parquet-extensions/src/main/java/io/druid/data/input/parquet/ParquetHadoopInputRowParser.java index a6a8ae2b506b..76d9cd703fd8 100755 --- a/extensions-contrib/parquet-extensions/src/main/java/io/druid/data/input/parquet/ParquetHadoopInputRowParser.java +++ b/extensions-contrib/parquet-extensions/src/main/java/io/druid/data/input/parquet/ParquetHadoopInputRowParser.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; @@ -77,7 +78,7 @@ private LogicalType determineTimestampSpecLogicalType(Schema schema, String time * imitate avro extension {@link io.druid.data.input.avro.AvroParsers#parseGenericRecord(GenericRecord, ParseSpec, ObjectFlattener)} */ @Override - public InputRow parse(GenericRecord record) + public List parseBatch(GenericRecord record) { // Map the record to a map GenericRecordAsMap genericRecordAsMap = new GenericRecordAsMap(record, binaryAsString); @@ -97,7 +98,7 @@ public InputRow parse(GenericRecord record) dateTime = timestampSpec.extractTimestamp(genericRecordAsMap); } - return new MapBasedInputRow(dateTime, dimensions, genericRecordAsMap); + return ImmutableList.of(new MapBasedInputRow(dateTime, dimensions, genericRecordAsMap)); } @JsonProperty diff --git a/extensions-contrib/parquet-extensions/src/test/java/io/druid/data/input/parquet/DruidParquetInputTest.java b/extensions-contrib/parquet-extensions/src/test/java/io/druid/data/input/parquet/DruidParquetInputTest.java index c470b6a0eb1d..f64233b20efe 100644 --- a/extensions-contrib/parquet-extensions/src/test/java/io/druid/data/input/parquet/DruidParquetInputTest.java +++ b/extensions-contrib/parquet-extensions/src/test/java/io/druid/data/input/parquet/DruidParquetInputTest.java @@ -57,7 +57,10 @@ public void testReadParquetFile() throws IOException, InterruptedException // field not read, should return null assertEquals(data.get("added"), null); assertEquals(data.get("page"), new Utf8("Gypsy Danger")); - assertEquals(config.getParser().parse(data).getDimension("page").get(0), "Gypsy Danger"); + assertEquals( + ((List) config.getParser().parseBatch(data)).get(0).getDimension("page").get(0), + "Gypsy Danger" + ); } @Test @@ -70,7 +73,7 @@ public void testBinaryAsString() throws IOException, InterruptedException config.intoConfiguration(job); GenericRecord data = getFirstRecord(job, ((StaticPathSpec) config.getPathSpec()).getPaths()); - InputRow row = config.getParser().parse(data); + InputRow row = ((List) config.getParser().parseBatch(data)).get(0); // without binaryAsString: true, the value would something like "[104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 3.... ]" assertEquals(row.getDimension("field").get(0), "hey this is &é(-è_çà)=^$ù*! Ω^^"); @@ -133,7 +136,7 @@ private List getAllRows(String configPath) throws IOException, Interru while (reader.nextKeyValue()) { reader.nextKeyValue(); GenericRecord data = (GenericRecord) reader.getCurrentValue(); - records.add(parser.parse(data)); + records.add(((List) parser.parseBatch(data)).get(0)); } return records; diff --git a/extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java b/extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java index bbd54aa44f96..f6759da27e90 100644 --- a/extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java +++ b/extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Iterators; import com.rabbitmq.client.AMQP; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; @@ -45,6 +46,7 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Iterator; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -117,7 +119,9 @@ public RabbitMQFirehoseFactory( ) throws Exception { this.connectionFactory = connectionFactory == null - ? connectionFactoryCOMPAT == null ? JacksonifiedConnectionFactory.makeDefaultConnectionFactory() : connectionFactoryCOMPAT + ? connectionFactoryCOMPAT == null + ? JacksonifiedConnectionFactory.makeDefaultConnectionFactory() + : connectionFactoryCOMPAT : connectionFactory; this.config = config == null ? RabbitMQFirehoseConfig.makeDefaultConfig() : config; @@ -190,10 +194,10 @@ public void shutdownCompleted(ShutdownSignalException cause) return new Firehose() { /** - * Storing the latest delivery as a member variable should be safe since this will only be run + * Storing the latest row as a member variable should be safe since this will only be run * by a single thread. */ - private Delivery delivery; + private InputRow nextRow; /** * Store the latest delivery tag to be able to commit (acknowledge) the message delivery up to @@ -201,17 +205,27 @@ public void shutdownCompleted(ShutdownSignalException cause) */ private long lastDeliveryTag; + private Iterator nextIterator = Iterators.emptyIterator(); + @Override public boolean hasMore() { - delivery = null; + nextRow = null; try { + if (nextIterator.hasNext()) { + nextRow = nextIterator.next(); + return true; + } // Wait for the next delivery. This will block until something is available. - delivery = consumer.nextDelivery(); + final Delivery delivery = consumer.nextDelivery(); if (delivery != null) { lastDeliveryTag = delivery.getEnvelope().getDeliveryTag(); - // If delivery is non-null, we report that there is something more to process. - return true; + nextIterator = firehoseParser.parseBatch(ByteBuffer.wrap(delivery.getBody())).iterator(); + if (nextIterator.hasNext()) { + nextRow = nextIterator.next(); + // If delivery is non-null, we report that there is something more to process. + return true; + } } } catch (InterruptedException e) { @@ -230,13 +244,13 @@ public boolean hasMore() @Override public InputRow nextRow() { - if (delivery == null) { + if (nextRow == null) { //Just making sure. log.wtf("I have nothing in delivery. Method hasMore() should have returned false."); return null; } - return firehoseParser.parse(ByteBuffer.wrap(delivery.getBody())); + return nextRow; } @Override diff --git a/extensions-contrib/thrift-extensions/src/main/java/io/druid/data/input/thrift/ThriftInputRowParser.java b/extensions-contrib/thrift-extensions/src/main/java/io/druid/data/input/thrift/ThriftInputRowParser.java index 94a07d22ecfe..87169252b8f8 100644 --- a/extensions-contrib/thrift-extensions/src/main/java/io/druid/data/input/thrift/ThriftInputRowParser.java +++ b/extensions-contrib/thrift-extensions/src/main/java/io/druid/data/input/thrift/ThriftInputRowParser.java @@ -22,12 +22,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.metamx.common.IAE; import com.twitter.elephantbird.mapreduce.io.ThriftWritable; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.InputRowParser; import io.druid.data.input.impl.ParseSpec; +import io.druid.java.util.common.parsers.Parser; import org.apache.hadoop.io.BytesWritable; import org.apache.thrift.TBase; import org.apache.thrift.TException; @@ -37,10 +39,9 @@ import java.net.URL; import java.net.URLClassLoader; import java.nio.ByteBuffer; +import java.util.List; import java.util.Map; -import io.druid.java.util.common.parsers.Parser; - /** * 1. load thrift class from classpath or provided jar * 2. deserialize content bytes and serialize to json @@ -89,7 +90,7 @@ public Class getThriftClass() @Override - public InputRow parse(Object input) + public List parseBatch(Object input) { if (parser == null) { // parser should be created when it is really used to avoid unnecessary initialization of the underlying @@ -137,13 +138,13 @@ public InputRow parse(Object input) throw new IAE("some thing wrong with your thrift?"); } - Map record = parser.parse(json); + Map record = parser.parseToMap(json); - return new MapBasedInputRow( + return ImmutableList.of(new MapBasedInputRow( parseSpec.getTimestampSpec().extractTimestamp(record), parseSpec.getDimensionsSpec().getDimensionNames(), record - ); + )); } @Override diff --git a/extensions-contrib/thrift-extensions/src/test/java/io/druid/data/input/thrift/ThriftInputRowParserTest.java b/extensions-contrib/thrift-extensions/src/test/java/io/druid/data/input/thrift/ThriftInputRowParserTest.java index c811bff64cbd..ed6a794c640b 100644 --- a/extensions-contrib/thrift-extensions/src/test/java/io/druid/data/input/thrift/ThriftInputRowParserTest.java +++ b/extensions-contrib/thrift-extensions/src/test/java/io/druid/data/input/thrift/ThriftInputRowParserTest.java @@ -146,17 +146,17 @@ public void testDisableJavaScript() expectedException.expect(CoreMatchers.instanceOf(IllegalStateException.class)); expectedException.expectMessage("JavaScript is disabled"); - parser.parse(ByteBuffer.allocate(1)); + parser.parseBatch(ByteBuffer.allocate(1)).get(0); } public void serializationAndTest(ThriftInputRowParser parser, byte[] bytes) throws TException { ByteBuffer buffer = ByteBuffer.wrap(bytes); - InputRow row1 = parser.parse(buffer); + InputRow row1 = parser.parseBatch(buffer).get(0); assertTrue(row1.getDimension("title").get(0).equals("title")); - InputRow row2 = parser.parse(new BytesWritable(bytes)); + InputRow row2 = parser.parseBatch(new BytesWritable(bytes)).get(0); assertTrue(row2.getDimension("lastName").get(0).equals("last")); } } diff --git a/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroHadoopInputRowParser.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroHadoopInputRowParser.java index 7b88aaa2f936..76debb8ff865 100644 --- a/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroHadoopInputRowParser.java +++ b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroHadoopInputRowParser.java @@ -26,6 +26,8 @@ import io.druid.java.util.common.parsers.ObjectFlattener; import org.apache.avro.generic.GenericRecord; +import java.util.List; + public class AvroHadoopInputRowParser implements InputRowParser { private final ParseSpec parseSpec; @@ -44,7 +46,7 @@ public AvroHadoopInputRowParser( } @Override - public InputRow parse(GenericRecord record) + public List parseBatch(GenericRecord record) { return AvroParsers.parseGenericRecord(record, parseSpec, avroFlattener); } diff --git a/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroStreamInputRowParser.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroStreamInputRowParser.java index eabfe5566c1d..f83efea2bbdf 100644 --- a/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroStreamInputRowParser.java +++ b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroStreamInputRowParser.java @@ -28,6 +28,7 @@ import org.apache.avro.generic.GenericRecord; import java.nio.ByteBuffer; +import java.util.List; import java.util.Objects; public class AvroStreamInputRowParser implements ByteBufferInputRowParser @@ -48,7 +49,7 @@ public AvroStreamInputRowParser( } @Override - public InputRow parse(ByteBuffer input) + public List parseBatch(ByteBuffer input) { return AvroParsers.parseGenericRecord(avroBytesDecoder.parse(input), parseSpec, avroFlattener); } diff --git a/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroParsers.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroParsers.java index b128c15d442f..ac38f669e0e6 100644 --- a/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroParsers.java +++ b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroParsers.java @@ -27,6 +27,8 @@ import io.druid.java.util.common.parsers.ObjectFlatteners; import org.apache.avro.generic.GenericRecord; +import java.util.List; + public class AvroParsers { private AvroParsers() @@ -50,12 +52,12 @@ public static ObjectFlattener makeFlattener( return ObjectFlatteners.create(flattenSpec, new AvroFlattenerMaker(fromPigAvroStorage, binaryAsString)); } - public static InputRow parseGenericRecord( + public static List parseGenericRecord( GenericRecord record, ParseSpec parseSpec, ObjectFlattener avroFlattener ) { - return new MapInputRowParser(parseSpec).parse(avroFlattener.flatten(record)); + return new MapInputRowParser(parseSpec).parseBatch(avroFlattener.flatten(record)); } } diff --git a/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroHadoopInputRowParserTest.java b/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroHadoopInputRowParserTest.java index 1a383cad09a0..38c2d33ec5a3 100644 --- a/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroHadoopInputRowParserTest.java +++ b/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroHadoopInputRowParserTest.java @@ -82,7 +82,7 @@ private void testParse(GenericRecord record, boolean fromPigAvroStorage) throws jsonMapper.writeValueAsBytes(parser), AvroHadoopInputRowParser.class ); - InputRow inputRow = parser2.parse(record); + InputRow inputRow = parser2.parseBatch(record).get(0); assertInputRowCorrect(inputRow, DIMENSIONS); } diff --git a/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java b/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java index 7df4430997e5..0f6e3ce9a416 100644 --- a/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java +++ b/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java @@ -214,7 +214,7 @@ public void testParse() throws SchemaValidationException, IOException // write avro datum to bytes writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null)); - InputRow inputRow = parser2.parse(ByteBuffer.wrap(out.toByteArray())); + InputRow inputRow = parser2.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); assertInputRowCorrect(inputRow, DIMENSIONS); } @@ -255,7 +255,7 @@ public void testParseSchemaless() throws SchemaValidationException, IOException // write avro datum to bytes writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null)); - InputRow inputRow = parser2.parse(ByteBuffer.wrap(out.toByteArray())); + InputRow inputRow = parser2.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); assertInputRowCorrect(inputRow, DIMENSIONS_SCHEMALESS); } diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index c037f8317456..f16fa37ec795 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -38,7 +38,7 @@ com.yahoo.datasketches sketches-core - 0.10.1 + 0.10.3 io.druid diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java new file mode 100644 index 000000000000..7ebb40b5d7a5 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -0,0 +1,268 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.metamx.common.IAE; +import com.yahoo.sketches.Util; +import com.yahoo.sketches.quantiles.DoublesSketch; +import com.yahoo.sketches.quantiles.DoublesUnion; + +import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorFactoryNotMergeableException; +import io.druid.query.aggregation.AggregatorUtil; +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.cache.CacheKeyBuilder; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; +import io.druid.segment.NilColumnValueSelector; +import io.druid.segment.column.ValueType; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + +public class DoublesSketchAggregatorFactory extends AggregatorFactory +{ + + private static final int DEFAULT_K = 128; + + // Used for sketch size estimation. + private static final long MAX_STREAM_LENGTH = 1_000_000_000; + + private final String name; + private final String fieldName; + private final int k; + private final byte cacheTypeId; + + @JsonCreator + public DoublesSketchAggregatorFactory( + @JsonProperty("name") final String name, + @JsonProperty("fieldName") final String fieldName, + @JsonProperty("k") final Integer k) + { + this(name, fieldName, k, AggregatorUtil.QUANTILES_DOUBLES_SKETCH_BUILD_CACHE_TYPE_ID); + } + + DoublesSketchAggregatorFactory(final String name, final String fieldName, final Integer k, final byte cacheTypeId) + { + if (name == null) { + throw new IAE("Must have a valid, non-null aggregator name"); + } + this.name = name; + if (fieldName == null) { + throw new IAE("Parameter fieldName must be specified"); + } + this.fieldName = fieldName; + this.k = k == null ? DEFAULT_K : k; + Util.checkIfPowerOf2(this.k, "k"); + this.cacheTypeId = cacheTypeId; + } + + @Override + public Aggregator factorize(final ColumnSelectorFactory metricFactory) + { + if (metricFactory.getColumnCapabilities(fieldName) != null + && ValueType.isNumeric(metricFactory.getColumnCapabilities(fieldName).getType())) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { + return new DoublesSketchNoOpAggregator(); + } + return new DoublesSketchBuildAggregator(selector, k); + } + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { + return new DoublesSketchNoOpAggregator(); + } + return new DoublesSketchMergeAggregator(selector, k); + } + + @Override + public BufferAggregator factorizeBuffered(final ColumnSelectorFactory metricFactory) + { + if (metricFactory.getColumnCapabilities(fieldName) != null + && ValueType.isNumeric(metricFactory.getColumnCapabilities(fieldName).getType())) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { + return new DoublesSketchNoOpBufferAggregator(); + } + return new DoublesSketchBuildBufferAggregator(selector, k, getMaxIntermediateSize()); + } + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { + return new DoublesSketchNoOpBufferAggregator(); + } + return new DoublesSketchMergeBufferAggregator(selector, k, getMaxIntermediateSize()); + } + + @Override + public Object deserialize(final Object object) + { + return DoublesSketchOperations.deserialize(object); + } + + public static final Comparator COMPARATOR = new Comparator() + { + @Override + public int compare(DoublesSketch a, DoublesSketch b) + { + return Long.compare(a.getN(), b.getN()); + } + }; + + @Override + public Comparator getComparator() + { + return COMPARATOR; + } + + @Override + public Object combine(final Object lhs, final Object rhs) + { + final DoublesUnion union = DoublesUnion.builder().setMaxK(k).build(); + union.update((DoublesSketch) lhs); + union.update((DoublesSketch) rhs); + return union.getResultAndReset(); + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public String getFieldName() + { + return fieldName; + } + + @JsonProperty + public int getK() + { + return k; + } + + @Override + public List requiredFields() + { + return Collections.singletonList(fieldName); + } + + // Quantiles sketches never stop growing, but they do so very slowly. + // This size must suffice for overwhelming majority of sketches, + // but some sketches may request more memory on heap and move there + @Override + public int getMaxIntermediateSize() + { + return DoublesSketch.getUpdatableStorageBytes(k, MAX_STREAM_LENGTH); + } + + @Override + public List getRequiredColumns() + { + return Collections. singletonList( + new DoublesSketchAggregatorFactory( + fieldName, + fieldName, + k) + ); + } + + @Override + public AggregatorFactory getCombiningFactory() + { + return new DoublesSketchMergeAggregatorFactory(name, k); + } + + @Override + public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException + { + if (other.getName().equals(this.getName()) && other instanceof DoublesSketchAggregatorFactory) { + // DoublesUnion supports inputs with different k. + // The result will have effective k between the specified k and the minimum k from all input sketches + // to achieve higher accuracy as much as possible. + return new DoublesSketchMergeAggregatorFactory(name, Math.max(k, ((DoublesSketchAggregatorFactory) other).k)); + } else { + throw new AggregatorFactoryNotMergeableException(this, other); + } + } + + @Override + public Object finalizeComputation(final Object object) + { + return ((DoublesSketch) object).getN(); + } + + @Override + public String getTypeName() + { + return DoublesSketchModule.DOUBLES_SKETCH; + } + + @Override + public byte[] getCacheKey() + { + return new CacheKeyBuilder(cacheTypeId).appendString(name).appendString(fieldName).appendInt(k).build(); + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || !getClass().equals(o.getClass())) { + return false; + } + final DoublesSketchAggregatorFactory that = (DoublesSketchAggregatorFactory) o; + if (!name.equals(that.name)) { + return false; + } + if (!fieldName.equals(that.fieldName)) { + return false; + } + if (k != that.k) { + return false; + } + return true; + } + + @Override + public int hashCode() + { + return Objects.hash(name, fieldName, k); // no need to use cacheTypeId here + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "name=" + name + + ", fieldName=" + fieldName + + ", k=" + k + + "}"; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildAggregator.java new file mode 100644 index 000000000000..f957f5bb0ce1 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildAggregator.java @@ -0,0 +1,78 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.yahoo.sketches.quantiles.UpdateDoublesSketch; + +import io.druid.query.aggregation.Aggregator; +import io.druid.segment.ColumnValueSelector; + +public class DoublesSketchBuildAggregator implements Aggregator +{ + + private final ColumnValueSelector valueSelector; + private final int size; + + private UpdateDoublesSketch sketch; + + public DoublesSketchBuildAggregator(final ColumnValueSelector valueSelector, final int size) + { + this.valueSelector = valueSelector; + this.size = size; + sketch = UpdateDoublesSketch.builder().setK(size).build(); + } + + @Override + public synchronized void aggregate() + { + sketch.update(valueSelector.getDouble()); + } + + @Override + public synchronized Object get() + { + return sketch; + } + + @Override + public float getFloat() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public synchronized void reset() + { + sketch = UpdateDoublesSketch.builder().setK(size).build(); + } + + @Override + public synchronized void close() + { + sketch = null; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java new file mode 100644 index 000000000000..1318c11a725d --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java @@ -0,0 +1,131 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.nio.ByteBuffer; +import java.util.IdentityHashMap; + +import com.yahoo.memory.WritableMemory; +import com.yahoo.sketches.quantiles.UpdateDoublesSketch; + +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.ColumnValueSelector; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; + +public class DoublesSketchBuildBufferAggregator implements BufferAggregator +{ + + private final ColumnValueSelector selector; + private final int size; + private final int maxIntermediateSize; + + private final IdentityHashMap memCache = new IdentityHashMap<>(); + private final IdentityHashMap> sketches = new IdentityHashMap<>(); + + public DoublesSketchBuildBufferAggregator(final ColumnValueSelector valueSelector, final int size, + final int maxIntermediateSize) + { + this.selector = valueSelector; + this.size = size; + this.maxIntermediateSize = maxIntermediateSize; + } + + @Override + public synchronized void init(final ByteBuffer buffer, final int position) + { + final WritableMemory mem = getMemory(buffer); + final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); + final UpdateDoublesSketch sketch = UpdateDoublesSketch.builder().setK(size).build(region); + putSketch(buffer, position, sketch); + } + + @Override + public synchronized void aggregate(final ByteBuffer buffer, final int position) + { + final UpdateDoublesSketch sketch = sketches.get(buffer).get(position); + sketch.update(selector.getDouble()); + } + + @Override + public synchronized Object get(final ByteBuffer buffer, final int position) + { + return sketches.get(buffer).get(position); + } + + @Override + public float getFloat(final ByteBuffer buffer, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong(final ByteBuffer buffer, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public synchronized void close() + { + sketches.clear(); + memCache.clear(); + } + + // A small number of sketches may run out of the given memory, request more memory on heap and move there. + // In that case we need to reuse the object from the cache as opposed to wrapping the new buffer. + @Override + public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) + { + UpdateDoublesSketch sketch = sketches.get(oldBuffer).get(oldPosition); + final WritableMemory oldRegion = getMemory(oldBuffer).writableRegion(oldPosition, maxIntermediateSize); + if (sketch.isSameResource(oldRegion)) { // sketch was not relocated on heap + final WritableMemory newRegion = getMemory(newBuffer).writableRegion(newPosition, maxIntermediateSize); + sketch = UpdateDoublesSketch.wrap(newRegion); + } + putSketch(newBuffer, newPosition, sketch); + + final Int2ObjectMap map = sketches.get(oldBuffer); + map.remove(oldPosition); + if (map.isEmpty()) { + sketches.remove(oldBuffer); + memCache.remove(oldBuffer); + } + } + + private WritableMemory getMemory(final ByteBuffer buffer) + { + return memCache.computeIfAbsent(buffer, buf -> WritableMemory.wrap(buf)); + } + + private void putSketch(final ByteBuffer buffer, final int position, final UpdateDoublesSketch sketch) + { + Int2ObjectMap map = sketches.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>()); + map.put(position, sketch); + } + + @Override + public void inspectRuntimeShape(final RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java new file mode 100644 index 000000000000..ab54f7d0b0b6 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchComplexMetricSerde.java @@ -0,0 +1,116 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.yahoo.memory.Memory; +import com.yahoo.sketches.quantiles.DoublesSketch; +import com.yahoo.sketches.quantiles.UpdateDoublesSketch; +import io.druid.data.input.InputRow; +import io.druid.java.util.common.IAE; +import io.druid.segment.GenericColumnSerializer; +import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.ObjectStrategy; +import io.druid.segment.serde.ComplexColumnPartSupplier; +import io.druid.segment.serde.ComplexMetricExtractor; +import io.druid.segment.serde.ComplexMetricSerde; +import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; +import io.druid.segment.writeout.SegmentWriteOutMedium; + +import java.nio.ByteBuffer; + +public class DoublesSketchComplexMetricSerde extends ComplexMetricSerde +{ + + private static final DoublesSketchObjectStrategy strategy = new DoublesSketchObjectStrategy(); + + @Override + public String getTypeName() + { + return DoublesSketchModule.DOUBLES_SKETCH; + } + + @Override + public ObjectStrategy getObjectStrategy() + { + return strategy; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + return new ComplexMetricExtractor() + { + private static final int MIN_K = 2; // package one input value into the smallest sketch + + @Override + public Class extractedClass() + { + return DoublesSketch.class; + } + + @Override + public Object extractValue(final InputRow inputRow, final String metricName) + { + final Object object = inputRow.getRaw(metricName); + if (object instanceof String) { // everything is a string during ingestion + String objectString = (String) object; + // Autodetection of the input format: a number or base64 encoded sketch + // A serialized DoublesSketch, as currently implemented, always has 0 in the first 6 bits. + // This corresponds to "A" in base64, so it is not a digit + if (Character.isDigit((objectString).charAt(0))) { + try { + Double doubleValue = Double.parseDouble(objectString); + UpdateDoublesSketch sketch = DoublesSketch.builder().setK(MIN_K).build(); + sketch.update(doubleValue); + return sketch; + } + catch (NumberFormatException e) { + throw new IAE("Expected a string with a number, received value " + objectString); + } + } + } else if (object instanceof Number) { // this is for reindexing + UpdateDoublesSketch sketch = DoublesSketch.builder().setK(MIN_K).build(); + sketch.update(((Number) object).doubleValue()); + return sketch; + } + + if (object == null || object instanceof DoublesSketch || object instanceof Memory) { + return object; + } + return DoublesSketchOperations.deserialize(object); + } + }; + } + + @Override + public void deserializeColumn(final ByteBuffer buffer, final ColumnBuilder builder) + { + final GenericIndexed column = GenericIndexed.read(buffer, strategy, builder.getFileMapper()); + builder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column)); + } + + // support large columns + @Override + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + { + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); + } +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java new file mode 100644 index 000000000000..b2b51640a833 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchJsonSerializer.java @@ -0,0 +1,39 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchJsonSerializer extends JsonSerializer +{ + + @Override + public void serialize(final DoublesSketch sketch, final JsonGenerator generator, final SerializerProvider provider) + throws IOException + { + generator.writeBinary(sketch.toByteArray(true)); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java new file mode 100644 index 000000000000..6152ff417a34 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java @@ -0,0 +1,80 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.yahoo.sketches.quantiles.DoublesSketch; +import com.yahoo.sketches.quantiles.DoublesUnion; + +import io.druid.query.aggregation.Aggregator; +import io.druid.segment.ColumnValueSelector; + +public class DoublesSketchMergeAggregator implements Aggregator +{ + + private final ColumnValueSelector selector; + private DoublesUnion union; + + public DoublesSketchMergeAggregator(final ColumnValueSelector selector, final int k) + { + this.selector = selector; + union = DoublesUnion.builder().setMaxK(k).build(); + } + + @Override + public synchronized void aggregate() + { + final DoublesSketch sketch = selector.getObject(); + if (sketch == null) { + return; + } + union.update(sketch); + } + + @Override + public synchronized Object get() + { + return union.getResult(); + } + + @Override + public float getFloat() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public synchronized void reset() + { + union.reset(); + } + + @Override + public synchronized void close() + { + union = null; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java new file mode 100644 index 000000000000..5c3d7b61dcbb --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregatorFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.sketches.quantiles.DoublesSketch; +import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.AggregatorUtil; +import io.druid.query.aggregation.BufferAggregator; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; +import io.druid.segment.NilColumnValueSelector; + +public class DoublesSketchMergeAggregatorFactory extends DoublesSketchAggregatorFactory +{ + + @JsonCreator + public DoublesSketchMergeAggregatorFactory( + @JsonProperty("name") final String name, + @JsonProperty("k") final Integer k) + { + super(name, name, k, AggregatorUtil.QUANTILES_DOUBLES_SKETCH_MERGE_CACHE_TYPE_ID); + } + + @Override + public Aggregator factorize(final ColumnSelectorFactory metricFactory) + { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); + if (selector instanceof NilColumnValueSelector) { + return new DoublesSketchNoOpAggregator(); + } + return new DoublesSketchMergeAggregator(selector, getK()); + } + + @Override + public BufferAggregator factorizeBuffered(final ColumnSelectorFactory metricFactory) + { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); + if (selector instanceof NilColumnValueSelector) { + return new DoublesSketchNoOpBufferAggregator(); + } + return new DoublesSketchMergeBufferAggregator(selector, getK(), getMaxIntermediateSize()); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java new file mode 100644 index 000000000000..8578e7f125cb --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregator.java @@ -0,0 +1,137 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.yahoo.memory.WritableMemory; +import com.yahoo.sketches.quantiles.DoublesSketch; +import com.yahoo.sketches.quantiles.DoublesUnion; +import com.yahoo.sketches.quantiles.DoublesUnionBuilder; +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.ColumnValueSelector; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; + +import java.nio.ByteBuffer; +import java.util.IdentityHashMap; + +public class DoublesSketchMergeBufferAggregator implements BufferAggregator +{ + + private final ColumnValueSelector selector; + private final int k; + private final int maxIntermediateSize; + private final IdentityHashMap memCache = new IdentityHashMap<>(); + private final IdentityHashMap> unions = new IdentityHashMap<>(); + + public DoublesSketchMergeBufferAggregator( + final ColumnValueSelector selector, + final int k, + final int maxIntermediateSize) + { + this.selector = selector; + this.k = k; + this.maxIntermediateSize = maxIntermediateSize; + } + + @Override + public synchronized void init(final ByteBuffer buffer, final int position) + { + final WritableMemory mem = getMemory(buffer); + final WritableMemory region = mem.writableRegion(position, maxIntermediateSize); + final DoublesUnion union = DoublesUnion.builder().setMaxK(k).build(region); + putUnion(buffer, position, union); + } + + @Override + public synchronized void aggregate(final ByteBuffer buffer, final int position) + { + final DoublesSketch sketch = selector.getObject(); + if (sketch == null) { + return; + } + final DoublesUnion union = unions.get(buffer).get(position); + union.update(sketch); + } + + @Override + public synchronized Object get(final ByteBuffer buffer, final int position) + { + return unions.get(buffer).get(position).getResult(); + } + + @Override + public float getFloat(final ByteBuffer buffer, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong(final ByteBuffer buffer, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public synchronized void close() + { + unions.clear(); + memCache.clear(); + } + + // A small number of sketches may run out of the given memory, request more memory on heap and move there. + // In that case we need to reuse the object from the cache as opposed to wrapping the new buffer. + @Override + public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) + { + DoublesUnion union = unions.get(oldBuffer).get(oldPosition); + final WritableMemory oldMem = getMemory(oldBuffer).writableRegion(oldPosition, maxIntermediateSize); + if (union.isSameResource(oldMem)) { // union was not relocated on heap + final WritableMemory newMem = getMemory(newBuffer).writableRegion(newPosition, maxIntermediateSize); + union = DoublesUnionBuilder.wrap(newMem); + } + putUnion(newBuffer, newPosition, union); + + Int2ObjectMap map = unions.get(oldBuffer); + map.remove(oldPosition); + if (map.isEmpty()) { + unions.remove(oldBuffer); + memCache.remove(oldBuffer); + } + } + + private WritableMemory getMemory(final ByteBuffer buffer) + { + return memCache.computeIfAbsent(buffer, buf -> WritableMemory.wrap(buf)); + } + + private void putUnion(final ByteBuffer buffer, final int position, final DoublesUnion union) + { + Int2ObjectMap map = unions.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>()); + map.put(position, union); + } + + @Override + public void inspectRuntimeShape(final RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java new file mode 100644 index 000000000000..0e3081965b03 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java @@ -0,0 +1,65 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Arrays; +import java.util.List; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import com.yahoo.sketches.quantiles.DoublesSketch; + +import io.druid.initialization.DruidModule; +import io.druid.segment.serde.ComplexMetrics; + +public class DoublesSketchModule implements DruidModule +{ + + public static final String DOUBLES_SKETCH = "quantilesDoublesSketch"; + + public static final String DOUBLES_SKETCH_HISTOGRAM_POST_AGG = "quantilesDoublesSketchToHistogram"; + public static final String DOUBLES_SKETCH_QUANTILE_POST_AGG = "quantilesDoublesSketchToQuantile"; + public static final String DOUBLES_SKETCH_QUANTILES_POST_AGG = "quantilesDoublesSketchToQuantiles"; + public static final String DOUBLES_SKETCH_TO_STRING_POST_AGG = "quantilesDoublesSketchToString"; + + @Override + public void configure(final Binder binder) + { + if (ComplexMetrics.getSerdeForType(DOUBLES_SKETCH) == null) { + ComplexMetrics.registerSerde(DOUBLES_SKETCH, new DoublesSketchComplexMetricSerde()); + } + } + + @Override + public List getJacksonModules() + { + return Arrays. asList( + new SimpleModule("DoublesQuantilesSketchModule").registerSubtypes( + new NamedType(DoublesSketchAggregatorFactory.class, DOUBLES_SKETCH), + new NamedType(DoublesSketchToHistogramPostAggregator.class, DOUBLES_SKETCH_HISTOGRAM_POST_AGG), + new NamedType(DoublesSketchToQuantilePostAggregator.class, DOUBLES_SKETCH_QUANTILE_POST_AGG), + new NamedType(DoublesSketchToQuantilesPostAggregator.class, DOUBLES_SKETCH_QUANTILES_POST_AGG), + new NamedType(DoublesSketchToStringPostAggregator.class, DOUBLES_SKETCH_TO_STRING_POST_AGG)) + .addSerializer(DoublesSketch.class, new DoublesSketchJsonSerializer())); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpAggregator.java new file mode 100644 index 000000000000..2cfeb03a534b --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpAggregator.java @@ -0,0 +1,60 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import io.druid.query.aggregation.Aggregator; + +public class DoublesSketchNoOpAggregator implements Aggregator +{ + + @Override + public Object get() + { + return DoublesSketchOperations.EMPTY_SKETCH; + } + + @Override + public void aggregate() + { + } + + @Override + public void reset() + { + } + + @Override + public void close() + { + } + + @Override + public float getFloat() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong() + { + throw new UnsupportedOperationException("Not implemented"); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpBufferAggregator.java new file mode 100644 index 000000000000..ad61ff5372f8 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchNoOpBufferAggregator.java @@ -0,0 +1,68 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.nio.ByteBuffer; + +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; + +public class DoublesSketchNoOpBufferAggregator implements BufferAggregator +{ + + @Override + public void init(final ByteBuffer buf, final int position) + { + } + + @Override + public void aggregate(final ByteBuffer buf, final int position) + { + } + + @Override + public Object get(final ByteBuffer buf, final int position) + { + return DoublesSketchOperations.EMPTY_SKETCH; + } + + @Override + public float getFloat(final ByteBuffer buf, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong(final ByteBuffer buf, final int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void close() + { + } + + @Override + public void inspectRuntimeShape(final RuntimeShapeInspector inspector) + { + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java new file mode 100644 index 000000000000..3f89c905803f --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchObjectStrategy.java @@ -0,0 +1,64 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.nio.ByteBuffer; + +import com.yahoo.memory.Memory; +import com.yahoo.sketches.quantiles.DoublesSketch; + +import io.druid.segment.data.ObjectStrategy; + +public class DoublesSketchObjectStrategy implements ObjectStrategy +{ + + private static final byte[] EMPTY_BYTES = new byte[] {}; + + @Override + public int compare(final DoublesSketch s1, final DoublesSketch s2) + { + return DoublesSketchAggregatorFactory.COMPARATOR.compare(s1, s2); + } + + @Override + public DoublesSketch fromByteBuffer(final ByteBuffer buffer, final int numBytes) + { + if (numBytes == 0) { + return DoublesSketchOperations.EMPTY_SKETCH; + } + return DoublesSketch.wrap(Memory.wrap(buffer).region(buffer.position(), numBytes)); + } + + @Override + public Class getClazz() + { + return DoublesSketch.class; + } + + @Override + public byte[] toBytes(final DoublesSketch sketch) + { + if (sketch == null || sketch.isEmpty()) { + return EMPTY_BYTES; + } + return sketch.toByteArray(true); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java new file mode 100644 index 000000000000..5a0f70e6b76a --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchOperations.java @@ -0,0 +1,59 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import org.apache.commons.codec.binary.Base64; + +import com.google.common.base.Charsets; +import com.yahoo.memory.Memory; +import com.yahoo.sketches.quantiles.DoublesSketch; + +import io.druid.java.util.common.ISE; + +public class DoublesSketchOperations +{ + + public static final DoublesSketch EMPTY_SKETCH = DoublesSketch.builder().build(); + + public static DoublesSketch deserialize(final Object serializedSketch) + { + if (serializedSketch instanceof String) { + return deserializeFromBase64EncodedString((String) serializedSketch); + } else if (serializedSketch instanceof byte[]) { + return deserializeFromByteArray((byte[]) serializedSketch); + } else if (serializedSketch instanceof DoublesSketch) { + return (DoublesSketch) serializedSketch; + } + throw new ISE( + "Object is not of a type that can be deserialized to a quantiles DoublsSketch: " + + serializedSketch.getClass()); + } + + public static DoublesSketch deserializeFromBase64EncodedString(final String str) + { + return deserializeFromByteArray(Base64.decodeBase64(str.getBytes(Charsets.UTF_8))); + } + + public static DoublesSketch deserializeFromByteArray(final byte[] data) + { + return DoublesSketch.wrap(Memory.wrap(data)); + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java new file mode 100644 index 000000000000..015d7787d43c --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java @@ -0,0 +1,153 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +import io.druid.java.util.common.IAE; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorUtil; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.cache.CacheKeyBuilder; + +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchToHistogramPostAggregator implements PostAggregator +{ + + private final String name; + private final PostAggregator field; + private final double[] splitPoints; + + @JsonCreator + public DoublesSketchToHistogramPostAggregator( + @JsonProperty("name") final String name, + @JsonProperty("field") final PostAggregator field, + @JsonProperty("splitPoints") final double[] splitPoints) + { + this.name = Preconditions.checkNotNull(name, "name is null"); + this.field = Preconditions.checkNotNull(field, "field is null"); + this.splitPoints = Preconditions.checkNotNull(splitPoints, "array of split points is null"); + } + + @Override + public Object compute(final Map combinedAggregators) + { + final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); + final double[] histogram = sketch.getPMF(splitPoints); + for (int i = 0; i < histogram.length; i++) { + histogram[i] *= sketch.getN(); + } + return histogram; + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public PostAggregator getField() + { + return field; + } + + @JsonProperty + public double[] getSplitPoints() + { + return splitPoints; + } + + @Override + public Comparator getComparator() + { + throw new IAE("Comparing histograms is not supported"); + } + + @Override + public Set getDependentFields() + { + return field.getDependentFields(); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "name='" + name + '\'' + + ", field=" + field + + ", splitPoints=" + Arrays.toString(splitPoints) + + "}"; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DoublesSketchToHistogramPostAggregator that = (DoublesSketchToHistogramPostAggregator) o; + if (!name.equals(that.name)) { + return false; + } + if (!Arrays.equals(splitPoints, that.splitPoints)) { + return false; + } + return field.equals(that.field); + } + + @Override + public int hashCode() + { + int hashCode = name.hashCode() * 31 + field.hashCode(); + hashCode = hashCode * 31 + Arrays.hashCode(splitPoints); + return hashCode; + } + + @Override + public byte[] getCacheKey() + { + final CacheKeyBuilder builder = new CacheKeyBuilder( + AggregatorUtil.QUANTILES_DOUBLES_SKETCH_TO_HISTOGRAM_CACHE_TYPE_ID).appendCacheable(field); + for (final double value: splitPoints) { + builder.appendDouble(value); + } + return builder.build(); + } + + @Override + public PostAggregator decorate(final Map map) + { + return this; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java new file mode 100644 index 000000000000..a9df26b57743 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java @@ -0,0 +1,149 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Comparator; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.primitives.Doubles; + +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorUtil; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.cache.CacheKeyBuilder; + +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchToQuantilePostAggregator implements PostAggregator +{ + + private final String name; + private final PostAggregator field; + private final double fraction; + + @JsonCreator + public DoublesSketchToQuantilePostAggregator( + @JsonProperty("name") final String name, + @JsonProperty("field") final PostAggregator field, + @JsonProperty("fraction") final double fraction) + { + this.name = Preconditions.checkNotNull(name, "name is null"); + this.field = Preconditions.checkNotNull(field, "field is null"); + this.fraction = fraction; + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public PostAggregator getField() + { + return field; + } + + @JsonProperty + public double getFraction() + { + return fraction; + } + + @Override + public Object compute(final Map combinedAggregators) + { + final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); + return sketch.getQuantile(fraction); + } + + @Override + public Comparator getComparator() + { + return new Comparator() + { + @Override + public int compare(final Double a, final Double b) + { + return Doubles.compare(a, b); + } + }; + } + + @Override + public Set getDependentFields() + { + return field.getDependentFields(); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "name='" + name + '\'' + + ", field=" + field + + ", fraction=" + fraction + + "}"; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DoublesSketchToQuantilePostAggregator that = (DoublesSketchToQuantilePostAggregator) o; + if (!name.equals(that.name)) { + return false; + } + if (fraction != that.fraction) { + return false; + } + return field.equals(that.field); + } + + @Override + public int hashCode() + { + return (name.hashCode() * 31 + field.hashCode()) * 31 + Double.hashCode(fraction); + } + + @Override + public byte[] getCacheKey() + { + return new CacheKeyBuilder(AggregatorUtil.QUANTILES_DOUBLES_SKETCH_TO_QUANTILE_CACHE_TYPE_ID) + .appendCacheable(field).appendDouble(fraction).build(); + } + + @Override + public PostAggregator decorate(final Map map) + { + return this; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java new file mode 100644 index 000000000000..45674dcf7f71 --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java @@ -0,0 +1,147 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +import io.druid.java.util.common.IAE; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorUtil; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.cache.CacheKeyBuilder; + +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchToQuantilesPostAggregator implements PostAggregator +{ + + private final String name; + private final PostAggregator field; + private final double[] fractions; + + @JsonCreator + public DoublesSketchToQuantilesPostAggregator( + @JsonProperty("name") final String name, + @JsonProperty("field") final PostAggregator field, + @JsonProperty("fractions") final double[] fractions) + { + this.name = Preconditions.checkNotNull(name, "name is null"); + this.field = Preconditions.checkNotNull(field, "field is null"); + this.fractions = Preconditions.checkNotNull(fractions, "array of fractions is null"); + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public PostAggregator getField() + { + return field; + } + + @JsonProperty + public double[] getFractions() + { + return fractions; + } + + @Override + public Object compute(final Map combinedAggregators) + { + final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); + return sketch.getQuantiles(fractions); + } + + @Override + public Comparator getComparator() + { + throw new IAE("Comparing arrays of quantiles is not supported"); + } + + @Override + public Set getDependentFields() + { + return field.getDependentFields(); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + "{" + + "name='" + name + '\'' + + ", field=" + field + + ", fractions=" + Arrays.toString(fractions) + + "}"; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DoublesSketchToQuantilesPostAggregator that = (DoublesSketchToQuantilesPostAggregator) o; + if (!name.equals(that.name)) { + return false; + } + if (!Arrays.equals(fractions, that.fractions)) { + return false; + } + return field.equals(that.field); + } + + @Override + public int hashCode() + { + return (name.hashCode() * 31 + field.hashCode()) * 31 + Arrays.hashCode(fractions); + } + + @Override + public byte[] getCacheKey() + { + final CacheKeyBuilder builder = new CacheKeyBuilder( + AggregatorUtil.QUANTILES_DOUBLES_SKETCH_TO_QUANTILES_CACHE_TYPE_ID).appendCacheable(field); + for (final double value: fractions) { + builder.appendDouble(value); + } + return builder.build(); + } + + @Override + public PostAggregator decorate(final Map map) + { + return this; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java new file mode 100644 index 000000000000..204b80cef50c --- /dev/null +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java @@ -0,0 +1,130 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.util.Comparator; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +import io.druid.java.util.common.IAE; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorUtil; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.cache.CacheKeyBuilder; + +import com.yahoo.sketches.quantiles.DoublesSketch; + +public class DoublesSketchToStringPostAggregator implements PostAggregator +{ + + private final String name; + private final PostAggregator field; + + @JsonCreator + public DoublesSketchToStringPostAggregator( + @JsonProperty("name") final String name, + @JsonProperty("field") final PostAggregator field) + { + this.name = Preconditions.checkNotNull(name, "name is null"); + this.field = Preconditions.checkNotNull(field, "field is null"); + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public PostAggregator getField() + { + return field; + } + + @Override + public Object compute(final Map combinedAggregators) + { + final DoublesSketch sketch = (DoublesSketch) field.compute(combinedAggregators); + return sketch.toString(); + } + + @Override + public Comparator getComparator() + { + throw new IAE("Comparing sketch summaries is not supported"); + } + + @Override + public Set getDependentFields() + { + return field.getDependentFields(); + } + + @Override + public String toString() + { + return this.getClass().getSimpleName() + "{" + + "name='" + name + '\'' + + ", field=" + field + + "}"; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DoublesSketchToStringPostAggregator that = (DoublesSketchToStringPostAggregator) o; + if (!name.equals(that.name)) { + return false; + } + return field.equals(that.field); + } + + @Override + public int hashCode() + { + return (name.hashCode() * 31 + field.hashCode()) * 31; + } + + @Override + public byte[] getCacheKey() + { + final CacheKeyBuilder builder = new CacheKeyBuilder( + AggregatorUtil.QUANTILES_DOUBLES_SKETCH_TO_STRING_CACHE_TYPE_ID).appendCacheable(field); + return builder.build(); + } + + @Override + public PostAggregator decorate(final Map map) + { + return this; + } + +} diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java index e0c1ba03bab8..6be52eb405ad 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java @@ -21,10 +21,10 @@ import com.yahoo.sketches.theta.Sketch; import io.druid.data.input.InputRow; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.serde.ComplexColumnPartSupplier; import io.druid.segment.serde.ComplexMetricExtractor; @@ -80,9 +80,9 @@ public ObjectStrategy getObjectStrategy() } @Override - public GenericColumnSerializer getSerializer(IOPeon peon, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { - return LargeColumnSupportedComplexColumnSerializer.create(peon, column, this.getObjectStrategy()); + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } } diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java index 4b613d848a6c..41e3d8ee410a 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java @@ -21,13 +21,14 @@ import com.yahoo.memory.Memory; import com.yahoo.memory.WritableMemory; +import com.yahoo.sketches.Family; import com.yahoo.sketches.theta.CompactSketch; import com.yahoo.sketches.theta.Sketch; import com.yahoo.sketches.theta.Union; /** */ -public class SynchronizedUnion implements Union +public class SynchronizedUnion extends Union { private final Union delegate; @@ -67,18 +68,21 @@ public synchronized void update(String datum) } @Override + @SuppressWarnings("ParameterPackage") public synchronized void update(byte[] data) { delegate.update(data); } @Override + @SuppressWarnings("ParameterPackage") public synchronized void update(int[] data) { delegate.update(data); } @Override + @SuppressWarnings("ParameterPackage") public synchronized void update(char[] chars) { delegate.update(chars); @@ -119,4 +123,11 @@ public synchronized boolean isSameResource(Memory mem) { return delegate.isSameResource(mem); } + + @Override + public synchronized Family getFamily() + { + return delegate.getFamily(); + } + } diff --git a/extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule index efd10f3a7e78..dfc5b69e00fb 100644 --- a/extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule +++ b/extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule @@ -1,2 +1,3 @@ io.druid.query.aggregation.datasketches.theta.SketchModule io.druid.query.aggregation.datasketches.theta.oldapi.OldApiSketchModule +io.druid.query.aggregation.datasketches.quantiles.DoublesSketchModule diff --git a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java new file mode 100644 index 000000000000..899c8392e4fd --- /dev/null +++ b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java @@ -0,0 +1,419 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import io.druid.data.input.Row; +import io.druid.initialization.DruidModule; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.java.util.common.granularity.Granularities; +import io.druid.java.util.common.guava.Sequence; +import io.druid.java.util.common.guava.Sequences; +import io.druid.query.aggregation.AggregationTestHelper; +import io.druid.query.groupby.GroupByQueryConfig; +import io.druid.query.groupby.GroupByQueryRunnerTest; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@RunWith(Parameterized.class) +public class DoublesSketchAggregatorTest +{ + + private final AggregationTestHelper helper; + private final AggregationTestHelper timeSeriesHelper; + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + public DoublesSketchAggregatorTest(final GroupByQueryConfig config) + { + DruidModule module = new DoublesSketchModule(); + module.configure(null); + helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( + module.getJacksonModules(), config, tempFolder); + timeSeriesHelper = AggregationTestHelper.createTimeseriesQueryAggregationTestHelper( + module.getJacksonModules(), + tempFolder); + } + + @Parameterized.Parameters(name = "{0}") + public static Collection constructorFeeder() + { + final List constructors = Lists.newArrayList(); + for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) { + constructors.add(new Object[] {config}); + } + return constructors; + } + + // this is to test Json properties and equals + @Test + public void serializeDeserializeFactoryWithFieldName() throws Exception + { + ObjectMapper objectMapper = new DefaultObjectMapper(); + DoublesSketchAggregatorFactory factory = new DoublesSketchAggregatorFactory("name", "filedName", 128); + + DoublesSketchAggregatorFactory other = objectMapper.readValue( + objectMapper.writeValueAsString(factory), + DoublesSketchAggregatorFactory.class); + + Assert.assertEquals(factory, other); + } + + @Test + public void ingestingSketches() throws Exception + { + Sequence seq = helper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_sketch_data.tsv").getFile()), + String.join("\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"product\"],", + " \"dimensionExclusions\": [],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"product\", \"sketch\"]", + " }", + "}"), + String.join("\n", + "[", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"k\": 128},", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"non_existent_sketch\", \"fieldName\": \"non_existent_sketch\", \"k\": 128}", + "]"), + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join("\n", + "{", + " \"queryType\": \"groupBy\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"dimensions\": [],", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"k\": 128},", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"non_existent_sketch\", \"fieldName\": \"non_existent_sketch\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + Row row = results.get(0); + + Object nonExistentSketchObject = row.getRaw("non_existent_sketch"); + Assert.assertTrue(nonExistentSketchObject instanceof Long); + long nonExistentSketchValue = (long) nonExistentSketchObject; + Assert.assertEquals(0, nonExistentSketchValue); + + Object sketchObject = row.getRaw("sketch"); + Assert.assertTrue(sketchObject instanceof Long); + long sketchValue = (long) sketchObject; + Assert.assertEquals(400, sketchValue); + + // post agg + Object quantilesObject = row.getRaw("quantiles"); + Assert.assertTrue(quantilesObject instanceof double[]); + double[] quantiles = (double[]) quantilesObject; + Assert.assertEquals(0, quantiles[0], 0.05); // min value + Assert.assertEquals(0.5, quantiles[1], 0.05); // median value + Assert.assertEquals(1, quantiles[2], 0.05); // max value + + // post agg + Object histogramObject = row.getRaw("histogram"); + Assert.assertTrue(histogramObject instanceof double[]); + double[] histogram = (double[]) histogramObject; + for (final double bin: histogram) { + Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly + // distributed into 4 bins + } + } + + @Test + public void buildingSketchesAtIngestionTime() throws Exception + { + Sequence seq = helper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), + String.join("\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"product\"],", + " \"dimensionExclusions\": [ \"sequenceNumber\"],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", + " }", + "}"), + "[{\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}]", + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join("\n", + "{", + " \"queryType\": \"groupBy\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"dimensions\": [],", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"k\": 128},", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"non_existent_sketch\", \"fieldName\": \"non_existent_sketch\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + Row row = results.get(0); + + Object sketchObject = row.getRaw("sketch"); + Assert.assertTrue(sketchObject instanceof Long); + long sketchValue = (long) sketchObject; + Assert.assertEquals(400, sketchValue); + + // post agg + Object quantilesObject = row.getRaw("quantiles"); + Assert.assertTrue(quantilesObject instanceof double[]); + double[] quantiles = (double[]) quantilesObject; + Assert.assertEquals(0, quantiles[0], 0.05); // min value + Assert.assertEquals(0.5, quantiles[1], 0.05); // median value + Assert.assertEquals(1, quantiles[2], 0.05); // max value + + // post agg + Object histogramObject = row.getRaw("histogram"); + Assert.assertTrue(histogramObject instanceof double[]); + double[] histogram = (double[]) histogramObject; + Assert.assertEquals(4, histogram.length); + for (final double bin: histogram) { + Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly + // distributed into 4 bins + } + } + + @Test + public void buildingSketchesAtQueryTime() throws Exception + { + Sequence seq = helper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), + String.join("\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"sequenceNumber\", \"product\"],", + " \"dimensionExclusions\": [],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", + " }", + "}"), + "[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]", + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join("\n", + "{", + " \"queryType\": \"groupBy\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"dimensions\": [],", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantile\", \"name\": \"quantile\", \"fraction\": 0.5, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + Row row = results.get(0); + + Object sketchObject = row.getRaw("sketch"); + Assert.assertTrue(sketchObject instanceof Long); + long sketchValue = (long) sketchObject; + Assert.assertEquals(400, sketchValue); + + // post agg + Object quantileObject = row.getRaw("quantile"); + Assert.assertTrue(quantileObject instanceof Double); + Assert.assertEquals(0.5, (double) quantileObject, 0.05); // median value + + // post agg + Object quantilesObject = row.getRaw("quantiles"); + Assert.assertTrue(quantilesObject instanceof double[]); + double[] quantiles = (double[]) quantilesObject; + Assert.assertEquals(0, quantiles[0], 0.05); // min value + Assert.assertEquals(0.5, quantiles[1], 0.05); // median value + Assert.assertEquals(1, quantiles[2], 0.05); // max value + + // post agg + Object histogramObject = row.getRaw("histogram"); + Assert.assertTrue(histogramObject instanceof double[]); + double[] histogram = (double[]) histogramObject; + for (final double bin: histogram) { + Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly + // distributed into 4 bins + } + } + + @Test + public void QueryingDataWithFieldNameValueAsFloatInsteadOfSketch() throws Exception + { + Sequence seq = helper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), + String.join( + "\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"sequenceNumber\", \"product\"],", + " \"dimensionExclusions\": [],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", + " }", + "}"), + "[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]", + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join( + "\n", + "{", + " \"queryType\": \"groupBy\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"dimensions\": [],", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantile\", \"name\": \"quantile\", \"fraction\": 0.5, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + Row row = results.get(0); + + Object sketchObject = row.getRaw("sketch"); + Assert.assertTrue(sketchObject instanceof Long); + long sketchValue = (long) sketchObject; + Assert.assertEquals(400, sketchValue); + + // post agg + Object quantileObject = row.getRaw("quantile"); + Assert.assertTrue(quantileObject instanceof Double); + Assert.assertEquals(0.5, (double) quantileObject, 0.05); // median value + + // post agg + Object quantilesObject = row.getRaw("quantiles"); + Assert.assertTrue(quantilesObject instanceof double[]); + double[] quantiles = (double[]) quantilesObject; + Assert.assertEquals(0, quantiles[0], 0.05); // min value + Assert.assertEquals(0.5, quantiles[1], 0.05); // median value + Assert.assertEquals(1, quantiles[2], 0.05); // max value + + // post agg + Object histogramObject = row.getRaw("histogram"); + Assert.assertTrue(histogramObject instanceof double[]); + double[] histogram = (double[]) histogramObject; + for (final double bin: histogram) { + Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly + // distributed into 4 bins + } + } + + @Test + public void TimeSeriesQueryInputAsFloat() throws Exception + { + Sequence seq = timeSeriesHelper.createIndexAndRunQueryOnSegment( + new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), + String.join( + "\n", + "{", + " \"type\": \"string\",", + " \"parseSpec\": {", + " \"format\": \"tsv\",", + " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", + " \"dimensionsSpec\": {", + " \"dimensions\": [\"sequenceNumber\", \"product\"],", + " \"dimensionExclusions\": [],", + " \"spatialDimensions\": []", + " },", + " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", + " }", + "}"), + "[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]", + 0, // minTimestamp + Granularities.NONE, + 10, // maxRowCount + String.join( + "\n", + "{", + " \"queryType\": \"timeseries\",", + " \"dataSource\": \"test_datasource\",", + " \"granularity\": \"ALL\",", + " \"aggregations\": [", + " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}", + " ],", + " \"postAggregations\": [", + " {\"type\": \"quantilesDoublesSketchToQuantile\", \"name\": \"quantile1\", \"fraction\": 0.5, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles1\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", + " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram1\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", + " ],", + " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", + "}")); + List results = Sequences.toList(seq, new ArrayList()); + Assert.assertEquals(1, results.size()); + } +} diff --git a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java new file mode 100644 index 000000000000..f5c11a056555 --- /dev/null +++ b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/GenerateTestData.java @@ -0,0 +1,72 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation.datasketches.quantiles; + +import java.io.BufferedWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Random; + +import org.apache.commons.codec.binary.Base64; + +import com.yahoo.sketches.quantiles.UpdateDoublesSketch; + +// This is used for generating test data for DoublesSketchAggregatorTest +public class GenerateTestData +{ + + public static void main(String[] args) throws Exception + { + Path buildPath = FileSystems.getDefault().getPath("doubles_build_data.tsv"); + Path sketchPath = FileSystems.getDefault().getPath("doubles_sketch_data.tsv"); + BufferedWriter buildData = Files.newBufferedWriter(buildPath, StandardCharsets.UTF_8); + BufferedWriter sketchData = Files.newBufferedWriter(sketchPath, StandardCharsets.UTF_8); + Random rand = new Random(); + int sequenceNumber = 0; + for (int i = 0; i < 20; i++) { + int product = rand.nextInt(10); + UpdateDoublesSketch sketch = UpdateDoublesSketch.builder().build(); + for (int j = 0; j < 20; j++) { + double value = rand.nextDouble(); + buildData.write("2016010101"); + buildData.write('\t'); + buildData.write(Integer.toString(sequenceNumber)); // dimension with unique numbers for ingesting raw data + buildData.write('\t'); + buildData.write(Integer.toString(product)); // product dimension + buildData.write('\t'); + buildData.write(Double.toString(value)); + buildData.newLine(); + sketch.update(value); + sequenceNumber++; + } + sketchData.write("2016010101"); + sketchData.write('\t'); + sketchData.write(Integer.toString(product)); // product dimension + sketchData.write('\t'); + sketchData.write(Base64.encodeBase64String(sketch.toByteArray(true))); + sketchData.newLine(); + } + buildData.close(); + sketchData.close(); + } + +} diff --git a/extensions-core/datasketches/src/test/resources/quantiles/doubles_build_data.tsv b/extensions-core/datasketches/src/test/resources/quantiles/doubles_build_data.tsv new file mode 100644 index 000000000000..bb59faf3da8b --- /dev/null +++ b/extensions-core/datasketches/src/test/resources/quantiles/doubles_build_data.tsv @@ -0,0 +1,400 @@ +2016010101 0 0 0.6529403005319299 +2016010101 1 0 0.9270214958987323 +2016010101 2 0 0.6383273609981486 +2016010101 3 0 0.8088289215633632 +2016010101 4 0 0.8163864917598281 +2016010101 5 0 0.38484848588530784 +2016010101 6 0 0.7690020468986823 +2016010101 7 0 0.6212078833139824 +2016010101 8 0 0.4915825094949512 +2016010101 9 0 0.688004059332008 +2016010101 10 0 0.2536908275250508 +2016010101 11 0 0.6618435914290263 +2016010101 12 0 0.7892773595797635 +2016010101 13 0 0.08857624134076048 +2016010101 14 0 0.11992633801904151 +2016010101 15 0 0.4959192800105586 +2016010101 16 0 0.5564893557708243 +2016010101 17 0 0.7755547456799993 +2016010101 18 0 0.06420706406984311 +2016010101 19 0 0.23085639094262378 +2016010101 20 7 0.012013916725163498 +2016010101 21 7 0.34077219818209503 +2016010101 22 7 0.8445966884204918 +2016010101 23 7 0.6466142718287953 +2016010101 24 7 0.43959032391415487 +2016010101 25 7 0.7768829233737787 +2016010101 26 7 0.5899544206136442 +2016010101 27 7 0.017782361911801825 +2016010101 28 7 0.5431916165782864 +2016010101 29 7 0.8218253174439416 +2016010101 30 7 0.6372788284951859 +2016010101 31 7 0.41403671834680933 +2016010101 32 7 0.042508330730374855 +2016010101 33 7 0.7416290691530969 +2016010101 34 7 0.6990557213726277 +2016010101 35 7 0.6302154208823348 +2016010101 36 7 0.021053567154993402 +2016010101 37 7 0.770280353784988 +2016010101 38 7 0.08205576978448703 +2016010101 39 7 0.2049660800682488 +2016010101 40 5 0.08129304678049831 +2016010101 41 5 0.17754747271638005 +2016010101 42 5 0.8441702357096768 +2016010101 43 5 0.9060464737257796 +2016010101 44 5 0.5970595512785409 +2016010101 45 5 0.843859346312315 +2016010101 46 5 0.1649847892987305 +2016010101 47 5 0.5279903496999094 +2016010101 48 5 0.08758749830556767 +2016010101 49 5 0.6088480522002063 +2016010101 50 5 0.31079133043670004 +2016010101 51 5 0.43062105356651226 +2016010101 52 5 0.8542989852099488 +2016010101 53 5 0.42443162807834045 +2016010101 54 5 0.5020327054358468 +2016010101 55 5 0.36453920012074237 +2016010101 56 5 0.9884597580348689 +2016010101 57 5 0.3770559586575706 +2016010101 58 5 0.5989237303385875 +2016010101 59 5 0.9926342802399872 +2016010101 60 4 0.7813961047849703 +2016010101 61 4 0.062171533805525425 +2016010101 62 4 0.5284977503473608 +2016010101 63 4 0.5924687065581794 +2016010101 64 4 0.06305234223879275 +2016010101 65 4 0.4959562731747129 +2016010101 66 4 0.6336733165353365 +2016010101 67 4 0.48860263540869875 +2016010101 68 4 0.9387610528974851 +2016010101 69 4 0.3391271652731308 +2016010101 70 4 0.5962837638971421 +2016010101 71 4 0.9190447294921896 +2016010101 72 4 0.33082943548872534 +2016010101 73 4 0.6236359023672029 +2016010101 74 4 0.27134427542016615 +2016010101 75 4 0.11665530238761901 +2016010101 76 4 0.10469260335277608 +2016010101 77 4 0.6824658847771211 +2016010101 78 4 0.6131047630496756 +2016010101 79 4 0.9838171536972515 +2016010101 80 4 0.7484669110852756 +2016010101 81 4 0.797620888697219 +2016010101 82 4 0.7166673353657907 +2016010101 83 4 0.46968710353176557 +2016010101 84 4 0.3998491199643106 +2016010101 85 4 0.6314883585976869 +2016010101 86 4 0.8305617875577815 +2016010101 87 4 0.6867651870284084 +2016010101 88 4 0.9961677044887979 +2016010101 89 4 0.19745766301180412 +2016010101 90 4 0.2737652043079263 +2016010101 91 4 0.2954503444695358 +2016010101 92 4 0.6191902196833489 +2016010101 93 4 0.6828058006233482 +2016010101 94 4 0.7967115641510757 +2016010101 95 4 0.5485460823820962 +2016010101 96 4 0.4278132830938558 +2016010101 97 4 0.32194908458166194 +2016010101 98 4 0.07094920295725238 +2016010101 99 4 0.4351839393889565 +2016010101 100 1 0.6160833396611648 +2016010101 101 1 0.4652667787803648 +2016010101 102 1 0.5026953463132913 +2016010101 103 1 0.4103237191034753 +2016010101 104 1 0.3298554666697301 +2016010101 105 1 0.16907537273919138 +2016010101 106 1 0.6945260598989513 +2016010101 107 1 0.917138530496438 +2016010101 108 1 0.8810129148605083 +2016010101 109 1 0.11845626048380542 +2016010101 110 1 0.8848971155827816 +2016010101 111 1 0.9969103769603667 +2016010101 112 1 0.06274198529295416 +2016010101 113 1 0.2923616769686519 +2016010101 114 1 0.12621083638328634 +2016010101 115 1 0.9655188575577313 +2016010101 116 1 0.6074995164352884 +2016010101 117 1 0.5501887988201414 +2016010101 118 1 0.9406914128003497 +2016010101 119 1 0.03264873659277656 +2016010101 120 6 0.004852543443656487 +2016010101 121 6 0.11161194329252788 +2016010101 122 6 0.9403527002796559 +2016010101 123 6 0.8951866979503953 +2016010101 124 6 0.07629846897033454 +2016010101 125 6 0.9898485014275873 +2016010101 126 6 0.42827377712188075 +2016010101 127 6 0.4274796777951825 +2016010101 128 6 0.5569522946332676 +2016010101 129 6 0.028195121559112635 +2016010101 130 6 0.8599127909482382 +2016010101 131 6 0.3516112293128607 +2016010101 132 6 0.3888868189342449 +2016010101 133 6 0.644589126160206 +2016010101 134 6 0.7398741071492928 +2016010101 135 6 0.1998479248216123 +2016010101 136 6 0.8803215884594476 +2016010101 137 6 0.7079531966558515 +2016010101 138 6 0.7904290564015343 +2016010101 139 6 0.475671788742007 +2016010101 140 3 0.034708334899357096 +2016010101 141 3 0.4134637419532796 +2016010101 142 3 0.9757934592902832 +2016010101 143 3 0.37422347371609666 +2016010101 144 3 0.5904996168737154 +2016010101 145 3 0.5883259679727514 +2016010101 146 3 0.3380286015499171 +2016010101 147 3 0.42174393035143043 +2016010101 148 3 0.4764900074141757 +2016010101 149 3 0.01864239537224921 +2016010101 150 3 0.9124007087743986 +2016010101 151 3 0.8951275235699193 +2016010101 152 3 0.7037272142266654 +2016010101 153 3 0.5685506209266902 +2016010101 154 3 0.4104883958833594 +2016010101 155 3 0.7794005551450208 +2016010101 156 3 0.2879354697088996 +2016010101 157 3 0.5243215707259823 +2016010101 158 3 0.22238840286136063 +2016010101 159 3 0.11336472553284738 +2016010101 160 4 0.9800770037725316 +2016010101 161 4 0.7628237317889158 +2016010101 162 4 0.5355335935170453 +2016010101 163 4 0.9676939330565402 +2016010101 164 4 0.657825753108034 +2016010101 165 4 0.9175328548944673 +2016010101 166 4 0.6834666043257283 +2016010101 167 4 0.08580759367942314 +2016010101 168 4 0.3134740602060899 +2016010101 169 4 0.3218818254752742 +2016010101 170 4 0.6119297354994999 +2016010101 171 4 0.07086832750773142 +2016010101 172 4 0.2700864307032772 +2016010101 173 4 0.7497315076673637 +2016010101 174 4 0.4959921300968493 +2016010101 175 4 0.09294825796093753 +2016010101 176 4 0.4954515904444161 +2016010101 177 4 0.8820366880191506 +2016010101 178 4 0.17978298283728522 +2016010101 179 4 0.05259679741524781 +2016010101 180 5 0.4711892966981096 +2016010101 181 5 0.5965662941715105 +2016010101 182 5 0.4775201668966973 +2016010101 183 5 0.05084576687030873 +2016010101 184 5 0.16680660677593928 +2016010101 185 5 0.9342287333653685 +2016010101 186 5 0.8153161893769392 +2016010101 187 5 0.9362517669519288 +2016010101 188 5 0.10865218471840699 +2016010101 189 5 0.44665378915111065 +2016010101 190 5 0.8804454791937898 +2016010101 191 5 0.20666928346935398 +2016010101 192 5 0.7052479677101612 +2016010101 193 5 0.5006205470200923 +2016010101 194 5 0.23220501028575968 +2016010101 195 5 0.11776507130391467 +2016010101 196 5 0.592011744069295 +2016010101 197 5 0.7089191450076786 +2016010101 198 5 0.7269340552231702 +2016010101 199 5 0.7049554871226075 +2016010101 200 1 0.44078367400761076 +2016010101 201 1 0.7715264806037321 +2016010101 202 1 0.10151701902103971 +2016010101 203 1 0.661891806135609 +2016010101 204 1 0.23095745116331567 +2016010101 205 1 0.46625278601359255 +2016010101 206 1 0.5912486124707177 +2016010101 207 1 0.963946871892115 +2016010101 208 1 0.8172596270687692 +2016010101 209 1 0.05745699928199144 +2016010101 210 1 0.40612684342877337 +2016010101 211 1 0.6330844777969608 +2016010101 212 1 0.3148973406065705 +2016010101 213 1 0.23230462811318175 +2016010101 214 1 0.9960772952945196 +2016010101 215 1 0.4581376339786414 +2016010101 216 1 0.7181494575770677 +2016010101 217 1 0.04277917580280799 +2016010101 218 1 0.11137419446625674 +2016010101 219 1 0.014716278313423037 +2016010101 220 2 0.8988603727313186 +2016010101 221 2 0.8192124226306603 +2016010101 222 2 0.9304683598956597 +2016010101 223 2 0.4375546733938238 +2016010101 224 2 0.7676359685332207 +2016010101 225 2 0.30977859822027964 +2016010101 226 2 0.008595955287459267 +2016010101 227 2 0.6790605343724216 +2016010101 228 2 0.36949588946147993 +2016010101 229 2 0.3826798435706562 +2016010101 230 2 0.13836513167087128 +2016010101 231 2 0.4451570472364902 +2016010101 232 2 0.8944067771338549 +2016010101 233 2 0.6068095655362902 +2016010101 234 2 0.7084870042917992 +2016010101 235 2 0.5867363290655241 +2016010101 236 2 0.6903863088381504 +2016010101 237 2 0.30984947936089124 +2016010101 238 2 0.31561088279452665 +2016010101 239 2 0.006286479849849758 +2016010101 240 5 0.34397466439693725 +2016010101 241 5 0.052476003295899964 +2016010101 242 5 0.726106045184451 +2016010101 243 5 0.01559115401009159 +2016010101 244 5 0.9219270739836661 +2016010101 245 5 0.5147917330760431 +2016010101 246 5 0.41919804470784205 +2016010101 247 5 0.4145101775865617 +2016010101 248 5 0.34153038022995796 +2016010101 249 5 0.9503817180587767 +2016010101 250 5 0.6958354849389804 +2016010101 251 5 0.46000811480536297 +2016010101 252 5 0.18379911670616378 +2016010101 253 5 0.20973108758556713 +2016010101 254 5 0.5979201603287885 +2016010101 255 5 0.5552419362393491 +2016010101 256 5 0.10996555307297629 +2016010101 257 5 0.3591453585622102 +2016010101 258 5 0.06098055111386691 +2016010101 259 5 0.5227270267924988 +2016010101 260 0 0.8492702312836989 +2016010101 261 0 0.5941242001151825 +2016010101 262 0 0.6840733026822607 +2016010101 263 0 0.8109777000249937 +2016010101 264 0 0.8599286045013937 +2016010101 265 0 0.7828806670746145 +2016010101 266 0 0.8102260971867188 +2016010101 267 0 0.38306094770114385 +2016010101 268 0 0.7093609268723879 +2016010101 269 0 0.4806583187577358 +2016010101 270 0 0.5766489331365172 +2016010101 271 0 0.7565067278238041 +2016010101 272 0 0.8262768908267573 +2016010101 273 0 0.7951015619138146 +2016010101 274 0 0.1938448910588796 +2016010101 275 0 0.8884608583839426 +2016010101 276 0 0.7046203516594505 +2016010101 277 0 0.5951074760704175 +2016010101 278 0 0.38207409719784036 +2016010101 279 0 0.2445271560830221 +2016010101 280 7 0.6032919624054952 +2016010101 281 7 0.1473220747987144 +2016010101 282 7 0.38396643099307604 +2016010101 283 7 0.4431561135554619 +2016010101 284 7 0.896578318093225 +2016010101 285 7 0.6729206122043515 +2016010101 286 7 0.8498821349478478 +2016010101 287 7 0.48231924024179784 +2016010101 288 7 0.005379480238994816 +2016010101 289 7 0.8017936717647264 +2016010101 290 7 0.08193232952990348 +2016010101 291 7 0.3422943366454193 +2016010101 292 7 0.6081556855207957 +2016010101 293 7 0.641193222941943 +2016010101 294 7 0.3716858024654186 +2016010101 295 7 0.0011169303830090849 +2016010101 296 7 0.4698784438339285 +2016010101 297 7 0.958198841287214 +2016010101 298 7 0.730945048929339 +2016010101 299 7 0.1858601884405512 +2016010101 300 5 0.1020825694779407 +2016010101 301 5 0.5742385074938443 +2016010101 302 5 0.9846817584978909 +2016010101 303 5 0.3858694391491331 +2016010101 304 5 0.9822246873202894 +2016010101 305 5 0.39822015482143314 +2016010101 306 5 0.6575924137957005 +2016010101 307 5 0.02359557062746842 +2016010101 308 5 0.42059510563039115 +2016010101 309 5 0.5970764856116284 +2016010101 310 5 0.2817399870096221 +2016010101 311 5 0.5334091165258412 +2016010101 312 5 0.31199853410796585 +2016010101 313 5 0.3156991306990594 +2016010101 314 5 0.9560285139855889 +2016010101 315 5 0.7846951771498516 +2016010101 316 5 0.009731486767097897 +2016010101 317 5 0.22625857375026215 +2016010101 318 5 0.8580955944724618 +2016010101 319 5 0.9622008926137687 +2016010101 320 5 0.023872302930851297 +2016010101 321 5 0.3580981601151092 +2016010101 322 5 0.9120442264954038 +2016010101 323 5 0.5968491989965334 +2016010101 324 5 0.5028516120506729 +2016010101 325 5 0.30590552314314 +2016010101 326 5 0.5566430714368423 +2016010101 327 5 0.6441099124064397 +2016010101 328 5 0.8765287851559298 +2016010101 329 5 0.38405928947408385 +2016010101 330 5 0.29654203975364 +2016010101 331 5 0.3606921959261904 +2016010101 332 5 0.9617038824842609 +2016010101 333 5 0.3103700669261584 +2016010101 334 5 0.4935170174690311 +2016010101 335 5 0.34757561267296444 +2016010101 336 5 0.1236918485545484 +2016010101 337 5 0.24925258973306597 +2016010101 338 5 0.4104821367672965 +2016010101 339 5 0.3621850216936935 +2016010101 340 6 0.3816099229918041 +2016010101 341 6 0.9496667754823915 +2016010101 342 6 0.5594605720642025 +2016010101 343 6 0.8537860901562698 +2016010101 344 6 0.74787202967909 +2016010101 345 6 0.29699361421249604 +2016010101 346 6 0.035943527086235605 +2016010101 347 6 0.20106098029261277 +2016010101 348 6 0.6589994525818863 +2016010101 349 6 0.3851541727199762 +2016010101 350 6 0.12262059605539744 +2016010101 351 6 0.33383436408012057 +2016010101 352 6 0.5087733967157267 +2016010101 353 6 0.34978350071897446 +2016010101 354 6 0.9171509423859847 +2016010101 355 6 0.6395164525815664 +2016010101 356 6 0.659637993918835 +2016010101 357 6 0.5689746534857604 +2016010101 358 6 0.03266513163571427 +2016010101 359 6 0.5863675010868861 +2016010101 360 9 0.8665167898047901 +2016010101 361 9 0.7933960420424948 +2016010101 362 9 0.8409667771425247 +2016010101 363 9 0.9544310598825743 +2016010101 364 9 0.36206869840549716 +2016010101 365 9 0.253957983880155 +2016010101 366 9 0.08496022679431525 +2016010101 367 9 0.5483782518766319 +2016010101 368 9 0.41440902281408365 +2016010101 369 9 0.2947889064970717 +2016010101 370 9 0.659477180019486 +2016010101 371 9 0.9016744422830162 +2016010101 372 9 0.4692828259677926 +2016010101 373 9 0.4221974527778145 +2016010101 374 9 0.26318360778150285 +2016010101 375 9 0.10064081807071767 +2016010101 376 9 0.7781802619858804 +2016010101 377 9 0.529215767115243 +2016010101 378 9 0.21094147073619007 +2016010101 379 9 0.18894985078463877 +2016010101 380 5 0.20683422198832369 +2016010101 381 5 0.9506923735546904 +2016010101 382 5 0.25734447316063913 +2016010101 383 5 0.6439025323539892 +2016010101 384 5 0.9099080819805052 +2016010101 385 5 0.9331714165375404 +2016010101 386 5 0.24979840404324272 +2016010101 387 5 0.40270120064812764 +2016010101 388 5 0.35895113537427137 +2016010101 389 5 0.44814114645480074 +2016010101 390 5 0.437368419580639 +2016010101 391 5 0.2777496228001308 +2016010101 392 5 0.09350862521048608 +2016010101 393 5 0.10366624548706516 +2016010101 394 5 0.8715309310993357 +2016010101 395 5 0.8953111125914557 +2016010101 396 5 0.9410866942183567 +2016010101 397 5 0.16367286942347592 +2016010101 398 5 0.6995415361957786 +2016010101 399 5 0.7170527361072194 diff --git a/extensions-core/datasketches/src/test/resources/quantiles/doubles_sketch_data.tsv b/extensions-core/datasketches/src/test/resources/quantiles/doubles_sketch_data.tsv new file mode 100644 index 000000000000..18872e35d7b6 --- /dev/null +++ b/extensions-core/datasketches/src/test/resources/quantiles/doubles_sketch_data.tsv @@ -0,0 +1,20 @@ +2016010101 0 AgMIEIAAAAAUAAAAAAAAALAmWsjfb7A/pV7y+yiq7T+wJlrI32+wPwDcwrvurLY/+Ie4E36zvj+Q6JXEs4zNPy764HN4PNA/kLgyi1uh2D9OQGR8FnbfP1J6Rjgkvd8/F3Pzw8LO4T/jOtta7+DjPwwhdIAtbeQ/Ns2gDuPk5D/DJ4ic0i3lPzgUyxYhBOY/rZQ/Lqqb6D+RgJ4vWNHoP0DJ25fCQek/FY7FMO3h6T/5JWCQ1h/qP6Ve8vsoqu0/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 7 AgMIEIAAAAAUAAAAAAAAAECZc5vAmog/5G1iou8G6z9AmXObwJqIP4BjbRuKNZI/AFCU+RCPlT9AypbkpsOlP3AnrF+bAbU/ZFpXGVQ8yj8QAKUxNs/VPyYiKd2Tf9o/nls2dD8i3D+YSJVi02HhP8JR1Rfo4OI/7/vDh7kq5D+TPdqRlmTkP3411GkQseQ/ZxGsGqpe5j+u0LjibLvnPyxACPwipug/l8KWkznc6D8eSq6bZEzqP+RtYqLvBus/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAAh/UAGfz7Q/pp1Q96jD7z8If1ABn8+0P0jIwmAibLY/VBAwuTgexT9082Ym4LnGP/zHBlIB5NM/yDqoOZxU1z9wjsxQryHYPywCf0bjKds/DGKCm0uP2z/T0WvkphDgPyQSkgRM5eA/QS/QoRwb4z8kSFMZYirjP6vcDemue+M/CLvaUOUA6z9IJlRMcQPrP21yT9NqVus/+t2pLFX+7D+sf8VbdqHvP6adUPeow+8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 4 AgMIEIAAAAAUAAAAAAAAABCw34Dy1K8/uPOLHG577z8QsN+A8tSvP4AQ2sMyJLA/mISIayLNuj+Y6ag0H929P0a7OGG0XdE/HJt+OU8s1T9SIQJtQrTVPyh39PxDRd8/BANiYb+93z8vDDgddOngPwFE0u6A9eI/TF4isMEU4z/3PkrhjZ7jPyzqqEfT9OM/gL9bQw1H5D8tHPixwtblP1u+aGcyAek/J10XftBo7T/RgJ6eVAruP7jzixxue+8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 4 AgMIEIAAAAAUAAAAAAAAADjwYBq6KbI/gYsDGJvg7z848GAauimyPxB6fe5KRsk/YkLSfV6F0T94q8WPqOjSPxRYUFXQmtQ/qDBlwyCX2T8Wruv2SmHbP4Th2rwN2ts/zGZBf1oP3j/EwoWDsI3hP5Ng8QFo0OM/ymr/Eic15D+4SOaMi9nlP9IvSvz6+eU/uz3wVfDu5j8T+yfhcPPnP8v2C0Cpfuk/z/PxPRyG6T9sxltQ9pPqP4GLAxib4O8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 1 AgMIEIAAAAAUAAAAAAAAAEDo189Vt6A/UM9Cl7Dm7z9A6NfPVbegP5ha69bbD7A/iM7IRCZTvj/4YFU7rSfAP7CmPAZDpMU/6MdLwA220j82NnAaWhzVPx6klGq+Qto/SLmxT+7G3T8aiwiNFBbgP2PbMYolm+E/s5xt06Jw4z+xkG5o9LbjPyCLL7eOOeY/l74V/0Ex7D/EFnjBE1HsP0VHTOcyWe0/kWCz4CQa7j8zN5zNh+XuP1DPQpew5u8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 6 AgMIEIAAAAAUAAAAAAAAAIBqSbZC4HM//Ai0w9as7z+Aakm2QuBzP2A1A5Qu35w/oGP25EuIsz9Yz0iumZK8PyAipOadlMk/poazYsyA1j9swUqKhePYP25v9bjTW9s/hEOeatZo2z8gthEWaHHeP605XJ6N0uE/w+MGYHmg5D+JiVd2jafmPxOhq3YMrec/9a9h4DFL6T/SHFHUZ4TrP9esDC6YK+w/klvwkl6l7D9UAc2LXhfuP/wItMPWrO8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 3 AgMIEIAAAAAUAAAAAAAAAMCKxfn9FpM/3a1pNLM57z/AisX5/RaTP2AslXZKxaE/IACvfHgFvT+8TKYiOXfMPxSFcOSIbdI/oj8xt0Ki1T/GZEADR/PXP8g37R5xRdo/BFpxoDB22j9aVglB2v3aP1q9rfHPft4/zl/bBz7H4D8kBWASkTHiP4uT+vqQ0+I/i73Yc1/l4j8vFE3v7oTmP6Og2m7Z8Og/x27veeKk7D98EKH4YjLtP92taTSzOe8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 4 AgMIEIAAAAAUAAAAAAAAAPCvi6n37ao/W3jYcspc7z/wr4up9+2qP9jJ9zxtJLI/EAGaiHz3tT/wkikAdcu3P5Ds1PcgA8c/SrG9mBhJ0T/8ry6B9Q/UPxD1ZTq2mdQ/4G9tlnq13z/Ut3XGVb7fPwgVwlgXI+E/9HItq+2U4z80GQKY6AzlP/CWL1v13uU/h8lG7sz95z+ZrJRQDWnoP9JCHQGlOew/Pd+Y3G1c7T+YfmBEWffuP1t42HLKXO8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAADAvstN0CKo/Fi/8Q8b17T8wL7LTdAiqP4AnASyh0Ls/IIbcCdolvj+IAG4861nFPzBYy5ojdMo/7ISSzuS4zT/IdkLG+ZXcP7LK4ib3J94/HB0Av7CP3j+LBaVhFQXgP3UE9JzC8eI/CZprMhIX4z+jeUrP/o7mP+NWnC9kkeY/0B7qM3ev5j+SBzE1C0PnP9i+KPoRF+o/S4xh/5ss7D9YNRmoM+XtPxYv/EPG9e0/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 1 AgMIEIAAAAAUAAAAAAAAAECSmXCRI44/xH/yfd3f7z9AkplwkSOOP+C4C/Im56U/cFWv+QBrrT9gyq70BP25P7DV2eoEg7w/YMnBhQOQzT+4Tzp2KLzNP3wq4CxHJ9Q/jnmjcfv92T9oTB66zDXcP0hTwIIgUt0/nDnm7BXX3T9ev8s1guviPyvI92A6QuQ/U/iauTcu5T9s4D2SFPvmP7wcRk1YsOg/pz9Tqf0m6j/ZdTscp9juP8R/8n3d3+8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 2 AgMIEIAAAAAUAAAAAAAAAIAHyxXav3k/I972lGXG7T+AB8sV2r95P4A7s5bBmoE/rHC32fK1wT841hadadPTP/BoIumS1NM/vk73/Pcy1D9MlU8W0qXXP/xbPpnTfdg/UA8cUeUA3D/evN37c33cP8bEFkSLxuI/Uhvc5Ptq4z9gSGQo3brlP8UcQgelF+Y/lGAi8Oyr5j/Fq4JOeZDoP6aedfj8Nuo/looj9vqe7D/vpBHUdsPsPyPe9pRlxu0/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAEBWpERB7o8/beu464Zp7j9AVqREQe6PP4Ab9nsi3qo/UMVzwdc4rz9ojiLWsya8P0yxpL26hsc/CDgR4XfYyj86Zms9otvVP+gxj0+uA9Y/lDti0Dz81j/W+iWyVYfaPxhPJAkk1No/FPQ+4MVw3T8vAjqDLHngPyLkmQcuuuA/y4CwvIrE4T8uZsd1KSLjP/GxZsdIROY//NevvkI85z910QE1bYDtP23ruOuGae4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 0 AgMIEIAAAAAUAAAAAAAAAOQazM3oz8g/th9Rd0Vu7D/kGszN6M/IP8wjLnWqTM8/wkIH6uZz2D9SE7AQEoTYPzBZ5xsbw94/Ogejduhz4j9EOCjBEAPjP0F7atUeC+M/3Mjise3j5T/CJc/6P4zmP9hEv68Vs+Y/e6HmmE016D8+MrjBWw3pPyZjrdR4cek/rAu5R1/t6T+PkGyBh/PpP9hT8TvccOo/E46awzgt6z8XTyf+iITrP7YfUXdFbuw/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 7 AgMIEIAAAAAUAAAAAAAAAAA64t2+TFI/BJnMnZCp7j8AOuLdvkxSPwCoIjvLCHY/YOfQY4T5tD/Uop4ic9vCPySDfUNEysc/tuBfgSbo1T/Mfn4/s8nXP3Ii+O/nktg/HpCvdatc3D+ULFcJfRLeP1Z8xIRR3t4/lxQP8ipO4z9gBobpAnbjP7x6XqanhOQ/hxrHzpCI5T9gYwrf5mPnP/fQyzNLqOk/AsjhBDwy6z80a1ADxbDsPwSZzJ2Qqe4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAIDHPwsa7oM/NOu2UYOC7z+Axz8LGu6DP0Cls/BvKZg/SD9mURUiuj+kNll7CvbMP3qiiycHCNI/oDEZs8j30z8GmW4gajTUPwj4arsVstg/LjtkY3B82T+QkuK7B+vaP1i82/6vEeE/9kY5bylg4j+uM10lQBvjPxw16z7/CuU/UgBmDzkc6T+WOmPkhHXrPxaSMxzJl+4/H6m+hlnK7j+7qat3Ym7vPzTrtlGDgu8/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAGAMeSH7cZg/mZh2OEfG7j9gDHkh+3GYP4DVUtxEqr8/uGSsRILnzz+QYqh2i/rSPxgyZML0k9M/TLvGaRrd0z+qUlTIrT7WP0jwnIsU69Y/9DlbuJQV1z80odEVCi7XPzYJAWptlNg/xn5A3lZF2j/8IYNmyJXfP9/zj0NcF+A/J7hrIQXQ4T9er8p9YxnjPykVGmSMnOQ/HupHGIYM7D9Ctalfdy/tP5mYdjhHxu4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 6 AgMIEIAAAAAUAAAAAAAAAGCaMPB7uaA/vm3Zk6tj7j9gmjDwe7mgPwAyraIwZ6I/wLvfORBkvz9s4m2/XbzJP/h2CoHxAdM/JlMAz4pd1T9ELRFW2mLWP1IMxQZMbNg/0gPwr12m2D8RAX8l30fgP7ZZjdsZ5+E/Py0fVQo14j+aXxPHhcPiP7jvIjXrduQ/sHkdBYYW5T/+kmIjwRvlPxwTolKR7uc/QwfgNDdS6z9oY+HuTFntP75t2ZOrY+4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 9 AgMIEIAAAAAUAAAAAAAAAPDVihP0v7U/ym6PAbOK7j/w1YoT9L+1P6CkQb6Yw7k/ECDaOoIvyD9QThdPIQDLP/Kx1PzYQNA/PPMQDwDY0D/AOShK0t3SPzakozAiLNc/+tcJbK2F2j9opwh3SAXbP9DKhtW6CN4/XzOJ51Xv4D+nujSMUIzhPz2LFONvGuU//OfzStrm6D+8JasYgGPpP3Iwmygz6eo/9q80a4G66z+30ydchNrsP8pujwGziu4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +2016010101 5 AgMIEIAAAAAUAAAAAAAAAAhHLGcu8Lc/GCufaRJs7j8IRyxnLvC3P9DoEP7eibo/kEm1ijvzxD8IFZE1i3nKP2S3+uNk+c8/vAAC9FR40D8UyZlapsbRPyTa0i4O+dY/PP61QdvF2T8yu5kc2P3bP+76ADRYrtw/8bPIe9ma5D9q8YTupGLmP3jPYpQY8uY/O8XQ1ZTj6z/wa4p9Y6bsP4wdz433He0/QudyTYrc7T+Gy8vXYR3uPxgrn2kSbO4/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java index b1ad2f883b11..c5c469e14456 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java @@ -21,10 +21,10 @@ import com.google.common.collect.Ordering; import io.druid.data.input.InputRow; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.serde.ComplexColumnPartSupplier; import io.druid.segment.serde.ComplexMetricExtractor; @@ -102,9 +102,9 @@ public void deserializeColumn( } @Override - public GenericColumnSerializer getSerializer(IOPeon peon, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { - return LargeColumnSupportedComplexColumnSerializer.create(peon, column, this.getObjectStrategy()); + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } @Override diff --git a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index f74ab9541369..45197c2c7e0e 100644 --- a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Iterables; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Sequences; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -39,7 +40,6 @@ import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.segment.IndexBuilder; import io.druid.segment.QueryableIndex; -import io.druid.segment.TestHelper; import io.druid.segment.column.ValueType; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.virtual.ExpressionVirtualColumn; @@ -92,7 +92,7 @@ public void setUp() throws Exception final QueryableIndex index = IndexBuilder.create() .tmpDir(temporaryFolder.newFolder()) - .indexMerger(TestHelper.getTestIndexMergerV9()) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( new IncrementalIndexSchema.Builder() .withMetrics( diff --git a/extensions-core/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightFirehoseFactory.java b/extensions-core/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightFirehoseFactory.java index d164571966b3..9d7c6c5ffbe7 100644 --- a/extensions-core/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightFirehoseFactory.java +++ b/extensions-core/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightFirehoseFactory.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterators; import com.google.common.collect.Sets; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; @@ -40,6 +41,7 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; @@ -106,10 +108,12 @@ public Firehose connect(final InputRowParser firehoseParser, File te return new Firehose() { + Iterator nextIterator = Iterators.emptyIterator(); + @Override public boolean hasMore() { - return iter.hasNext(); + return nextIterator.hasNext() || iter.hasNext(); } @Nullable @@ -117,13 +121,16 @@ public boolean hasMore() public InputRow nextRow() { try { - final byte[] message = iter.next().message(); + if (!nextIterator.hasNext()) { + final byte[] message = iter.next().message(); - if (message == null) { - return null; + if (message == null) { + return null; + } + nextIterator = theParser.parseBatch(ByteBuffer.wrap(message)).iterator(); } - return theParser.parse(ByteBuffer.wrap(message)); + return nextIterator.next(); } catch (InvalidMessageException e) { diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java index 0b24cbe126b7..312fd1c8cbca 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java @@ -39,6 +39,7 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -65,6 +66,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.collect.Utils; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.parsers.ParseException; @@ -82,6 +84,7 @@ import io.druid.segment.realtime.appenderator.AppenderatorDriver; import io.druid.segment.realtime.appenderator.AppenderatorDriverAddResult; import io.druid.segment.realtime.appenderator.Appenderators; +import io.druid.segment.realtime.appenderator.SegmentIdentifier; import io.druid.segment.realtime.appenderator.SegmentsAndMetadata; import io.druid.segment.realtime.appenderator.TransactionalSegmentPublisher; import io.druid.segment.realtime.firehose.ChatHandler; @@ -103,6 +106,7 @@ import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.joda.time.DateTime; +import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DefaultValue; @@ -119,6 +123,7 @@ import java.nio.ByteBuffer; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -660,57 +665,87 @@ public void run() try { final byte[] valueBytes = record.value(); - final InputRow row = valueBytes == null ? null : parser.parse(ByteBuffer.wrap(valueBytes)); + final List rows = valueBytes == null + ? Utils.nullableListOf((InputRow) null) + : parser.parseBatch(ByteBuffer.wrap(valueBytes)); + boolean isPersistRequired = false; + + for (InputRow row : rows) { + if (row != null && withinMinMaxRecordTime(row)) { + SequenceMetadata sequenceToUse = null; + for (SequenceMetadata sequence : sequences) { + if (sequence.canHandle(record)) { + sequenceToUse = sequence; + } + } - if (row != null && withinMinMaxRecordTime(row)) { - SequenceMetadata sequenceToUse = null; - for (SequenceMetadata sequence : sequences) { - if (sequence.canHandle(record)) { - sequenceToUse = sequence; + if (sequenceToUse == null) { + throw new ISE( + "WTH?! cannot find any valid sequence for record with partition [%d] and offset [%d]. Current sequences: %s", + record.partition(), + record.offset(), + sequences + ); } - } - if (sequenceToUse == null) { - throw new ISE( - "WTH?! cannot find any valid sequence for record with partition [%d] and offset [%d]. Current sequences: %s", - record.partition(), - record.offset(), - sequences + final AppenderatorDriverAddResult addResult = driver.add( + row, + sequenceToUse.getSequenceName(), + committerSupplier, + // skip segment lineage check as there will always be one segment + // for combination of sequence and segment granularity. + // It is necessary to skip it as the task puts messages polled from all the + // assigned Kafka partitions into a single Druid segment, thus ordering of + // messages among replica tasks across assigned partitions is not guaranteed + // which may cause replica tasks to ask for segments with different interval + // in different order which might cause SegmentAllocateAction to fail. + true, + // do not allow incremental persists to happen until all the rows from this batch + // of rows are indexed + false ); - } - - final AppenderatorDriverAddResult addResult = driver.add( - row, - sequenceToUse.getSequenceName(), - committerSupplier, - true - // skip segment lineage check as there will always be one segment - // for combination of sequence and segment granularity. - // It is necessary to skip it as the task puts messages polled from all the - // assigned Kafka partitions into a single Druid segment, thus ordering of - // messages among replica tasks across assigned partitions is not guaranteed - // which may cause replica tasks to ask for segments with different interval - // in different order which might cause SegmentAllocateAction to fail. - ); - if (addResult.isOk()) { - // If the number of rows in the segment exceeds the threshold after adding a row, - // move the segment out from the active segments of AppenderatorDriver to make a new segment. - if (addResult.getNumRowsInSegment() > tuningConfig.getMaxRowsPerSegment()) { - if (!sequenceToUse.isCheckpointed()) { - sequenceToCheckpoint = sequenceToUse; + if (addResult.isOk()) { + // If the number of rows in the segment exceeds the threshold after adding a row, + // move the segment out from the active segments of AppenderatorDriver to make a new segment. + if (addResult.getNumRowsInSegment() > tuningConfig.getMaxRowsPerSegment()) { + if (!sequenceToUse.isCheckpointed()) { + sequenceToCheckpoint = sequenceToUse; + } } + isPersistRequired |= addResult.isPersistRequired(); + } else { + // Failure to allocate segment puts determinism at risk, bail out to be safe. + // May want configurable behavior here at some point. + // If we allow continuing, then consider blacklisting the interval for a while to avoid constant checks. + throw new ISE("Could not allocate segment for row with timestamp[%s]", row.getTimestamp()); } + + fireDepartmentMetrics.incrementProcessed(); } else { - // Failure to allocate segment puts determinism at risk, bail out to be safe. - // May want configurable behavior here at some point. - // If we allow continuing, then consider blacklisting the interval for a while to avoid constant checks. - throw new ISE("Could not allocate segment for row with timestamp[%s]", row.getTimestamp()); + fireDepartmentMetrics.incrementThrownAway(); } + } + if (isPersistRequired) { + Futures.addCallback( + driver.persistAsync(committerSupplier.get()), + new FutureCallback() + { + @Override + public void onSuccess(@Nullable Object result) + { + log.info("Persist completed with metadata [%s]", result); + } + + @Override + public void onFailure(Throwable t) + { + log.error("Persist failed, dying"); + throwableAtomicReference.set(t); + } + } + ); - fireDepartmentMetrics.incrementProcessed(); - } else { - fireDepartmentMetrics.incrementThrownAway(); } } catch (ParseException e) { @@ -1045,33 +1080,49 @@ public void run() try { final byte[] valueBytes = record.value(); - final InputRow row = valueBytes == null ? null : parser.parse(ByteBuffer.wrap(valueBytes)); - - if (row != null && withinMinMaxRecordTime(row)) { - final String sequenceName = sequenceNames.get(record.partition()); - final AppenderatorDriverAddResult addResult = driver.add( - row, - sequenceName, - committerSupplier - ); + final List rows = valueBytes == null + ? Utils.nullableListOf((InputRow) null) + : parser.parseBatch(ByteBuffer.wrap(valueBytes)); + boolean isPersistRequired = false; + final Map> segmentsToMoveOut = new HashMap<>(); + + for (InputRow row : rows) { + if (row != null && withinMinMaxRecordTime(row)) { + final String sequenceName = sequenceNames.get(record.partition()); + final AppenderatorDriverAddResult addResult = driver.add( + row, + sequenceName, + committerSupplier, + false, + false + ); - if (addResult.isOk()) { - // If the number of rows in the segment exceeds the threshold after adding a row, - // move the segment out from the active segments of AppenderatorDriver to make a new segment. - if (addResult.getNumRowsInSegment() > tuningConfig.getMaxRowsPerSegment()) { - driver.moveSegmentOut(sequenceName, ImmutableList.of(addResult.getSegmentIdentifier())); + if (addResult.isOk()) { + // If the number of rows in the segment exceeds the threshold after adding a row, + // move the segment out from the active segments of AppenderatorDriver to make a new segment. + if (addResult.getNumRowsInSegment() > tuningConfig.getMaxRowsPerSegment()) { + segmentsToMoveOut.computeIfAbsent(sequenceName, k -> new HashSet<>()) + .add(addResult.getSegmentIdentifier()); + } + isPersistRequired |= addResult.isPersistRequired(); + } else { + // Failure to allocate segment puts determinism at risk, bail out to be safe. + // May want configurable behavior here at some point. + // If we allow continuing, then consider blacklisting the interval for a while to avoid constant checks. + throw new ISE("Could not allocate segment for row with timestamp[%s]", row.getTimestamp()); } + fireDepartmentMetrics.incrementProcessed(); } else { - // Failure to allocate segment puts determinism at risk, bail out to be safe. - // May want configurable behavior here at some point. - // If we allow continuing, then consider blacklisting the interval for a while to avoid constant checks. - throw new ISE("Could not allocate segment for row with timestamp[%s]", row.getTimestamp()); + fireDepartmentMetrics.incrementThrownAway(); } - - fireDepartmentMetrics.incrementProcessed(); - } else { - fireDepartmentMetrics.incrementThrownAway(); } + if (isPersistRequired) { + driver.persist(committerSupplier.get()); + } + segmentsToMoveOut.entrySet().forEach(sequenceSegments -> driver.moveSegmentOut( + sequenceSegments.getKey(), + sequenceSegments.getValue().stream().collect(Collectors.toList()) + )); } catch (ParseException e) { if (tuningConfig.isReportParseExceptions()) { diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaTuningConfig.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaTuningConfig.java index 6da2ef6a34ce..5210028ded3b 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaTuningConfig.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaTuningConfig.java @@ -25,9 +25,12 @@ import io.druid.segment.indexing.RealtimeTuningConfig; import io.druid.segment.indexing.TuningConfig; import io.druid.segment.realtime.appenderator.AppenderatorConfig; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import org.joda.time.Period; +import javax.annotation.Nullable; import java.io.File; +import java.util.Objects; public class KafkaTuningConfig implements TuningConfig, AppenderatorConfig { @@ -44,20 +47,23 @@ public class KafkaTuningConfig implements TuningConfig, AppenderatorConfig private final boolean reportParseExceptions; private final long handoffConditionTimeout; private final boolean resetOffsetAutomatically; + @Nullable + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; @JsonCreator public KafkaTuningConfig( - @JsonProperty("maxRowsInMemory") Integer maxRowsInMemory, - @JsonProperty("maxRowsPerSegment") Integer maxRowsPerSegment, - @JsonProperty("intermediatePersistPeriod") Period intermediatePersistPeriod, - @JsonProperty("basePersistDirectory") File basePersistDirectory, - @JsonProperty("maxPendingPersists") Integer maxPendingPersists, - @JsonProperty("indexSpec") IndexSpec indexSpec, + @JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory, + @JsonProperty("maxRowsPerSegment") @Nullable Integer maxRowsPerSegment, + @JsonProperty("intermediatePersistPeriod") @Nullable Period intermediatePersistPeriod, + @JsonProperty("basePersistDirectory") @Nullable File basePersistDirectory, + @JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists, + @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec, // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12. - @JsonProperty("buildV9Directly") Boolean buildV9Directly, - @JsonProperty("reportParseExceptions") Boolean reportParseExceptions, - @JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout, - @JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically + @JsonProperty("buildV9Directly") @Nullable Boolean buildV9Directly, + @JsonProperty("reportParseExceptions") @Nullable Boolean reportParseExceptions, + @JsonProperty("handoffConditionTimeout") @Nullable Long handoffConditionTimeout, + @JsonProperty("resetOffsetAutomatically") @Nullable Boolean resetOffsetAutomatically, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { // Cannot be a static because default basePersistDirectory is unique per-instance @@ -80,6 +86,7 @@ public KafkaTuningConfig( this.resetOffsetAutomatically = resetOffsetAutomatically == null ? DEFAULT_RESET_OFFSET_AUTOMATICALLY : resetOffsetAutomatically; + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } public static KafkaTuningConfig copyOf(KafkaTuningConfig config) @@ -94,7 +101,8 @@ public static KafkaTuningConfig copyOf(KafkaTuningConfig config) true, config.reportParseExceptions, config.handoffConditionTimeout, - config.resetOffsetAutomatically + config.resetOffsetAutomatically, + config.segmentWriteOutMediumFactory ); } @@ -169,6 +177,14 @@ public boolean isResetOffsetAutomatically() return resetOffsetAutomatically; } + @Override + @JsonProperty + @Nullable + public SegmentWriteOutMediumFactory getSegmentWriteOutMediumFactory() + { + return segmentWriteOutMediumFactory; + } + public KafkaTuningConfig withBasePersistDirectory(File dir) { return new KafkaTuningConfig( @@ -181,7 +197,8 @@ public KafkaTuningConfig withBasePersistDirectory(File dir) true, reportParseExceptions, handoffConditionTimeout, - resetOffsetAutomatically + resetOffsetAutomatically, + segmentWriteOutMediumFactory ); } @@ -194,54 +211,34 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - KafkaTuningConfig that = (KafkaTuningConfig) o; - - if (maxRowsInMemory != that.maxRowsInMemory) { - return false; - } - if (maxRowsPerSegment != that.maxRowsPerSegment) { - return false; - } - if (maxPendingPersists != that.maxPendingPersists) { - return false; - } - if (reportParseExceptions != that.reportParseExceptions) { - return false; - } - if (handoffConditionTimeout != that.handoffConditionTimeout) { - return false; - } - if (resetOffsetAutomatically != that.resetOffsetAutomatically) { - return false; - } - if (intermediatePersistPeriod != null - ? !intermediatePersistPeriod.equals(that.intermediatePersistPeriod) - : that.intermediatePersistPeriod != null) { - return false; - } - if (basePersistDirectory != null - ? !basePersistDirectory.equals(that.basePersistDirectory) - : that.basePersistDirectory != null) { - return false; - } - return indexSpec != null ? indexSpec.equals(that.indexSpec) : that.indexSpec == null; - + return maxRowsInMemory == that.maxRowsInMemory && + maxRowsPerSegment == that.maxRowsPerSegment && + maxPendingPersists == that.maxPendingPersists && + reportParseExceptions == that.reportParseExceptions && + handoffConditionTimeout == that.handoffConditionTimeout && + resetOffsetAutomatically == that.resetOffsetAutomatically && + Objects.equals(intermediatePersistPeriod, that.intermediatePersistPeriod) && + Objects.equals(basePersistDirectory, that.basePersistDirectory) && + Objects.equals(indexSpec, that.indexSpec) && + Objects.equals(segmentWriteOutMediumFactory, that.segmentWriteOutMediumFactory); } @Override public int hashCode() { - int result = maxRowsInMemory; - result = 31 * result + maxRowsPerSegment; - result = 31 * result + (intermediatePersistPeriod != null ? intermediatePersistPeriod.hashCode() : 0); - result = 31 * result + (basePersistDirectory != null ? basePersistDirectory.hashCode() : 0); - result = 31 * result + maxPendingPersists; - result = 31 * result + (indexSpec != null ? indexSpec.hashCode() : 0); - result = 31 * result + (reportParseExceptions ? 1 : 0); - result = 31 * result + (int) (handoffConditionTimeout ^ (handoffConditionTimeout >>> 32)); - result = 31 * result + (resetOffsetAutomatically ? 1 : 0); - return result; + return Objects.hash( + maxRowsInMemory, + maxRowsPerSegment, + intermediatePersistPeriod, + basePersistDirectory, + maxPendingPersists, + indexSpec, + reportParseExceptions, + handoffConditionTimeout, + resetOffsetAutomatically, + segmentWriteOutMediumFactory + ); } @Override @@ -257,6 +254,7 @@ public String toString() ", reportParseExceptions=" + reportParseExceptions + ", handoffConditionTimeout=" + handoffConditionTimeout + ", resetOffsetAutomatically=" + resetOffsetAutomatically + + ", segmentWriteOutMediumFactory=" + segmentWriteOutMediumFactory + '}'; } } diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorSpec.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorSpec.java index 559428fc2e49..d11709ecc620 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorSpec.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorSpec.java @@ -88,6 +88,7 @@ public KafkaSupervisorSpec( null, null, null, + null, null ); this.ioConfig = Preconditions.checkNotNull(ioConfig, "ioConfig"); diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfig.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfig.java index 587d1a300ddd..8d043877c38c 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfig.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfig.java @@ -21,10 +21,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.indexing.kafka.KafkaTuningConfig; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.IndexSpec; import org.joda.time.Duration; import org.joda.time.Period; +import javax.annotation.Nullable; import java.io.File; public class KafkaSupervisorTuningConfig extends KafkaTuningConfig @@ -48,6 +50,7 @@ public KafkaSupervisorTuningConfig( @JsonProperty("reportParseExceptions") Boolean reportParseExceptions, @JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout, @JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, @JsonProperty("workerThreads") Integer workerThreads, @JsonProperty("chatThreads") Integer chatThreads, @JsonProperty("chatRetries") Long chatRetries, @@ -66,7 +69,8 @@ public KafkaSupervisorTuningConfig( true, reportParseExceptions, handoffConditionTimeout, - resetOffsetAutomatically + resetOffsetAutomatically, + segmentWriteOutMediumFactory ); this.workerThreads = workerThreads; @@ -126,6 +130,7 @@ public String toString() ", reportParseExceptions=" + isReportParseExceptions() + ", handoffConditionTimeout=" + getHandoffConditionTimeout() + ", resetOffsetAutomatically=" + isResetOffsetAutomatically() + + ", segmentWriteOutMediumFactory=" + getSegmentWriteOutMediumFactory() + ", workerThreads=" + workerThreads + ", chatThreads=" + chatThreads + ", chatRetries=" + chatRetries + diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java index e33405bc652f..9d94127d4de3 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -1629,7 +1629,8 @@ private KafkaIndexTask createTask( true, reportParseExceptions, handoffConditionTimeout, - resetOffsetAutomatically + resetOffsetAutomatically, + null ); final Map context = isIncrementalHandoffSupported ? ImmutableMap.of(KafkaSupervisor.IS_INCREMENTAL_HANDOFF_SUPPORTED, true) diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaTuningConfigTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaTuningConfigTest.java index 3bd5a3891007..386664e4c0dc 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaTuningConfigTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaTuningConfigTest.java @@ -111,6 +111,7 @@ public void testCopyOf() throws Exception true, true, 5L, + null, null ); KafkaTuningConfig copy = KafkaTuningConfig.copyOf(original); diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 347813858c65..cd159edf2731 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -194,6 +194,7 @@ public void setupTest() throws Exception false, null, null, + null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/data/input/MapPopulator.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/data/input/MapPopulator.java index dcf90849c6bc..0c472bf30b37 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/data/input/MapPopulator.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/data/input/MapPopulator.java @@ -91,7 +91,7 @@ public boolean processLine(String line) throws IOException if (lines == Integer.MAX_VALUE) { throw new ISE("Cannot read more than %,d lines", Integer.MAX_VALUE); } - final Map kvMap = parser.parse(line); + final Map kvMap = parser.parseToMap(line); map.putAll(kvMap); lines++; entries += kvMap.size(); diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java index ffa9f898cfe2..ff0a93c4da2c 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java @@ -213,9 +213,9 @@ private DelegateParser( } @Override - public Map parse(String input) + public Map parseToMap(String input) { - final Map inner = delegate.parse(input); + final Map inner = delegate.parseToMap(input); final String k = Preconditions.checkNotNull( inner.get(key), "Key column [%s] missing data in line [%s]", @@ -612,7 +612,7 @@ public ObjectMapperFlatDataParser( parser = new Parser() { @Override - public Map parse(String input) + public Map parseToMap(String input) { try { return jsonFactory.createParser(input).readValueAs(JacksonUtils.TYPE_REFERENCE_MAP_STRING_STRING); diff --git a/extensions-core/lookups-cached-global/src/test/java/io/druid/query/lookup/namespace/UriExtractionNamespaceTest.java b/extensions-core/lookups-cached-global/src/test/java/io/druid/query/lookup/namespace/UriExtractionNamespaceTest.java index 488a4779d46e..d9870554e8eb 100644 --- a/extensions-core/lookups-cached-global/src/test/java/io/druid/query/lookup/namespace/UriExtractionNamespaceTest.java +++ b/extensions-core/lookups-cached-global/src/test/java/io/druid/query/lookup/namespace/UriExtractionNamespaceTest.java @@ -87,7 +87,7 @@ public void testCSV() "col3" ), "col2", "col3" ); - Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parse("A,B,C")); + Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parseToMap("A,B,C")); } @Test(expected = IllegalArgumentException.class) @@ -100,7 +100,7 @@ public void testBadCSV() "col3" ), "col2", "col3ADFSDF" ); - Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parse("A,B,C")); + Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parseToMap("A,B,C")); } @Test(expected = NullPointerException.class) @@ -113,7 +113,7 @@ public void testBadCSV2() "col3" ), "col2", "col3" ); - Map map = parser.getParser().parse("A"); + Map map = parser.getParser().parseToMap("A"); } @Test @@ -125,7 +125,7 @@ public void testTSV() null, "col2", "col3" ); - Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parse("A|B|C")); + Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parseToMap("A|B|C")); } @Test @@ -137,7 +137,7 @@ public void testWithListDelimiterTSV() "\\u0002", "col2", "col3" ); - Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parse("A\\u0001B\\u0001C")); + Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parseToMap("A\\u0001B\\u0001C")); } @Test(expected = IllegalArgumentException.class) @@ -149,8 +149,8 @@ public void testBadTSV() null, "col2", "col3" ); - Map map = parser.getParser().parse("A,B,C"); - Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parse("A,B,C")); + Map map = parser.getParser().parseToMap("A,B,C"); + Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parseToMap("A,B,C")); } @@ -163,8 +163,8 @@ public void testBadTSV2() null, "col2", "col3" ); - Map map = parser.getParser().parse("A"); - Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parse("A,B,C")); + Map map = parser.getParser().parseToMap("A"); + Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parseToMap("A,B,C")); } @Test @@ -180,7 +180,7 @@ public void testJSONFlatDataParser() Assert.assertEquals( ImmutableMap.of("B", "C"), parser.getParser() - .parse( + .parseToMap( StringUtils.format( "{\"%s\":\"B\", \"%s\":\"C\", \"FOO\":\"BAR\"}", keyField, @@ -204,7 +204,7 @@ public void testJSONFlatDataParserBad() Assert.assertEquals( ImmutableMap.of("B", "C"), parser.getParser() - .parse( + .parseToMap( StringUtils.format( "{\"%sDFSDFDS\":\"B\", \"%s\":\"C\", \"FOO\":\"BAR\"}", keyField, @@ -227,7 +227,7 @@ public void testJSONFlatDataParserBad2() Assert.assertEquals( ImmutableMap.of("B", "C"), parser.getParser() - .parse( + .parseToMap( StringUtils.format( "{\"%sDFSDFDS\":\"B\", \"%s\":\"C\", \"FOO\":\"BAR\"}", keyField, @@ -250,7 +250,7 @@ public void testJSONFlatDataParserBad3() Assert.assertEquals( ImmutableMap.of("B", "C"), parser.getParser() - .parse( + .parseToMap( StringUtils.format( "{\"%sDFSDFDS\":\"B\", \"%s\":\"C\", \"FOO\":\"BAR\"}", keyField, @@ -273,7 +273,7 @@ public void testJSONFlatDataParserBad4() Assert.assertEquals( ImmutableMap.of("B", "C"), parser.getParser() - .parse( + .parseToMap( StringUtils.format( "{\"%sDFSDFDS\":\"B\", \"%s\":\"C\", \"FOO\":\"BAR\"}", keyField, @@ -289,7 +289,7 @@ public void testObjectMapperFlatDataParser() UriExtractionNamespace.ObjectMapperFlatDataParser parser = new UriExtractionNamespace.ObjectMapperFlatDataParser( registerTypes(new ObjectMapper()) ); - Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parse("{\"B\":\"C\"}")); + Assert.assertEquals(ImmutableMap.of("B", "C"), parser.getParser().parseToMap("{\"B\":\"C\"}")); } @Test @@ -398,7 +398,7 @@ public void testFlatDataNumeric() "num string value", ImmutableMap.of("B", nString), parser.getParser() - .parse( + .parseToMap( StringUtils.format( "{\"%s\":\"B\", \"%s\":\"%d\", \"FOO\":\"BAR\"}", keyField, @@ -411,7 +411,7 @@ public void testFlatDataNumeric() "num string key", ImmutableMap.of(nString, "C"), parser.getParser() - .parse( + .parseToMap( StringUtils.format( "{\"%s\":\"%d\", \"%s\":\"C\", \"FOO\":\"BAR\"}", keyField, @@ -424,7 +424,7 @@ public void testFlatDataNumeric() "num value", ImmutableMap.of("B", nString), parser.getParser() - .parse( + .parseToMap( StringUtils.format( "{\"%s\":\"B\", \"%s\":%d, \"FOO\":\"BAR\"}", keyField, @@ -437,7 +437,7 @@ public void testFlatDataNumeric() "num key", ImmutableMap.of(nString, "C"), parser.getParser() - .parse( + .parseToMap( StringUtils.format( "{\"%s\":%d, \"%s\":\"C\", \"FOO\":\"BAR\"}", keyField, @@ -458,7 +458,7 @@ public void testSimpleJsonNumeric() final String nString = StringUtils.format("%d", n); Assert.assertEquals( ImmutableMap.of("key", nString), - parser.getParser().parse(StringUtils.format("{\"key\":%d}", n)) + parser.getParser().parseToMap(StringUtils.format("{\"key\":%d}", n)) ); } } diff --git a/extensions-core/protobuf-extensions/src/main/java/io/druid/data/input/protobuf/ProtobufInputRowParser.java b/extensions-core/protobuf-extensions/src/main/java/io/druid/data/input/protobuf/ProtobufInputRowParser.java index ec9910742345..b24fabe3c1b2 100644 --- a/extensions-core/protobuf-extensions/src/main/java/io/druid/data/input/protobuf/ProtobufInputRowParser.java +++ b/extensions-core/protobuf-extensions/src/main/java/io/druid/data/input/protobuf/ProtobufInputRowParser.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.Descriptor; @@ -41,6 +42,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.ByteBuffer; +import java.util.List; import java.util.Map; import java.util.Set; @@ -79,7 +81,7 @@ public ProtobufInputRowParser withParseSpec(ParseSpec parseSpec) } @Override - public InputRow parse(ByteBuffer input) + public List parseBatch(ByteBuffer input) { if (parser == null) { // parser should be created when it is really used to avoid unnecessary initialization of the underlying @@ -95,12 +97,12 @@ public InputRow parse(ByteBuffer input) throw new ParseException(e, "Protobuf message could not be parsed"); } - Map record = parser.parse(json); - return new MapBasedInputRow( + Map record = parser.parseToMap(json); + return ImmutableList.of(new MapBasedInputRow( parseSpec.getTimestampSpec().extractTimestamp(record), parseSpec.getDimensionsSpec().getDimensionNames(), record - ); + )); } private Descriptor getDescriptor(String descriptorFilePath) diff --git a/extensions-core/protobuf-extensions/src/test/java/io/druid/data/input/protobuf/ProtobufInputRowParserTest.java b/extensions-core/protobuf-extensions/src/test/java/io/druid/data/input/protobuf/ProtobufInputRowParserTest.java index 0d07758c0d0a..92fa60827d8a 100644 --- a/extensions-core/protobuf-extensions/src/test/java/io/druid/data/input/protobuf/ProtobufInputRowParserTest.java +++ b/extensions-core/protobuf-extensions/src/test/java/io/druid/data/input/protobuf/ProtobufInputRowParserTest.java @@ -156,7 +156,7 @@ public void testParse() throws Exception ByteArrayOutputStream out = new ByteArrayOutputStream(); event.writeTo(out); - InputRow row = parser.parse(ByteBuffer.wrap(out.toByteArray())); + InputRow row = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); System.out.println(row); assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); @@ -199,7 +199,7 @@ public void testDisableJavaScript() expectedException.expect(CoreMatchers.instanceOf(IllegalStateException.class)); expectedException.expectMessage("JavaScript is disabled"); - parser.parse(ByteBuffer.allocate(1)); + parser.parseBatch(ByteBuffer.allocate(1)).get(0); } private void assertDimensionEquals(InputRow row, String dimension, Object expected) diff --git a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java index 1ed59fff48c1..cc41bf73bfd8 100644 --- a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java +++ b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java @@ -21,10 +21,10 @@ import com.google.common.collect.Ordering; import io.druid.data.input.InputRow; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.serde.ComplexColumnPartSupplier; import io.druid.segment.serde.ComplexMetricExtractor; @@ -122,9 +122,9 @@ public int compare(VarianceAggregatorCollector o1, VarianceAggregatorCollector o } @Override - public GenericColumnSerializer getSerializer(IOPeon peon, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { - return LargeColumnSupportedComplexColumnSerializer.create(peon, column, this.getObjectStrategy()); + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerMapper.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerMapper.java index 2a6a37883ab3..dce2634454ea 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerMapper.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerMapper.java @@ -19,19 +19,21 @@ package io.druid.indexer; +import com.google.common.collect.ImmutableList; import io.druid.data.input.InputRow; import io.druid.data.input.impl.InputRowParser; import io.druid.data.input.impl.StringInputRowParser; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.RE; +import io.druid.java.util.common.collect.Utils; import io.druid.java.util.common.logger.Logger; import io.druid.java.util.common.parsers.ParseException; import io.druid.segment.indexing.granularity.GranularitySpec; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; -import javax.annotation.Nullable; import java.io.IOException; +import java.util.List; public abstract class HadoopDruidIndexerMapper extends Mapper { @@ -61,9 +63,9 @@ public HadoopDruidIndexerConfig getConfig() protected void map(Object key, Object value, Context context) throws IOException, InterruptedException { try { - final InputRow inputRow; + final List inputRows; try { - inputRow = parseInputRow(value, parser); + inputRows = parseInputRow(value, parser); } catch (ParseException e) { if (reportParseExceptions) { @@ -74,16 +76,17 @@ protected void map(Object key, Object value, Context context) throws IOException return; // we're ignoring this invalid row } - if (inputRow == null) { - // Throw away null rows from the parser. - log.debug("Throwing away row [%s]", value); - return; - } - - if (!granularitySpec.bucketIntervals().isPresent() - || granularitySpec.bucketInterval(DateTimes.utc(inputRow.getTimestampFromEpoch())) - .isPresent()) { - innerMap(inputRow, context, reportParseExceptions); + for (InputRow inputRow : inputRows) { + if (inputRow == null) { + // Throw away null rows from the parser. + log.debug("Throwing away row [%s]", value); + continue; + } + if (!granularitySpec.bucketIntervals().isPresent() + || granularitySpec.bucketInterval(DateTimes.utc(inputRow.getTimestampFromEpoch())) + .isPresent()) { + innerMap(inputRow, context, reportParseExceptions); + } } } catch (RuntimeException e) { @@ -91,21 +94,20 @@ protected void map(Object key, Object value, Context context) throws IOException } } - @Nullable - public static InputRow parseInputRow(Object value, InputRowParser parser) + private static List parseInputRow(Object value, InputRowParser parser) { if (parser instanceof StringInputRowParser && value instanceof Text) { //Note: This is to ensure backward compatibility with 0.7.0 and before //HadoopyStringInputRowParser can handle this and this special case is not needed //except for backward compatibility - return ((StringInputRowParser) parser).parse(value.toString()); + return Utils.nullableListOf(((StringInputRowParser) parser).parse(value.toString())); } else if (value instanceof InputRow) { - return (InputRow) value; + return ImmutableList.of((InputRow) value); } else if (value == null) { // Pass through nulls so they get thrown away. - return null; + return Utils.nullableListOf((InputRow) null); } else { - return parser.parse(value); + return parser.parseBatch(value); } } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopyStringInputRowParser.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopyStringInputRowParser.java index b7ba8d3b8213..c39dca8eb44d 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopyStringInputRowParser.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopyStringInputRowParser.java @@ -20,16 +20,17 @@ package io.druid.indexer; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; import io.druid.data.input.InputRow; import io.druid.data.input.impl.InputRowParser; import io.druid.data.input.impl.ParseSpec; import io.druid.data.input.impl.StringInputRowParser; import io.druid.java.util.common.IAE; - import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import java.nio.ByteBuffer; +import java.util.List; /** */ @@ -43,13 +44,13 @@ public HadoopyStringInputRowParser(@JsonProperty("parseSpec") ParseSpec parseSpe } @Override - public InputRow parse(Object input) + public List parseBatch(Object input) { if (input instanceof Text) { - return parser.parse(((Text) input).toString()); + return ImmutableList.of(parser.parse(((Text) input).toString())); } else if (input instanceof BytesWritable) { BytesWritable valueBytes = (BytesWritable) input; - return parser.parse(ByteBuffer.wrap(valueBytes.getBytes(), 0, valueBytes.getLength())); + return parser.parseBatch(ByteBuffer.wrap(valueBytes.getBytes(), 0, valueBytes.getLength())); } else { throw new IAE("can't convert type [%s] to InputRow", input.getClass().getName()); } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java index 0e4689218207..a4e6f9a35032 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java @@ -503,7 +503,7 @@ private File persist( ) throws IOException { return HadoopDruidIndexerConfig.INDEX_MERGER_V9.persist( - index, interval, file, config.getIndexSpec(), progressIndicator + index, interval, file, config.getIndexSpec(), progressIndicator, null ); } @@ -516,7 +516,7 @@ protected File mergeQueryableIndex( { boolean rollup = config.getSchema().getDataSchema().getGranularitySpec().isRollup(); return HadoopDruidIndexerConfig.INDEX_MERGER_V9.mergeQueryableIndex( - indexes, rollup, aggs, file, config.getIndexSpec(), progressIndicator + indexes, rollup, aggs, file, config.getIndexSpec(), progressIndicator, null ); } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/updater/HadoopConverterJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/updater/HadoopConverterJob.java index d19fc4026fed..eff3c4f4e109 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/updater/HadoopConverterJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/updater/HadoopConverterJob.java @@ -535,7 +535,8 @@ protected void map( inDir, outDir, config.getIndexSpec(), - JobHelper.progressIndicatorForContext(context) + JobHelper.progressIndicatorForContext(context), + null ); } catch (Exception e) { diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/updater/HadoopConverterJobTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/updater/HadoopConverterJobTest.java index a06df9281160..844df90b095d 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/updater/HadoopConverterJobTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/updater/HadoopConverterJobTest.java @@ -56,8 +56,8 @@ import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.segment.IndexSpec; import io.druid.segment.TestIndex; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.indexing.DataSchema; import io.druid.segment.indexing.granularity.UniformGranularitySpec; @@ -293,8 +293,8 @@ public MetadataSegmentManagerConfig get() DATASOURCE, interval, new IndexSpec(new RoaringBitmapSerdeFactory(null), - CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, - CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, + CompressionStrategy.UNCOMPRESSED, + CompressionStrategy.UNCOMPRESSED, CompressionFactory.LongEncodingStrategy.LONGS), oldSemgments, true, @@ -399,8 +399,8 @@ public MetadataSegmentManagerConfig get() DATASOURCE, interval, new IndexSpec(new RoaringBitmapSerdeFactory(null), - CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, - CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, + CompressionStrategy.UNCOMPRESSED, + CompressionStrategy.UNCOMPRESSED, CompressionFactory.LongEncodingStrategy.LONGS), oldSemgments, true, diff --git a/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java b/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java index 808cda53b948..b6e2a26d88fc 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java @@ -130,7 +130,7 @@ public int add(InputRow row, Supplier committerSupplier) throws Index return -1; } - final int numRows = sink.add(row); + final int numRows = sink.add(row, false); if (!sink.canAppendRow()) { persist(committerSupplier.get()); @@ -182,7 +182,14 @@ public void finishJob() } fileToUpload = new File(tmpSegmentDir, "merged"); - indexMergerV9.mergeQueryableIndex(indexes, schema.getGranularitySpec().isRollup(), schema.getAggregators(), fileToUpload, config.getIndexSpec()); + indexMergerV9.mergeQueryableIndex( + indexes, + schema.getGranularitySpec().isRollup(), + schema.getAggregators(), + fileToUpload, + config.getIndexSpec(), + config.getSegmentWriteOutMediumFactory() + ); } // Map merged segment so we can extract dimensions @@ -230,7 +237,8 @@ private void spillIfSwappable() indexMergerV9.persist( indexToPersist.getIndex(), dirToPersist, - config.getIndexSpec() + config.getIndexSpec(), + config.getSegmentWriteOutMediumFactory() ); indexToPersist.swapSegment(null); diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/AppendTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/AppendTask.java index 97cd6f6d6b91..c181d2eb2fce 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/AppendTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/AppendTask.java @@ -29,6 +29,7 @@ import io.druid.indexing.common.TaskToolbox; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.guava.Comparators; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.IndexMerger; import io.druid.segment.IndexSpec; @@ -63,10 +64,11 @@ public AppendTask( @JsonProperty("indexSpec") IndexSpec indexSpec, // This parameter is left for compatibility when reading existing JSONs, to be removed in Druid 0.12. @JsonProperty("buildV9Directly") Boolean buildV9Directly, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, @JsonProperty("context") Map context ) { - super(id, dataSource, segments, context); + super(id, dataSource, segments, segmentWriteOutMediumFactory, context); this.indexSpec = indexSpec == null ? new IndexSpec() : indexSpec; this.aggregators = aggregators; } @@ -139,7 +141,8 @@ public boolean apply(Rowboat input) adapters, aggregators == null ? null : aggregators.toArray(new AggregatorFactory[aggregators.size()]), outDir, - indexSpec + indexSpec, + getSegmentWriteOutMediumFactory() ); } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentBackwardsCompatibleTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentBackwardsCompatibleTask.java index 0a2b9d2a12be..ca252c160612 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentBackwardsCompatibleTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentBackwardsCompatibleTask.java @@ -21,10 +21,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.IndexSpec; import io.druid.timeline.DataSegment; import org.joda.time.Interval; +import javax.annotation.Nullable; + @Deprecated public class ConvertSegmentBackwardsCompatibleTask extends ConvertSegmentTask { @@ -36,7 +39,8 @@ public ConvertSegmentBackwardsCompatibleTask( @JsonProperty("segment") DataSegment segment, @JsonProperty("indexSpec") IndexSpec indexSpec, @JsonProperty("force") Boolean force, - @JsonProperty("validate") Boolean validate + @JsonProperty("validate") Boolean validate, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { super( @@ -47,6 +51,7 @@ public ConvertSegmentBackwardsCompatibleTask( indexSpec, force == null ? false : force, validate == null ? false : validate, + segmentWriteOutMediumFactory, null ); } @@ -60,10 +65,11 @@ public SubTask( @JsonProperty("segment") DataSegment segment, @JsonProperty("indexSpec") IndexSpec indexSpec, @JsonProperty("force") Boolean force, - @JsonProperty("validate") Boolean validate + @JsonProperty("validate") Boolean validate, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { - super(groupId, segment, indexSpec, force, validate, null); + super(groupId, segment, indexSpec, force, validate, segmentWriteOutMediumFactory, null); } } } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java index 5ab8251d7f5d..4605995ee613 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java @@ -36,12 +36,14 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.FunctionalIterable; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.IndexIO; import io.druid.segment.IndexSpec; import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.util.Collections; @@ -61,12 +63,6 @@ public class ConvertSegmentTask extends AbstractFixedIntervalTask private static final Logger log = new Logger(ConvertSegmentTask.class); - @JsonIgnore - private final DataSegment segment; - private final IndexSpec indexSpec; - private final boolean force; - private final boolean validate; - /** * Create a segment converter task to convert a segment to the most recent version including the specified indexSpec * @@ -84,11 +80,22 @@ public static ConvertSegmentTask create( IndexSpec indexSpec, boolean force, boolean validate, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, Map context ) { final String id = makeId(dataSource, interval); - return new ConvertSegmentTask(id, dataSource, interval, null, indexSpec, force, validate, context); + return new ConvertSegmentTask( + id, + dataSource, + interval, + null, + indexSpec, + force, + validate, + segmentWriteOutMediumFactory, + context + ); } /** @@ -106,13 +113,24 @@ public static ConvertSegmentTask create( IndexSpec indexSpec, boolean force, boolean validate, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, Map context ) { final Interval interval = segment.getInterval(); final String dataSource = segment.getDataSource(); final String id = makeId(dataSource, interval); - return new ConvertSegmentTask(id, dataSource, interval, segment, indexSpec, force, validate, context); + return new ConvertSegmentTask( + id, + dataSource, + interval, + segment, + indexSpec, + force, + validate, + segmentWriteOutMediumFactory, + context + ); } protected static String makeId(String dataSource, Interval interval) @@ -131,22 +149,41 @@ private static ConvertSegmentTask createFromJson( @JsonProperty("indexSpec") IndexSpec indexSpec, @JsonProperty("force") Boolean force, @JsonProperty("validate") Boolean validate, - @JsonProperty("context") Map context + @JsonProperty("context") Map context, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { final boolean isForce = force == null ? false : force; final boolean isValidate = validate == null ? true : validate; if (id == null) { if (segment == null) { - return create(dataSource, interval, indexSpec, isForce, isValidate, context); + return create(dataSource, interval, indexSpec, isForce, isValidate, segmentWriteOutMediumFactory, context); } else { - return create(segment, indexSpec, isForce, isValidate, context); + return create(segment, indexSpec, isForce, isValidate, segmentWriteOutMediumFactory, context); } } - return new ConvertSegmentTask(id, dataSource, interval, segment, indexSpec, isForce, isValidate, context); + return new ConvertSegmentTask( + id, + dataSource, + interval, + segment, + indexSpec, + isForce, + isValidate, + segmentWriteOutMediumFactory, + context + ); } - protected ConvertSegmentTask( + @JsonIgnore + private final DataSegment segment; + private final IndexSpec indexSpec; + private final boolean force; + private final boolean validate; + @Nullable + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; + + ConvertSegmentTask( String id, String dataSource, Interval interval, @@ -154,6 +191,7 @@ protected ConvertSegmentTask( IndexSpec indexSpec, boolean force, boolean validate, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, Map context ) { @@ -162,6 +200,7 @@ protected ConvertSegmentTask( this.indexSpec = indexSpec == null ? new IndexSpec() : indexSpec; this.force = force; this.validate = validate; + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } @JsonProperty @@ -194,6 +233,13 @@ public DataSegment getSegment() return segment; } + @JsonProperty + @Nullable + public SegmentWriteOutMediumFactory getSegmentWriteOutMediumFactory() + { + return segmentWriteOutMediumFactory; + } + @Override public TaskStatus run(TaskToolbox toolbox) throws Exception { @@ -261,7 +307,7 @@ protected Iterable generateSubTasks( @Override public Task apply(DataSegment input) { - return new SubTask(groupId, input, indexSpec, force, validate, context); + return new SubTask(groupId, input, indexSpec, force, validate, segmentWriteOutMediumFactory, context); } } ); @@ -293,6 +339,8 @@ public static class SubTask extends AbstractFixedIntervalTask private final IndexSpec indexSpec; private final boolean force; private final boolean validate; + @Nullable + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; @JsonCreator public SubTask( @@ -301,6 +349,7 @@ public SubTask( @JsonProperty("indexSpec") IndexSpec indexSpec, @JsonProperty("force") Boolean force, @JsonProperty("validate") Boolean validate, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, @JsonProperty("context") Map context ) { @@ -321,6 +370,7 @@ public SubTask( this.indexSpec = indexSpec == null ? new IndexSpec() : indexSpec; this.force = force == null ? false : force; this.validate = validate == null ? true : validate; + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } @JsonProperty @@ -352,7 +402,7 @@ public TaskStatus run(TaskToolbox toolbox) throws Exception { log.info("Subs are good! Italian BMT and Meatball are probably my favorite."); try { - convertSegment(toolbox, segment, indexSpec, force, validate); + convertSegment(toolbox); } catch (Exception e) { log.error(e, "Conversion failed."); @@ -360,48 +410,42 @@ public TaskStatus run(TaskToolbox toolbox) throws Exception } return success(); } - } - private static void convertSegment( - TaskToolbox toolbox, - final DataSegment segment, - IndexSpec indexSpec, - boolean force, - boolean validate - ) - throws SegmentLoadingException, IOException - { - log.info("Converting segment[%s]", segment); - final TaskActionClient actionClient = toolbox.getTaskActionClient(); - final List currentSegments = actionClient.submit( - new SegmentListUsedAction(segment.getDataSource(), segment.getInterval(), null) - ); + private void convertSegment(TaskToolbox toolbox) throws SegmentLoadingException, IOException + { + log.info("Converting segment[%s]", segment); + final TaskActionClient actionClient = toolbox.getTaskActionClient(); + final List currentSegments = actionClient.submit( + new SegmentListUsedAction(segment.getDataSource(), segment.getInterval(), null) + ); - for (DataSegment currentSegment : currentSegments) { - final String version = currentSegment.getVersion(); - final Integer binaryVersion = currentSegment.getBinaryVersion(); + for (DataSegment currentSegment : currentSegments) { + final String version = currentSegment.getVersion(); + final Integer binaryVersion = currentSegment.getBinaryVersion(); - if (!force && (version.startsWith(segment.getVersion()) && CURR_VERSION_INTEGER.equals(binaryVersion))) { - log.info("Skipping already updated segment[%s].", segment); - return; + if (!force && (version.startsWith(segment.getVersion()) && CURR_VERSION_INTEGER.equals(binaryVersion))) { + log.info("Skipping already updated segment[%s].", segment); + return; + } } - } - final Map localSegments = toolbox.fetchSegments(Collections.singletonList(segment)); + final Map localSegments = toolbox.fetchSegments(Collections.singletonList(segment)); - final File location = localSegments.get(segment); - final File outLocation = new File(location, "v9_out"); - if (toolbox.getIndexIO().convertSegment(location, outLocation, indexSpec, force, validate)) { - final int outVersion = IndexIO.getVersionFromDir(outLocation); + final File location = localSegments.get(segment); + final File outLocation = new File(location, "v9_out"); + IndexIO indexIO = toolbox.getIndexIO(); + if (indexIO.convertSegment(location, outLocation, indexSpec, force, validate, segmentWriteOutMediumFactory)) { + final int outVersion = IndexIO.getVersionFromDir(outLocation); - // Appending to the version makes a new version that inherits most comparability parameters of the original - // version, but is "newer" than said original version. - DataSegment updatedSegment = segment.withVersion(StringUtils.format("%s_v%s", segment.getVersion(), outVersion)); - updatedSegment = toolbox.getSegmentPusher().push(outLocation, updatedSegment); + // Appending to the version makes a new version that inherits most comparability parameters of the original + // version, but is "newer" than said original version. + DataSegment updatedSegment = segment.withVersion(StringUtils.format("%s_v%s", segment.getVersion(), outVersion)); + updatedSegment = toolbox.getSegmentPusher().push(outLocation, updatedSegment); - actionClient.submit(new SegmentInsertAction(Sets.newHashSet(updatedSegment))); - } else { - log.info("Conversion failed."); + actionClient.submit(new SegmentInsertAction(Sets.newHashSet(updatedSegment))); + } else { + log.info("Conversion failed."); + } } } } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopConverterTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopConverterTask.java index 2a8e5a6b1483..c06455d76017 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopConverterTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopConverterTask.java @@ -34,10 +34,12 @@ import io.druid.indexing.common.actions.TaskActionClient; import io.druid.java.util.common.UOE; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.IndexSpec; import io.druid.timeline.DataSegment; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.io.IOException; import java.net.URI; import java.util.Arrays; @@ -52,6 +54,12 @@ public class HadoopConverterTask extends ConvertSegmentTask private static final String TYPE = "hadoop_convert_segment"; private static final Logger log = new Logger(HadoopConverterTask.class); + private final List hadoopDependencyCoordinates; + private final URI distributedSuccessCache; + private final String jobPriority; + private final String segmentOutputPath; + private final String classpathPrefix; + @JsonCreator public HadoopConverterTask( @JsonProperty("id") String id, @@ -65,6 +73,7 @@ public HadoopConverterTask( @JsonProperty("jobPriority") String jobPriority, @JsonProperty("segmentOutputPath") String segmentOutputPath, @JsonProperty("classpathPrefix") String classpathPrefix, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, @JsonProperty("context") Map context ) { @@ -81,6 +90,7 @@ public HadoopConverterTask( indexSpec, force, validate == null ? true : validate, + segmentWriteOutMediumFactory, context ); this.hadoopDependencyCoordinates = hadoopDependencyCoordinates; @@ -90,12 +100,6 @@ public HadoopConverterTask( this.classpathPrefix = classpathPrefix; } - private final List hadoopDependencyCoordinates; - private final URI distributedSuccessCache; - private final String jobPriority; - private final String segmentOutputPath; - private final String classpathPrefix; - @JsonProperty public List getHadoopDependencyCoordinates() { diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java index 93f6f105c359..1a9bade78c9f 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java @@ -57,6 +57,7 @@ import io.druid.java.util.common.guava.Comparators; import io.druid.java.util.common.logger.Logger; import io.druid.java.util.common.parsers.ParseException; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.DruidMetrics; import io.druid.segment.IndexSpec; import io.druid.segment.indexing.DataSchema; @@ -115,6 +116,22 @@ public class IndexTask extends AbstractTask private static final HashFunction hashFunction = Hashing.murmur3_128(); private static final String TYPE = "index"; + private static String makeGroupId(IndexIngestionSpec ingestionSchema) + { + return makeGroupId(ingestionSchema.ioConfig.appendToExisting, ingestionSchema.dataSchema.getDataSource()); + } + + private static String makeGroupId(boolean isAppendToExisting, String dataSource) + { + if (isAppendToExisting) { + // Shared locking group for all tasks that append, since they are OK to run concurrently. + return StringUtils.format("%s_append_%s", TYPE, dataSource); + } else { + // Return null, one locking group per task. + return null; + } + } + @JsonIgnore private final IndexIngestionSpec ingestionSchema; @@ -156,22 +173,6 @@ public IndexTask( this.ingestionSchema = ingestionSchema; } - private static String makeGroupId(IndexIngestionSpec ingestionSchema) - { - return makeGroupId(ingestionSchema.ioConfig.appendToExisting, ingestionSchema.dataSchema.getDataSource()); - } - - private static String makeGroupId(boolean isAppendToExisting, String dataSource) - { - if (isAppendToExisting) { - // Shared locking group for all tasks that append, since they are OK to run concurrently. - return StringUtils.format("%s_append_%s", TYPE, dataSource); - } else { - // Return null, one locking group per task. - return null; - } - } - @Override public int getPriority() { @@ -964,6 +965,8 @@ public static class IndexTuningConfig implements TuningConfig, AppenderatorConfi private final boolean forceGuaranteedRollup; private final boolean reportParseExceptions; private final long publishTimeout; + @Nullable + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; @JsonCreator public IndexTuningConfig( @@ -979,7 +982,8 @@ public IndexTuningConfig( @JsonProperty("forceExtendableShardSpecs") @Nullable Boolean forceExtendableShardSpecs, @JsonProperty("forceGuaranteedRollup") @Nullable Boolean forceGuaranteedRollup, @JsonProperty("reportParseExceptions") @Nullable Boolean reportParseExceptions, - @JsonProperty("publishTimeout") @Nullable Long publishTimeout + @JsonProperty("publishTimeout") @Nullable Long publishTimeout, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { this( @@ -993,13 +997,14 @@ public IndexTuningConfig( forceGuaranteedRollup, reportParseExceptions, publishTimeout, - null + null, + segmentWriteOutMediumFactory ); } private IndexTuningConfig() { - this(null, null, null, null, null, null, null, null, null, null, null); + this(null, null, null, null, null, null, null, null, null, null, null, null); } private IndexTuningConfig( @@ -1013,7 +1018,8 @@ private IndexTuningConfig( @Nullable Boolean forceGuaranteedRollup, @Nullable Boolean reportParseExceptions, @Nullable Long publishTimeout, - @Nullable File basePersistDirectory + @Nullable File basePersistDirectory, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { Preconditions.checkArgument( @@ -1047,6 +1053,8 @@ private IndexTuningConfig( !(this.forceExtendableShardSpecs && this.forceGuaranteedRollup), "Perfect rollup cannot be guaranteed with extendable shardSpecs" ); + + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } public IndexTuningConfig withBasePersistDirectory(File dir) @@ -1062,7 +1070,8 @@ public IndexTuningConfig withBasePersistDirectory(File dir) forceGuaranteedRollup, reportParseExceptions, publishTimeout, - dir + dir, + segmentWriteOutMediumFactory ); } @@ -1152,64 +1161,36 @@ public Period getIntermediatePersistPeriod() return new Period(Integer.MAX_VALUE); // intermediate persist doesn't make much sense for batch jobs } + @Nullable + @Override + @JsonProperty + public SegmentWriteOutMediumFactory getSegmentWriteOutMediumFactory() + { + return segmentWriteOutMediumFactory; + } + @Override public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { return false; } - - final IndexTuningConfig that = (IndexTuningConfig) o; - - if (!Objects.equals(targetPartitionSize, that.targetPartitionSize)) { - return false; - } - - if (maxRowsInMemory != that.maxRowsInMemory) { - return false; - } - - if (maxTotalRows != that.maxTotalRows) { - return false; - } - - if (!Objects.equals(numShards, that.numShards)) { - return false; - } - - if (!Objects.equals(indexSpec, that.indexSpec)) { - return false; - } - - if (!Objects.equals(basePersistDirectory, that.basePersistDirectory)) { - return false; - } - - if (maxPendingPersists != that.maxPendingPersists) { - return false; - } - - if (forceExtendableShardSpecs != that.forceExtendableShardSpecs) { - return false; - } - - if (forceGuaranteedRollup != that.forceGuaranteedRollup) { - return false; - } - - if (reportParseExceptions != that.reportParseExceptions) { - return false; - } - - if (publishTimeout != that.publishTimeout) { - return false; - } - - return true; + IndexTuningConfig that = (IndexTuningConfig) o; + return maxRowsInMemory == that.maxRowsInMemory && + maxTotalRows == that.maxTotalRows && + maxPendingPersists == that.maxPendingPersists && + forceExtendableShardSpecs == that.forceExtendableShardSpecs && + forceGuaranteedRollup == that.forceGuaranteedRollup && + reportParseExceptions == that.reportParseExceptions && + publishTimeout == that.publishTimeout && + Objects.equals(targetPartitionSize, that.targetPartitionSize) && + Objects.equals(numShards, that.numShards) && + Objects.equals(indexSpec, that.indexSpec) && + Objects.equals(basePersistDirectory, that.basePersistDirectory) && + Objects.equals(segmentWriteOutMediumFactory, that.segmentWriteOutMediumFactory); } @Override @@ -1226,7 +1207,8 @@ public int hashCode() forceExtendableShardSpecs, forceGuaranteedRollup, reportParseExceptions, - publishTimeout + publishTimeout, + segmentWriteOutMediumFactory ); } } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTask.java index d897df5c089f..fe9ca37123cc 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTask.java @@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import io.druid.indexing.common.TaskToolbox; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.IndexMerger; import io.druid.segment.IndexSpec; @@ -58,10 +59,11 @@ public MergeTask( @JsonProperty("indexSpec") IndexSpec indexSpec, // This parameter is left for compatibility when reading existing JSONs, to be removed in Druid 0.12. @JsonProperty("buildV9Directly") Boolean buildV9Directly, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, @JsonProperty("context") Map context ) { - super(id, dataSource, segments, context); + super(id, dataSource, segments, segmentWriteOutMediumFactory, context); this.aggregators = Preconditions.checkNotNull(aggregators, "null aggregations"); this.rollup = rollup == null ? Boolean.TRUE : rollup; this.indexSpec = indexSpec == null ? new IndexSpec() : indexSpec; @@ -92,7 +94,8 @@ public QueryableIndex apply(@Nullable File input) rollup, aggregators.toArray(new AggregatorFactory[aggregators.size()]), outDir, - indexSpec + indexSpec, + getSegmentWriteOutMediumFactory() ); } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java index 5efab3f0abb7..8a6cb91dea52 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java @@ -45,6 +45,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.IndexIO; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NoneShardSpec; @@ -61,15 +62,19 @@ */ public abstract class MergeTaskBase extends AbstractFixedIntervalTask { + private static final EmittingLogger log = new EmittingLogger(MergeTaskBase.class); + @JsonIgnore private final List segments; - - private static final EmittingLogger log = new EmittingLogger(MergeTaskBase.class); + @JsonIgnore + @Nullable + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; protected MergeTaskBase( final String id, final String dataSource, final List segments, + final @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, Map context ) { @@ -104,6 +109,7 @@ public boolean apply(@Nullable DataSegment segment) verifyInputSegments(segments); this.segments = segments; + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } protected void verifyInputSegments(List segments) @@ -254,6 +260,13 @@ public List getSegments() return segments; } + @JsonProperty + @Nullable + public SegmentWriteOutMediumFactory getSegmentWriteOutMediumFactory() + { + return segmentWriteOutMediumFactory; + } + @Override public String toString() { @@ -262,6 +275,7 @@ public String toString() .add("dataSource", getDataSource()) .add("interval", getInterval()) .add("segments", segments) + .add("segmentWriteOutMediumFactory", segmentWriteOutMediumFactory) .toString(); } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/SameIntervalMergeTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/SameIntervalMergeTask.java index d5f59af6a3a2..cdb60be12f6c 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/SameIntervalMergeTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/SameIntervalMergeTask.java @@ -26,11 +26,13 @@ import io.druid.indexing.common.TaskToolbox; import io.druid.indexing.common.actions.SegmentListUsedAction; import io.druid.java.util.common.DateTimes; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.IndexSpec; import io.druid.timeline.DataSegment; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.util.List; import java.util.Map; @@ -43,6 +45,8 @@ public class SameIntervalMergeTask extends AbstractFixedIntervalTask private final List aggregators; private final Boolean rollup; private final IndexSpec indexSpec; + @Nullable + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; public SameIntervalMergeTask( @JsonProperty("id") String id, @@ -54,6 +58,7 @@ public SameIntervalMergeTask( // This parameter is left for compatibility when reading existing JSONs, to be removed in Druid 0.12. @SuppressWarnings("unused") @JsonProperty("buildV9Directly") Boolean buildV9Directly, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, @JsonProperty("context") Map context ) { @@ -66,6 +71,7 @@ public SameIntervalMergeTask( this.aggregators = Preconditions.checkNotNull(aggregators, "null aggregations"); this.rollup = rollup == null ? Boolean.TRUE : rollup; this.indexSpec = indexSpec == null ? new IndexSpec() : indexSpec; + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } @JsonProperty("aggregations") @@ -130,6 +136,7 @@ public TaskStatus run(TaskToolbox toolbox) throws Exception aggregators, rollup, indexSpec, + segmentWriteOutMediumFactory, getContext() ); final TaskStatus status = mergeTask.run(toolbox); @@ -141,13 +148,14 @@ public TaskStatus run(TaskToolbox toolbox) throws Exception public static class SubTask extends MergeTask { - public SubTask( + private SubTask( String baseId, String dataSource, List segments, List aggregators, Boolean rollup, IndexSpec indexSpec, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, Map context ) { @@ -159,6 +167,7 @@ public SubTask( rollup, indexSpec, true, + segmentWriteOutMediumFactory, context ); } diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java index c2f2a1d78330..26cdfb718130 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java @@ -880,7 +880,7 @@ private boolean announceTask( task.getId(), elapsed, config.getTaskAssignmentTimeout() - ); + ).emit(); taskComplete(taskRunnerWorkItem, theZkWorker, TaskStatus.failure(task.getId())); break; } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/TestUtils.java b/indexing-service/src/test/java/io/druid/indexing/common/TestUtils.java index f655f57225c9..00c09435a9a2 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/TestUtils.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/TestUtils.java @@ -35,6 +35,7 @@ import io.druid.segment.column.ColumnConfig; import io.druid.segment.realtime.firehose.ChatHandlerProvider; import io.druid.segment.realtime.firehose.NoopChatHandlerProvider; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.server.security.AuthConfig; import io.druid.server.security.AuthorizerMapper; @@ -56,6 +57,7 @@ public TestUtils() this.jsonMapper = new DefaultObjectMapper(); indexIO = new IndexIO( jsonMapper, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -65,7 +67,7 @@ public int columnCacheSizeBytes() } } ); - indexMergerV9 = new IndexMergerV9(jsonMapper, indexIO); + indexMergerV9 = new IndexMergerV9(jsonMapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); final List list = new ServerModule().getJacksonModules(); for (Module module : list) { diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java index c707979fbce2..7bccdbab3d83 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java @@ -69,7 +69,7 @@ import io.druid.segment.column.Column; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ValueType; -import io.druid.segment.data.CompressedObjectStrategy.CompressionStrategy; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; import io.druid.segment.data.ListIndexed; import io.druid.segment.data.RoaringBitmapSerdeFactory; @@ -78,6 +78,7 @@ import io.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import io.druid.segment.loading.SegmentLoadingException; import io.druid.segment.transform.TransformingInputRowParser; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NumberedShardSpec; import org.hamcrest.CoreMatchers; @@ -240,7 +241,8 @@ private static IndexTuningConfig createTuningConfig() false, true, false, - 100L + 100L, + null ); } @@ -499,7 +501,7 @@ private static class TestTaskToolbox extends TaskToolbox indexIO, null, null, - new IndexMergerV9(objectMapper, indexIO), + new IndexMergerV9(objectMapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()), null, null, null, @@ -549,7 +551,7 @@ private static class TestIndexIO extends IndexIO Map segmentFileMap ) { - super(mapper, () -> 0); + super(mapper, OffHeapMemorySegmentWriteOutMediumFactory.instance(), () -> 0); queryableIndexMap = new HashMap<>(segmentFileMap.size()); for (Entry entry : segmentFileMap.entrySet()) { diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/ConvertSegmentTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/ConvertSegmentTaskTest.java index a512da941174..be94ec3bcbb6 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/ConvertSegmentTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/ConvertSegmentTaskTest.java @@ -51,7 +51,7 @@ public void testSerializationSimple() throws Exception DateTime start = DateTimes.nowUtc(); final Interval interval = new Interval(start.minus(1000), start); - ConvertSegmentTask task = ConvertSegmentTask.create(dataSource, interval, null, false, true, null); + ConvertSegmentTask task = ConvertSegmentTask.create(dataSource, interval, null, false, true, null, null); Task task2 = jsonMapper.readValue(jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(task), Task.class); Assert.assertEquals(task, task2); @@ -68,7 +68,7 @@ public void testSerializationSimple() throws Exception 102937 ); - task = ConvertSegmentTask.create(segment, null, false, true, null); + task = ConvertSegmentTask.create(segment, null, false, true, null, null); task2 = jsonMapper.readValue(jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(task), Task.class); Assert.assertEquals(task, task2); diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/HadoopConverterTaskSerDeTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/HadoopConverterTaskSerDeTest.java index be8885818c02..11a76b70098c 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/HadoopConverterTaskSerDeTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/HadoopConverterTaskSerDeTest.java @@ -25,8 +25,8 @@ import io.druid.indexing.common.TestUtils; import io.druid.java.util.common.Intervals; import io.druid.segment.IndexSpec; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NoneShardSpec; @@ -54,8 +54,8 @@ public class HadoopConverterTaskSerDeTest private static final int BINARY_VERSION = 34718; private static final long SEGMENT_SIZE = 7483901348790L; private static final IndexSpec INDEX_SPEC = new IndexSpec(new ConciseBitmapSerdeFactory(), - CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.LZF, + CompressionStrategy.LZ4, + CompressionStrategy.LZF, CompressionFactory.LongEncodingStrategy.LONGS); private static final DataSegment DATA_SEGMENT = new DataSegment( DATA_SOURCE, @@ -97,6 +97,7 @@ public void testSimpleConverterTaskSerDe() throws IOException PRIORITY, OUTPUT_PATH, CLASSPATH_PREFIX, + null, null ); final String strOrig = jsonMapper.writeValueAsString(orig); @@ -122,6 +123,7 @@ public void testSimpleSubTaskSerDe() throws IOException PRIORITY, OUTPUT_PATH, CLASSPATH_PREFIX, + null, null ); HadoopConverterTask.ConverterSubTask subTask = new HadoopConverterTask.ConverterSubTask( @@ -175,6 +177,7 @@ public void testSubTask() PRIORITY, OUTPUT_PATH, CLASSPATH_PREFIX, + null, null ); HadoopConverterTask.ConverterSubTask subTask = new HadoopConverterTask.ConverterSubTask( @@ -203,6 +206,7 @@ public void testNullValidate() PRIORITY, OUTPUT_PATH, CLASSPATH_PREFIX, + null, null ); Assert.assertTrue(orig.isValidate()); @@ -223,6 +227,7 @@ public void testMinimal() null, OUTPUT_PATH, null, + null, null ); Assert.assertEquals(DATA_SOURCE, parent.getDataSource()); @@ -248,6 +253,7 @@ public void testGetDataSegment() PRIORITY, OUTPUT_PATH, CLASSPATH_PREFIX, + null, null ); orig.getSegment(); @@ -268,6 +274,7 @@ public void testNull1() null, OUTPUT_PATH, null, + null, null ); } @@ -287,6 +294,7 @@ public void testNull2() null, OUTPUT_PATH, null, + null, null ); } @@ -306,6 +314,7 @@ public void testNull3() null, OUTPUT_PATH, null, + null, null ); } @@ -325,6 +334,7 @@ public void testNull4() null, null, null, + null, null ); } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java index f9c85918b9fa..79310c9771de 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java @@ -1147,6 +1147,7 @@ private static IndexTuningConfig createTuningConfig( forceExtendableShardSpecs, forceGuaranteedRollup, reportParseException, + null, null ); } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/MergeTaskBaseTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/MergeTaskBaseTest.java index 0a61e0bdbbd3..a51323803dce 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/MergeTaskBaseTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/MergeTaskBaseTest.java @@ -44,7 +44,7 @@ public class MergeTaskBaseTest .add(segmentBuilder.interval(Intervals.of("2012-01-03/2012-01-05")).build()) .build(); - final MergeTaskBase testMergeTaskBase = new MergeTaskBase(null, "foo", segments, null) + final MergeTaskBase testMergeTaskBase = new MergeTaskBase(null, "foo", segments, null, null) { @Override protected File merge(TaskToolbox toolbox, Map segments, File outDir) throws Exception diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index 13ecca4ac629..f7fb51847443 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -198,7 +198,7 @@ public boolean hasMore() public InputRow nextRow() { synchronized (this) { - final InputRow row = parser.parse(queue.remove(0)); + final InputRow row = parser.parseBatch(queue.remove(0)).get(0); if (row != null && row.getRaw(FAIL_DIM) != null) { throw new ParseException(FAIL_DIM); } @@ -937,6 +937,7 @@ private RealtimeIndexTask makeRealtimeTask( 0, reportParseExceptions, handoffTimeout, + null, null ); return new RealtimeIndexTask( diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java index bde1e9b12b0d..63389a804bae 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java @@ -91,6 +91,7 @@ public void testRun() throws Exception true, indexSpec, true, + null, null ); diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java index 83cdb0887c3d..b0484c882239 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java @@ -35,12 +35,13 @@ import io.druid.indexing.common.task.IndexTask.IndexTuningConfig; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.segment.IndexSpec; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.indexing.DataSchema; import io.druid.segment.indexing.RealtimeIOConfig; @@ -189,7 +190,7 @@ public void testIndexTaskSerde() throws Exception jsonMapper ), new IndexIOConfig(new LocalFirehoseFactory(new File("lol"), "rofl", null), true), - new IndexTuningConfig(10000, 10, null, 9999, null, indexSpec, 3, true, true, false, null, null) + new IndexTuningConfig(10000, 10, null, 9999, null, indexSpec, 3, true, true, false, null, null, null) ), null ); @@ -252,7 +253,7 @@ public void testIndexTaskwithResourceSerde() throws Exception jsonMapper ), new IndexIOConfig(new LocalFirehoseFactory(new File("lol"), "rofl", null), true), - new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null) + new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null, null) ), null ); @@ -298,6 +299,7 @@ public void testMergeTaskSerde() throws Exception true, indexSpec, true, + null, null ); @@ -347,6 +349,7 @@ public void testSameIntervalMergeTaskSerde() throws Exception true, indexSpec, true, + null, null ); @@ -414,6 +417,7 @@ public void testVersionConverterTaskSerde() throws Exception null, false, true, + TmpFileSegmentWriteOutMediumFactory.instance(), null ); @@ -429,7 +433,8 @@ public void testVersionConverterTaskSerde() throws Exception Assert.assertEquals(task.getGroupId(), task2.getGroupId()); Assert.assertEquals(task.getDataSource(), task2.getDataSource()); Assert.assertEquals(task.getInterval(), task2.getInterval()); - Assert.assertEquals(task.getSegment(), task.getSegment()); + Assert.assertEquals(task.getSegment(), task2.getSegment()); + Assert.assertEquals(task.getSegmentWriteOutMediumFactory(), task2.getSegmentWriteOutMediumFactory()); } @Test @@ -441,6 +446,7 @@ public void testVersionConverterSubTaskSerde() throws Exception indexSpec, false, true, + null, null ); @@ -503,6 +509,7 @@ public Plumber findPlumber( 0, true, null, + null, null ) ), @@ -567,6 +574,7 @@ public void testAppendTaskSerde() throws Exception ), indexSpec, true, + null, null ); @@ -665,6 +673,7 @@ public void testSegmentConvetSerdeReflection() throws IOException indexSpec, false, true, + TmpFileSegmentWriteOutMediumFactory.instance(), null ); final String json = jsonMapper.writeValueAsString(task); @@ -686,41 +695,43 @@ public void testSegmentConvertSerde() throws IOException 0, 12345L ); - final ConvertSegmentTask convertSegmentTaskOriginal = ConvertSegmentTask.create( + final ConvertSegmentTask originalTask = ConvertSegmentTask.create( segment, new IndexSpec( new RoaringBitmapSerdeFactory(null), - CompressedObjectStrategy.CompressionStrategy.LZF, - CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, + CompressionStrategy.LZF, + CompressionStrategy.UNCOMPRESSED, CompressionFactory.LongEncodingStrategy.LONGS ), false, true, + TmpFileSegmentWriteOutMediumFactory.instance(), null ); - final String json = jsonMapper.writeValueAsString(convertSegmentTaskOriginal); + final String json = jsonMapper.writeValueAsString(originalTask); final Task task = jsonMapper.readValue(json, Task.class); Assert.assertTrue(task instanceof ConvertSegmentTask); final ConvertSegmentTask convertSegmentTask = (ConvertSegmentTask) task; - Assert.assertEquals(convertSegmentTaskOriginal.getDataSource(), convertSegmentTask.getDataSource()); - Assert.assertEquals(convertSegmentTaskOriginal.getInterval(), convertSegmentTask.getInterval()); + Assert.assertEquals(originalTask.getDataSource(), convertSegmentTask.getDataSource()); + Assert.assertEquals(originalTask.getInterval(), convertSegmentTask.getInterval()); Assert.assertEquals( - convertSegmentTaskOriginal.getIndexSpec().getBitmapSerdeFactory().getClass().getCanonicalName(), + originalTask.getIndexSpec().getBitmapSerdeFactory().getClass().getCanonicalName(), convertSegmentTask.getIndexSpec() .getBitmapSerdeFactory() .getClass() .getCanonicalName() ); Assert.assertEquals( - convertSegmentTaskOriginal.getIndexSpec().getDimensionCompression(), + originalTask.getIndexSpec().getDimensionCompression(), convertSegmentTask.getIndexSpec().getDimensionCompression() ); Assert.assertEquals( - convertSegmentTaskOriginal.getIndexSpec().getMetricCompression(), + originalTask.getIndexSpec().getMetricCompression(), convertSegmentTask.getIndexSpec().getMetricCompression() ); Assert.assertEquals(false, convertSegmentTask.isForce()); Assert.assertEquals(segment, convertSegmentTask.getSegment()); + Assert.assertEquals(originalTask.getSegmentWriteOutMediumFactory(), convertSegmentTask.getSegmentWriteOutMediumFactory()); } @Test diff --git a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java index 8907f4d1a151..30a84e1ca85e 100644 --- a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java @@ -164,13 +164,13 @@ public static Collection constructorFeeder() throws IOException .buildOnheap(); for (Integer i = 0; i < MAX_ROWS; ++i) { - index.add(ROW_PARSER.parse(buildRow(i.longValue()))); + index.add(ROW_PARSER.parseBatch(buildRow(i.longValue())).get(0)); } if (!persistDir.mkdirs() && !persistDir.exists()) { throw new IOE("Could not create directory at [%s]", persistDir.getAbsolutePath()); } - INDEX_MERGER_V9.persist(index, persistDir, indexSpec); + INDEX_MERGER_V9.persist(index, persistDir, indexSpec, null); final IndexerSQLMetadataStorageCoordinator mdc = new IndexerSQLMetadataStorageCoordinator(null, null, null) { diff --git a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java index 912e741a57a8..92a88939d8da 100644 --- a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTimelineTest.java @@ -231,7 +231,7 @@ private static Map persist(File tmpDir, InputRow... rows) } try { - INDEX_MERGER_V9.persist(index, persistDir, new IndexSpec()); + INDEX_MERGER_V9.persist(index, persistDir, new IndexSpec(), null); } catch (IOException e) { throw Throwables.propagate(e); diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java index 23cad0663207..42b51ec5168e 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java @@ -670,7 +670,7 @@ public void testIndexTask() throws Exception mapper ), new IndexIOConfig(new MockFirehoseFactory(false), false), - new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null) + new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null, null) ), null ); @@ -728,7 +728,7 @@ public void testIndexTaskFailure() throws Exception mapper ), new IndexIOConfig(new MockExceptionalFirehoseFactory(), false), - new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null) + new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null, null) ), null ); @@ -1093,7 +1093,7 @@ public void testResumeTasks() throws Exception mapper ), new IndexIOConfig(new MockFirehoseFactory(false), false), - new IndexTuningConfig(10000, 10, null, null, null, indexSpec, null, false, null, null, null, null) + new IndexTuningConfig(10000, 10, null, null, null, indexSpec, null, false, null, null, null, null, null) ), null ); @@ -1215,6 +1215,7 @@ private RealtimeIndexTask newRealtimeIndexTask() 0, null, null, + null, null ); FireDepartment fireDepartment = new FireDepartment(dataSchema, realtimeIOConfig, realtimeTuningConfig); diff --git a/java-util/src/main/java/io/druid/java/util/common/collect/Utils.java b/java-util/src/main/java/io/druid/java/util/common/collect/Utils.java index 0a741858bd3e..392790665ff0 100644 --- a/java-util/src/main/java/io/druid/java/util/common/collect/Utils.java +++ b/java-util/src/main/java/io/druid/java/util/common/collect/Utils.java @@ -23,8 +23,11 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; +import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; public class Utils @@ -33,7 +36,8 @@ public static Map zipMap(K[] keys, V[] values) { Preconditions.checkArgument(values.length == keys.length, "number of values[%s] different than number of keys[%s]", - values.length, keys.length); + values.length, keys.length + ); return zipMapPartial(keys, values); } @@ -42,7 +46,8 @@ public static Map zipMapPartial(K[] keys, V[] values) { Preconditions.checkArgument(values.length <= keys.length, "number of values[%s] exceeds number of keys[%s]", - values.length, keys.length); + values.length, keys.length + ); Map retVal = new LinkedHashMap<>(); @@ -53,8 +58,10 @@ public static Map zipMapPartial(K[] keys, V[] values) return retVal; } - /** Create a Map from iterables of keys and values. Will throw an exception if there are more keys than values, - * or more values than keys. */ + /** + * Create a Map from iterables of keys and values. Will throw an exception if there are more keys than values, + * or more values than keys. + */ public static Map zipMap(Iterable keys, Iterable values) { Map retVal = new LinkedHashMap<>(); @@ -67,20 +74,24 @@ public static Map zipMap(Iterable keys, Iterable values) Preconditions.checkArgument(valsIter.hasNext(), "number of values[%s] less than number of keys, broke on key[%s]", - retVal.size(), key); + retVal.size(), key + ); retVal.put(key, valsIter.next()); } Preconditions.checkArgument(!valsIter.hasNext(), "number of values[%s] exceeds number of keys[%s]", - retVal.size() + Iterators.size(valsIter), retVal.size()); + retVal.size() + Iterators.size(valsIter), retVal.size() + ); return retVal; } - /** Create a Map from iterables of keys and values. If there are more keys than values, or more values than keys, - * the excess will be omitted. */ + /** + * Create a Map from iterables of keys and values. If there are more keys than values, or more values than keys, + * the excess will be omitted. + */ public static Map zipMapPartial(Iterable keys, Iterable values) { Map retVal = new LinkedHashMap<>(); @@ -101,4 +112,20 @@ public static Map zipMapPartial(Iterable keys, Iterable value return retVal; } + + @SafeVarargs + public static List nullableListOf(@Nullable T... elements) + { + final List list; + if (elements == null) { + list = new ArrayList<>(1); + list.add(null); + } else { + list = new ArrayList<>(elements.length); + for (T element : elements) { + list.add(element); + } + } + return list; + } } diff --git a/java-util/src/main/java/io/druid/java/util/common/io/smoosh/FileSmoosher.java b/java-util/src/main/java/io/druid/java/util/common/io/smoosh/FileSmoosher.java index 4dd66dba8823..2db064d9e6bf 100644 --- a/java-util/src/main/java/io/druid/java/util/common/io/smoosh/FileSmoosher.java +++ b/java-util/src/main/java/io/druid/java/util/common/io/smoosh/FileSmoosher.java @@ -46,7 +46,9 @@ import java.io.Writer; import java.nio.ByteBuffer; import java.nio.channels.Channels; +import java.nio.channels.FileChannel; import java.nio.channels.GatheringByteChannel; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -291,21 +293,20 @@ private SmooshedWriter delegateSmooshedWriter(final String name, final long size return new SmooshedWriter() { - private final FileOutputStream out = new FileOutputStream(tmpFile); - private final GatheringByteChannel channel = out.getChannel(); - private final Closer closer = Closer.create(); + private final GatheringByteChannel channel = + FileChannel.open( + tmpFile.toPath(), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); private int currOffset = 0; - { - closer.register(out); - closer.register(channel); - } - @Override public void close() throws IOException { - closer.close(); + channel.close(); completedFiles.add(tmpFile); filesInProcess.remove(tmpFile); diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/AbstractFlatTextFormatParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/AbstractFlatTextFormatParser.java index 7d3f43c9c336..cc25a7d6325c 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/AbstractFlatTextFormatParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/AbstractFlatTextFormatParser.java @@ -127,7 +127,7 @@ public void setFieldNames(final String header) } @Override - public Map parse(final String input) + public Map parseToMap(final String input) { if (!supportSkipHeaderRows && (hasHeaderRow || maxSkipHeaderRows > 0)) { throw new UnsupportedOperationException("hasHeaderRow or maxSkipHeaderRows is not supported. " diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java index 7cf75daac442..d2e54eed78e6 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java @@ -67,7 +67,7 @@ public void setFieldNames(Iterable fieldNames) * @return A map of field names and values */ @Override - public Map parse(String input) + public Map parseToMap(String input) { try { JsonNode document = mapper.readValue(input, JsonNode.class); diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONToLowerParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONToLowerParser.java index 714ef7b72492..5a04d9b75bf5 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONToLowerParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONToLowerParser.java @@ -109,7 +109,7 @@ public void setFieldNames(Iterable fieldNames) } @Override - public Map parse(String input) + public Map parseToMap(String input) { try { Map map = new LinkedHashMap<>(); diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JavaScriptParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JavaScriptParser.java index 6cbb6e430516..8ad76fc42367 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JavaScriptParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JavaScriptParser.java @@ -74,7 +74,7 @@ public Function getFn() } @Override - public Map parse(String input) + public Map parseToMap(String input) { try { final Object compiled = fn.apply(input); diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/Parser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/Parser.java index 8cc6fd6a1d6b..d39ef6ecaa15 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/Parser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/Parser.java @@ -43,7 +43,7 @@ default void startFileFromBeginning() * @throws ParseException if the String cannot be parsed */ @Nullable - Map parse(String input); + Map parseToMap(String input); /** * Set the fieldNames that you expect to see in parsed Maps. Deprecated; Parsers should not, in general, be diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/Parsers.java b/java-util/src/main/java/io/druid/java/util/common/parsers/Parsers.java index 733dae3e49bc..9c08feddd1b5 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/Parsers.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/Parsers.java @@ -40,7 +40,7 @@ public static Function> toFunction(final Parser p) public Map apply(String input) { try { - return p.parse(input); + return p.parseToMap(input); } catch (Exception e) { return null; diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/RegexParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/RegexParser.java index c9ca448ce98d..fa92bc22907b 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/RegexParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/RegexParser.java @@ -84,7 +84,7 @@ public RegexParser( } @Override - public Map parse(String input) + public Map parseToMap(String input) { try { final Matcher matcher = compiled.matcher(input); diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/ToLowerCaseParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/ToLowerCaseParser.java index 6363e7d7d18c..b95302b63a0d 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/ToLowerCaseParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/ToLowerCaseParser.java @@ -37,9 +37,9 @@ public ToLowerCaseParser(Parser baseParser) } @Override - public Map parse(String input) + public Map parseToMap(String input) { - Map line = baseParser.parse(input); + Map line = baseParser.parseToMap(input); Map retVal = Maps.newLinkedHashMap(); for (Map.Entry entry : line.entrySet()) { String k = StringUtils.toLowerCase(entry.getKey()); diff --git a/java-util/src/test/java/io/druid/java/util/common/parsers/FlatTextFormatParserTest.java b/java-util/src/test/java/io/druid/java/util/common/parsers/FlatTextFormatParserTest.java index 29a410814986..bfb54dab3159 100644 --- a/java-util/src/test/java/io/druid/java/util/common/parsers/FlatTextFormatParserTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/parsers/FlatTextFormatParserTest.java @@ -86,7 +86,7 @@ public void testWithHeader() final String header = concat(format, "time", "value1", "value2"); final Parser parser = parserFactory.get(format, header); final String body = concat(format, "hello", "world", "foo"); - final Map jsonMap = parser.parse(body); + final Map jsonMap = parser.parseToMap(body); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "value1", "world", "value2", "foo"), @@ -99,7 +99,7 @@ public void testWithoutHeader() { final Parser parser = parserFactory.get(format); final String body = concat(format, "hello", "world", "foo"); - final Map jsonMap = parser.parse(body); + final Map jsonMap = parser.parseToMap(body); Assert.assertEquals( "jsonMap", ImmutableMap.of("column_1", "hello", "column_2", "world", "column_3", "foo"), @@ -120,9 +120,9 @@ public void testWithSkipHeaderRows() }; int index; for (index = 0; index < skipHeaderRows; index++) { - Assert.assertNull(parser.parse(body[index])); + Assert.assertNull(parser.parseToMap(body[index])); } - final Map jsonMap = parser.parse(body[index]); + final Map jsonMap = parser.parseToMap(body[index]); Assert.assertEquals( "jsonMap", ImmutableMap.of("column_1", "hello", "column_2", "world", "column_3", "foo"), @@ -139,8 +139,8 @@ public void testWithHeaderRow() concat(format, "time", "value1", "value2"), concat(format, "hello", "world", "foo") }; - Assert.assertNull(parser.parse(body[0])); - final Map jsonMap = parser.parse(body[1]); + Assert.assertNull(parser.parseToMap(body[0])); + final Map jsonMap = parser.parseToMap(body[1]); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "value1", "world", "value2", "foo"), @@ -157,8 +157,8 @@ public void testWithHeaderRowOfEmptyColumns() concat(format, "time", "", "value2", ""), concat(format, "hello", "world", "foo", "bar") }; - Assert.assertNull(parser.parse(body[0])); - final Map jsonMap = parser.parse(body[1]); + Assert.assertNull(parser.parseToMap(body[0])); + final Map jsonMap = parser.parseToMap(body[1]); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "column_2", "world", "value2", "foo", "column_4", "bar"), @@ -175,8 +175,8 @@ public void testWithDifferentHeaderRows() concat(format, "time", "value1", "value2"), concat(format, "hello", "world", "foo") }; - Assert.assertNull(parser.parse(body[0])); - Map jsonMap = parser.parse(body[1]); + Assert.assertNull(parser.parseToMap(body[0])); + Map jsonMap = parser.parseToMap(body[1]); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "value1", "world", "value2", "foo"), @@ -188,8 +188,8 @@ public void testWithDifferentHeaderRows() concat(format, "time", "value1", "value2", "value3"), concat(format, "hello", "world", "foo", "bar") }; - Assert.assertNull(parser.parse(body2[0])); - jsonMap = parser.parse(body2[1]); + Assert.assertNull(parser.parseToMap(body2[0])); + jsonMap = parser.parseToMap(body2[1]); Assert.assertEquals( "jsonMap", ImmutableMap.of("time", "hello", "value1", "world", "value2", "foo", "value3", "bar"), @@ -212,7 +212,7 @@ public void testWithoutStartFileFromBeginning() concat(format, "header", "line", "2"), concat(format, "hello", "world", "foo") }; - parser.parse(body[0]); + parser.parseToMap(body[0]); } private static class FlatTextFormatParserFactory diff --git a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java index a0eee328c3d7..06bcdf69e29f 100644 --- a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java @@ -57,7 +57,7 @@ public void testSimple() { List fields = new ArrayList<>(); final Parser jsonParser = new JSONPathParser(new JSONPathSpec(true, fields), null); - final Map jsonMap = jsonParser.parse(json); + final Map jsonMap = jsonParser.parseToMap(json); Assert.assertEquals( "jsonMap", ImmutableMap.of("one", "foo", "two", ImmutableList.of("bar", "baz"), "three", "qux"), @@ -70,7 +70,7 @@ public void testWithNumbers() { List fields = new ArrayList<>(); final Parser jsonParser = new JSONPathParser(new JSONPathSpec(true, fields), null); - final Map jsonMap = jsonParser.parse(numbersJson); + final Map jsonMap = jsonParser.parseToMap(numbersJson); Assert.assertEquals( "jsonMap", ImmutableMap.of("five", 5.0, "six", 6L, "many", 1234567878900L, "toomany", 1.23456789E21), @@ -83,7 +83,7 @@ public void testWithWhackyCharacters() { List fields = new ArrayList<>(); final Parser jsonParser = new JSONPathParser(new JSONPathSpec(true, fields), null); - final Map jsonMap = jsonParser.parse(whackyCharacterJson); + final Map jsonMap = jsonParser.parseToMap(whackyCharacterJson); Assert.assertEquals( "jsonMap", ImmutableMap.of("one", "foo?"), @@ -113,7 +113,7 @@ public void testNestingWithFieldDiscovery() final Parser jsonParser = new JSONPathParser(new JSONPathSpec(true, fields), null); - final Map jsonMap = jsonParser.parse(nestedJson); + final Map jsonMap = jsonParser.parseToMap(nestedJson); // Root fields Assert.assertEquals(ImmutableList.of(1L, 2L, 3L), jsonMap.get("baz")); @@ -174,7 +174,7 @@ public void testNestingNoDiscovery() fields.add(new JSONPathFieldSpec(JSONPathFieldType.JQ, "jq-met-array", ".met.a")); final Parser jsonParser = new JSONPathParser(new JSONPathSpec(false, fields), null); - final Map jsonMap = jsonParser.parse(nestedJson); + final Map jsonMap = jsonParser.parseToMap(nestedJson); // Root fields Assert.assertEquals("text", jsonMap.get("simpleVal")); @@ -211,7 +211,7 @@ public void testRejectDuplicates() thrown.expectMessage("Cannot have duplicate field definition: met-array"); final Parser jsonParser = new JSONPathParser(new JSONPathSpec(false, fields), null); - final Map jsonMap = jsonParser.parse(nestedJson); + final Map jsonMap = jsonParser.parseToMap(nestedJson); } @Test @@ -225,7 +225,7 @@ public void testRejectDuplicates2() thrown.expectMessage("Cannot have duplicate field definition: met-array"); final Parser jsonParser = new JSONPathParser(new JSONPathSpec(false, fields), null); - final Map jsonMap = jsonParser.parse(nestedJson); + final Map jsonMap = jsonParser.parseToMap(nestedJson); } @Test @@ -237,6 +237,6 @@ public void testParseFail() thrown.expectMessage("Unable to parse row [" + notJson + "]"); final Parser jsonParser = new JSONPathParser(new JSONPathSpec(true, fields), null); - final Map jsonMap = jsonParser.parse(notJson); + final Map jsonMap = jsonParser.parseToMap(notJson); } } diff --git a/java-util/src/test/java/io/druid/java/util/common/parsers/JavaScriptParserTest.java b/java-util/src/test/java/io/druid/java/util/common/parsers/JavaScriptParserTest.java index a0bf7d6346e0..839f5485261f 100644 --- a/java-util/src/test/java/io/druid/java/util/common/parsers/JavaScriptParserTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/parsers/JavaScriptParserTest.java @@ -41,7 +41,7 @@ public void testParse() ); String data = "foo-val1"; - final Map parsed = parser.parse(data); + final Map parsed = parser.parseToMap(data); ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put("one", "foo"); builder.put("two", "val1"); @@ -62,7 +62,7 @@ public void testParseWithMultiVal() ); String data = "val1-val2"; - final Map parsed = parser.parse(data); + final Map parsed = parser.parseToMap(data); ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put("one", Lists.newArrayList("val1", "val2")); Assert.assertEquals( diff --git a/java-util/src/test/java/io/druid/java/util/common/parsers/RegexParserTest.java b/java-util/src/test/java/io/druid/java/util/common/parsers/RegexParserTest.java index 30eba31cdb96..641b6e8120d0 100644 --- a/java-util/src/test/java/io/druid/java/util/common/parsers/RegexParserTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/parsers/RegexParserTest.java @@ -66,7 +66,7 @@ public void testAWSLog() ); String data = "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be mybucket [06/Feb/2014:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 3E57427F3EXAMPLE REST.GET.VERSIONING - \"GET /mybucket?versioning HTTP/1.1\" 200 - 113 - 7 - \"-\" \"S3Console/0.4\" -"; - final Map parsed = parser.parse(data); + final Map parsed = parser.parseToMap(data); ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put("Bucket Owner", "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"); builder.put("Bucket", "mybucket"); @@ -127,7 +127,7 @@ public void testAWSLogWithCrazyUserAgent() ); String data = "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be mybucket [06/Feb/2014:00:01:00 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 7B4A0FABBEXAMPLE REST.GET.VERSIONING - \"GET /mybucket?versioning HTTP/1.1\" 200 - 139 139 27 26 \"-\" \"() { foo;};echo; /bin/bash -c \"expr 299663299665 / 3; echo 333:; uname -a; echo 333:; id;\"\" -"; - final Map parsed = parser.parse(data); + final Map parsed = parser.parseToMap(data); ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put("Bucket Owner", "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"); builder.put("Bucket", "mybucket"); @@ -175,7 +175,7 @@ public void testMultiVal() ); String data = "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be mybucket@mybucket2"; - final Map parsed = parser.parse(data); + final Map parsed = parser.parseToMap(data); ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put("Bucket Owner", "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"); builder.put("Bucket", Lists.newArrayList("mybucket", "mybucket2")); @@ -199,7 +199,7 @@ public void testMultiValWithRegexSplit() ); String data = "1a2"; - final Map parsed = parser.parse(data); + final Map parsed = parser.parseToMap(data); ImmutableMap.Builder builder = ImmutableMap.builder(); builder.put("column_1", Lists.newArrayList("1", "2")); @@ -226,6 +226,6 @@ public void testFailure() ); String data = "BBBB"; - parser.parse(data); + parser.parseToMap(data); } } diff --git a/pom.xml b/pom.xml index b243e3ec8796..fab82a8f5703 100644 --- a/pom.xml +++ b/pom.xml @@ -227,7 +227,7 @@ com.ning compress-lzf - 1.0.3 + 1.0.4 io.airlift @@ -709,6 +709,8 @@ asm-commons 5.2 + + org.codehaus.jackson jackson-core-asl diff --git a/processing/pom.xml b/processing/pom.xml index 54963de0fb8c..fc084247c28c 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -35,6 +35,11 @@ druid-common ${project.parent.version} + + io.druid + java-util + ${project.parent.version} + io.druid druid-hll diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index 4f37e4edaf2b..a67a24b7bbb7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -74,6 +74,14 @@ public class AggregatorUtil public static final byte TIMESTAMP_CACHE_TYPE_ID = 0x19; public static final byte VARIANCE_CACHE_TYPE_ID = 0x1A; + // Quantiles sketch aggregator + public static final byte QUANTILES_DOUBLES_SKETCH_BUILD_CACHE_TYPE_ID = 0x1B; + public static final byte QUANTILES_DOUBLES_SKETCH_MERGE_CACHE_TYPE_ID = 0x1C; + public static final byte QUANTILES_DOUBLES_SKETCH_TO_HISTOGRAM_CACHE_TYPE_ID = 0x1D; + public static final byte QUANTILES_DOUBLES_SKETCH_TO_QUANTILE_CACHE_TYPE_ID = 0x1E; + public static final byte QUANTILES_DOUBLES_SKETCH_TO_QUANTILES_CACHE_TYPE_ID = 0x1F; + public static final byte QUANTILES_DOUBLES_SKETCH_TO_STRING_CACHE_TYPE_ID = 0x20; + /** * returns the list of dependent postAggregators that should be calculated in order to calculate given postAgg * diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java index 26fda75b9aa3..93b5aacea0ab 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java @@ -23,10 +23,10 @@ import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogCollector; import io.druid.hll.HyperLogLogHash; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; import io.druid.segment.serde.ComplexColumnPartSupplier; import io.druid.segment.serde.ComplexMetricExtractor; @@ -148,9 +148,9 @@ public int compare(HyperLogLogCollector o1, HyperLogLogCollector o2) } @Override - public GenericColumnSerializer getSerializer(IOPeon peon, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { - return LargeColumnSupportedComplexColumnSerializer.create(peon, column, this.getObjectStrategy()); + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } } diff --git a/processing/src/main/java/io/druid/query/expression/ExprUtils.java b/processing/src/main/java/io/druid/query/expression/ExprUtils.java index d4d3e4ae816a..0a78c20e5d2e 100644 --- a/processing/src/main/java/io/druid/query/expression/ExprUtils.java +++ b/processing/src/main/java/io/druid/query/expression/ExprUtils.java @@ -29,6 +29,8 @@ import org.joda.time.Period; import org.joda.time.chrono.ISOChronology; +import javax.annotation.Nullable; + public class ExprUtils { private static final Expr.ObjectBinding NIL_BINDINGS = name -> null; @@ -50,8 +52,8 @@ public static DateTimeZone toTimeZone(final Expr timeZoneArg) public static PeriodGranularity toPeriodGranularity( final Expr periodArg, - final Expr originArg, - final Expr timeZoneArg, + @Nullable final Expr originArg, + @Nullable final Expr timeZoneArg, final Expr.ObjectBinding bindings ) { diff --git a/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java index fd81e51fc947..984fb8592c08 100644 --- a/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java @@ -21,7 +21,6 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; -import io.druid.java.util.common.granularity.Granularity; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -52,7 +51,7 @@ public Expr apply(final List args) } } - private static PeriodGranularity getGranularity(final List args, final Expr.ObjectBinding bindings) + private static PeriodGranularity computeGranularity(final List args, final Expr.ObjectBinding bindings) { return ExprUtils.toPeriodGranularity( args.get(1), @@ -62,15 +61,31 @@ private static PeriodGranularity getGranularity(final List args, final Exp ); } - private static class TimestampFloorExpr implements Expr + public static class TimestampFloorExpr implements Expr { private final Expr arg; - private final Granularity granularity; + private final PeriodGranularity granularity; public TimestampFloorExpr(final List args) { this.arg = args.get(0); - this.granularity = getGranularity(args, ExprUtils.nilBindings()); + this.granularity = computeGranularity(args, ExprUtils.nilBindings()); + } + + /** + * Exposed for Druid SQL: this is used by Expressions.toQueryGranularity. + */ + public Expr getArg() + { + return arg; + } + + /** + * Exposed for Druid SQL: this is used by Expressions.toQueryGranularity. + */ + public PeriodGranularity getGranularity() + { + return granularity; } @Nonnull @@ -88,7 +103,7 @@ public void visit(final Visitor visitor) } } - private static class TimestampFloorDynamicExpr implements Expr + public static class TimestampFloorDynamicExpr implements Expr { private final List args; @@ -101,7 +116,7 @@ public TimestampFloorDynamicExpr(final List args) @Override public ExprEval eval(final ObjectBinding bindings) { - final PeriodGranularity granularity = getGranularity(args, bindings); + final PeriodGranularity granularity = computeGranularity(args, bindings); return ExprEval.of(granularity.bucketStart(DateTimes.utc(args.get(0).eval(bindings).asLong())).getMillis()); } diff --git a/processing/src/main/java/io/druid/query/topn/Generic1AggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/Generic1AggPooledTopNScannerPrototype.java index d98d1fbd2ee0..9163eb5ceaca 100644 --- a/processing/src/main/java/io/druid/query/topn/Generic1AggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/Generic1AggPooledTopNScannerPrototype.java @@ -28,6 +28,18 @@ public final class Generic1AggPooledTopNScannerPrototype implements Generic1AggPooledTopNScanner { + /** + * Any changes to this method should be coordinated with {@link TopNUtils}, {@link + * PooledTopNAlgorithm#computeSpecializedScanAndAggregateImplementations} and downstream methods. + * + * It should be checked with a tool like https://github.com/AdoptOpenJDK/jitwatch that C2 compiler output for this + * method doesn't have any method calls in the while loop, i. e. all method calls are inlined. To be able to see + * assembly of this method in JITWatch and other similar tools, {@link + * PooledTopNAlgorithm#specializeGeneric1AggPooledTopN} should be turned off. Note that in this case the benchmark + * should be "naturally monomorphic", i. e. execute this method always with the same runtime shape. + * + * If the while loop contains not inlined method calls, it should be considered as a performance bug. + */ @Override public long scanAndAggregate( DimensionSelector dimensionSelector, diff --git a/processing/src/main/java/io/druid/query/topn/Generic2AggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/Generic2AggPooledTopNScannerPrototype.java index 491cb71f7669..402535e6dd39 100644 --- a/processing/src/main/java/io/druid/query/topn/Generic2AggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/Generic2AggPooledTopNScannerPrototype.java @@ -28,6 +28,18 @@ public final class Generic2AggPooledTopNScannerPrototype implements Generic2AggPooledTopNScanner { + /** + * Any changes to this method should be coordinated with {@link TopNUtils}, {@link + * PooledTopNAlgorithm#computeSpecializedScanAndAggregateImplementations} and downstream methods. + * + * It should be checked with a tool like https://github.com/AdoptOpenJDK/jitwatch that C2 compiler output for this + * method doesn't have any method calls in the while loop, i. e. all method calls are inlined. To be able to see + * assembly of this method in JITWatch and other similar tools, {@link + * PooledTopNAlgorithm#specializeGeneric2AggPooledTopN} should be turned off. Note that in this case the benchmark + * should be "naturally monomorphic", i. e. execute this method always with the same runtime shape. + * + * If the while loop contains not inlined method calls, it should be considered as a performance bug. + */ @Override public long scanAndAggregate( DimensionSelector dimensionSelector, diff --git a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java index 173cb9914e5b..a3106f7e2a2d 100644 --- a/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/Historical1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -35,6 +35,18 @@ public class Historical1SimpleDoubleAggPooledTopNScannerPrototype SimpleDoubleBufferAggregator > { + /** + * Any changes to this method should be coordinated with {@link TopNUtils}, {@link + * PooledTopNAlgorithm#computeSpecializedScanAndAggregateImplementations} and downstream methods. + * + * It should be checked with a tool like https://github.com/AdoptOpenJDK/jitwatch that C2 compiler output for this + * method doesn't have any method calls in the while loop, i. e. all method calls are inlined. To be able to see + * assembly of this method in JITWatch and other similar tools, {@link + * PooledTopNAlgorithm#specializeHistorical1SimpleDoubleAggPooledTopN} should be turned off. Note that in this case + * the benchmark should be "naturally monomorphic", i. e. execute this method always with the same runtime shape. + * + * If the while loop contains not inlined method calls, it should be considered as a performance bug. + */ @Override public long scanAndAggregate( HistoricalDimensionSelector dimensionSelector, diff --git a/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java index 10ca29f51beb..916e3c4a0ee4 100644 --- a/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java +++ b/processing/src/main/java/io/druid/query/topn/HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype.java @@ -34,6 +34,19 @@ public class HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPr SimpleDoubleBufferAggregator > { + /** + * Any changes to this method should be coordinated with {@link TopNUtils}, {@link + * PooledTopNAlgorithm#computeSpecializedScanAndAggregateImplementations} and downstream methods. + * + * It should be checked with a tool like https://github.com/AdoptOpenJDK/jitwatch that C2 compiler output for this + * method doesn't have any method calls in the while loop, i. e. all method calls are inlined. To be able to see + * assembly of this method in JITWatch and other similar tools, {@link + * PooledTopNAlgorithm#specializeHistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopN} should be turned off. + * Note that in this case the benchmark should be "naturally monomorphic", i. e. execute this method always with the + * same runtime shape. + * + * If the while loop contains not inlined method calls, it should be considered as a performance bug. + */ @Override public long scanAndAggregate( SingleValueHistoricalDimensionSelector dimensionSelector, diff --git a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java index ab40484f8dd2..d1bbf954d95c 100644 --- a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java @@ -36,6 +36,7 @@ import io.druid.segment.Capabilities; import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; +import io.druid.segment.FilteredOffset; import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.Offset; @@ -128,17 +129,21 @@ private static void computeSpecializedScanAndAggregateImplementations() if (theAggregators.length == 1) { BufferAggregator aggregator = theAggregators[0]; final Cursor cursor = params.getCursor(); - if (cursor instanceof HistoricalCursor && aggregator instanceof SimpleDoubleBufferAggregator) { - if (params.getDimSelector() instanceof SingleValueHistoricalDimensionSelector && - ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalColumnSelector) { - return scanAndAggregateHistorical1SimpleDoubleAgg( - params, - positions, - (SimpleDoubleBufferAggregator) aggregator, - (HistoricalCursor) cursor, - defaultHistoricalSingleValueDimSelector1SimpleDoubleAggScanner - ); - } + if (cursor instanceof HistoricalCursor && + // FilteredOffset.clone() is not supported. This condition should be removed if + // HistoricalSingleValueDimSelector1SimpleDoubleAggPooledTopNScannerPrototype + // doesn't clone offset anymore. + !(((HistoricalCursor) cursor).getOffset() instanceof FilteredOffset) && + aggregator instanceof SimpleDoubleBufferAggregator && + params.getDimSelector() instanceof SingleValueHistoricalDimensionSelector && + ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalColumnSelector) { + return scanAndAggregateHistorical1SimpleDoubleAgg( + params, + positions, + (SimpleDoubleBufferAggregator) aggregator, + (HistoricalCursor) cursor, + defaultHistoricalSingleValueDimSelector1SimpleDoubleAggScanner + ); } } return -1; @@ -149,17 +154,21 @@ private static void computeSpecializedScanAndAggregateImplementations() if (theAggregators.length == 1) { BufferAggregator aggregator = theAggregators[0]; final Cursor cursor = params.getCursor(); - if (cursor instanceof HistoricalCursor && aggregator instanceof SimpleDoubleBufferAggregator) { - if (params.getDimSelector() instanceof HistoricalDimensionSelector && - ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalColumnSelector) { - return scanAndAggregateHistorical1SimpleDoubleAgg( - params, - positions, - (SimpleDoubleBufferAggregator) aggregator, - (HistoricalCursor) cursor, - defaultHistorical1SimpleDoubleAggScanner - ); - } + if (cursor instanceof HistoricalCursor && + // FilteredOffset.clone() is not supported. This condition should be removed if + // Historical1SimpleDoubleAggPooledTopNScannerPrototype + // doesn't clone offset anymore. + !(((HistoricalCursor) cursor).getOffset() instanceof FilteredOffset) && + aggregator instanceof SimpleDoubleBufferAggregator && + params.getDimSelector() instanceof HistoricalDimensionSelector && + ((SimpleDoubleBufferAggregator) aggregator).getSelector() instanceof HistoricalColumnSelector) { + return scanAndAggregateHistorical1SimpleDoubleAgg( + params, + positions, + (SimpleDoubleBufferAggregator) aggregator, + (HistoricalCursor) cursor, + defaultHistorical1SimpleDoubleAggScanner + ); } } return -1; diff --git a/processing/src/main/java/io/druid/query/topn/TopNUtils.java b/processing/src/main/java/io/druid/query/topn/TopNUtils.java index 08afbcb3afdc..c1fe093898cb 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNUtils.java +++ b/processing/src/main/java/io/druid/query/topn/TopNUtils.java @@ -35,6 +35,10 @@ final class TopNUtils * Casting to the specific Offset subtype helps Hotspot JIT (OpenJDK 8) to generate better assembly. It shouldn't be * so, because the Offset subtype is still always the same (otherwise cast wouldn't be possible), so JIT should * generate equivalent code. In OpenJDK 9 Hotspot could be improved and this "casting hack" is not needed anymore. + * + * TODO check if offset.clone() is also necessary for generating better assembly, or it could be removed. + * + * See also javadoc comments to methods, where this method is used. */ static Object copyOffset(HistoricalCursor cursor) { diff --git a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java index 8e1079cf63f9..45043e18a47c 100644 --- a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedSupplier.java @@ -19,23 +19,26 @@ package io.druid.segment; +import com.google.common.annotations.VisibleForTesting; +import io.druid.io.Channels; import io.druid.java.util.common.IAE; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.Closer; +import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.IndexedIterable; import io.druid.segment.data.IndexedMultivalue; import io.druid.segment.data.WritableSupplier; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; /** * Format - @@ -55,7 +58,7 @@ public class CompressedVSizeIndexedSupplier implements WritableSupplier objectsIterable, - int maxValue, + final Iterable objectsIterable, + final int maxValue, final ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression + final CompressionStrategy compression, + final Closer closer ) { Iterator objects = objectsIterable.iterator(); - List offsetList = new ArrayList<>(); - List values = new ArrayList<>(); + IntList offsetList = new IntArrayList(); + IntList values = new IntArrayList(); int offset = 0; while (objects.hasNext()) { @@ -125,14 +128,16 @@ public static CompressedVSizeIndexedSupplier fromIterable( offsetMax, CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForValue(offsetMax), byteOrder, - compression + compression, + closer ); CompressedVSizeIntsIndexedSupplier valuesSupplier = CompressedVSizeIntsIndexedSupplier.fromList( values, maxValue, CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForValue(maxValue), byteOrder, - compression + compression, + closer ); return new CompressedVSizeIndexedSupplier(headerSupplier, valuesSupplier); } diff --git a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedV3Supplier.java b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedV3Supplier.java index 26e10afa5543..6d4f0330942e 100644 --- a/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedV3Supplier.java +++ b/processing/src/main/java/io/druid/segment/CompressedVSizeIndexedV3Supplier.java @@ -19,22 +19,23 @@ package io.druid.segment; +import com.google.common.annotations.VisibleForTesting; import io.druid.java.util.common.IAE; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.Closer; +import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.data.CompressedIntsIndexedSupplier; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.IndexedMultivalue; import io.druid.segment.data.WritableSupplier; +import it.unimi.dsi.fastutil.ints.IntArrayList; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; /** * The format is mostly the same with CompressedVSizeIndexedSupplier(which has version 0x2, so we call it V2), @@ -50,7 +51,7 @@ public class CompressedVSizeIndexedV3Supplier implements WritableSupplier objectsIterable, - int offsetChunkFactor, - int maxValue, + final Iterable objectsIterable, + final int offsetChunkFactor, + final int maxValue, final ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression + final CompressionStrategy compression, + final Closer closer ) { Iterator objects = objectsIterable.iterator(); - List offsetList = new ArrayList<>(); - List values = new ArrayList<>(); + IntArrayList offsetList = new IntArrayList(); + IntArrayList values = new IntArrayList(); int offset = 0; while (objects.hasNext()) { @@ -110,30 +106,32 @@ public static CompressedVSizeIndexedV3Supplier fromIterable( offsetList, offsetChunkFactor, byteOrder, - compression + compression, + closer ); CompressedVSizeIntsIndexedSupplier valuesSupplier = CompressedVSizeIntsIndexedSupplier.fromList( values, maxValue, CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForValue(maxValue), byteOrder, - compression + compression, + closer ); return new CompressedVSizeIndexedV3Supplier(headerSupplier, valuesSupplier); } @Override - public long getSerializedSize() + public long getSerializedSize() throws IOException { return 1 + offsetSupplier.getSerializedSize() + valueSupplier.getSerializedSize(); } @Override - public void writeToChannel(WritableByteChannel channel) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { channel.write(ByteBuffer.wrap(new byte[]{VERSION})); - offsetSupplier.writeToChannel(channel); - valueSupplier.writeToChannel(channel); + offsetSupplier.writeTo(channel, smoosher); + valueSupplier.writeTo(channel, smoosher); } @Override diff --git a/processing/src/main/java/io/druid/segment/DimensionHandler.java b/processing/src/main/java/io/druid/segment/DimensionHandler.java index a44e1d78efa9..d3b79d3784ed 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandler.java @@ -22,11 +22,10 @@ import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.Indexed; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.Closeable; -import java.io.File; import java.io.IOException; /** @@ -95,8 +94,7 @@ default MultiValueHandling getMultivalueHandling() * See {@link DimensionMergerV9} interface for more information. * * @param indexSpec Specification object for the index merge - * @param outDir Location to store files generated by the merging process - * @param ioPeon ioPeon object passed in by IndexMerger, manages files created by the merging process + * @param segmentWriteOutMedium this SegmentWriteOutMedium object could be used internally in the created merger, if needed * @param capabilities The ColumnCapabilities of the dimension represented by this DimensionHandler * @param progress ProgressIndicator used by the merging process @@ -104,8 +102,7 @@ default MultiValueHandling getMultivalueHandling() */ DimensionMergerV9 makeMerger( IndexSpec indexSpec, - File outDir, - IOPeon ioPeon, + SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, ProgressIndicator progress ) throws IOException; diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java b/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java index 04b4fd884171..5f9207f31890 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java @@ -28,9 +28,6 @@ */ public interface DoubleColumnSelector extends ColumnValueSelector { - @Override - double getDouble(); - /** * @deprecated This method is marked as deprecated in DoubleColumnSelector to minimize the probability of accidential * calling. "Polymorphism" of DoubleColumnSelector should be used only when operating on {@link ColumnValueSelector} diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java index 5475d187951a..5eaff725765e 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java @@ -19,7 +19,6 @@ package io.druid.segment; - import com.google.common.primitives.Ints; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.MutableBitmap; @@ -27,10 +26,10 @@ import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.ByteBufferWriter; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.DoubleSupplierSerializer; -import io.druid.segment.data.IOPeon; +import io.druid.segment.writeout.SegmentWriteOutMedium; import javax.annotation.Nullable; import java.io.IOException; @@ -41,34 +40,41 @@ public class DoubleColumnSerializer implements GenericColumnSerializer { public static DoubleColumnSerializer create( - IOPeon ioPeon, + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, - CompressedObjectStrategy.CompressionStrategy compression, + CompressionStrategy compression, BitmapSerdeFactory bitmapSerdeFactory ) { - return new DoubleColumnSerializer(ioPeon, filenameBase, IndexIO.BYTE_ORDER, compression, bitmapSerdeFactory); + return new DoubleColumnSerializer( + segmentWriteOutMedium, + filenameBase, + IndexIO.BYTE_ORDER, + compression, + bitmapSerdeFactory + ); } - private final IOPeon ioPeon; + private final SegmentWriteOutMedium segmentWriteOutMedium; private final String filenameBase; private final ByteOrder byteOrder; - private final CompressedObjectStrategy.CompressionStrategy compression; + private final CompressionStrategy compression; private final BitmapSerdeFactory bitmapSerdeFactory; + private DoubleSupplierSerializer writer; private ByteBufferWriter nullValueBitmapWriter; private MutableBitmap nullRowsBitmap; private int rowCount = 0; - public DoubleColumnSerializer( - IOPeon ioPeon, + private DoubleColumnSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression, + CompressionStrategy compression, BitmapSerdeFactory bitmapSerdeFactory ) { - this.ioPeon = ioPeon; + this.segmentWriteOutMedium = segmentWriteOutMedium; this.filenameBase = filenameBase; this.byteOrder = byteOrder; this.compression = compression; @@ -79,15 +85,14 @@ public DoubleColumnSerializer( public void open() throws IOException { writer = CompressionFactory.getDoubleSerializer( - ioPeon, + segmentWriteOutMedium, StringUtils.format("%s.double_column", filenameBase), byteOrder, compression ); writer.open(); nullValueBitmapWriter = new ByteBufferWriter<>( - ioPeon, - StringUtils.format("%s.nullBitmap", filenameBase), + segmentWriteOutMedium, bitmapSerdeFactory.getObjectStrategy() ); nullValueBitmapWriter.open(); @@ -107,16 +112,9 @@ public void serialize(@Nullable Object obj) throws IOException } @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { - writer.close(); nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); - nullValueBitmapWriter.close(); - } - - @Override - public long getSerializedSize() - { long bitmapSize = nullRowsBitmap.isEmpty() ? 0L : nullValueBitmapWriter.getSerializedSize(); @@ -124,12 +122,12 @@ public long getSerializedSize() } @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { channel.write(ByteBuffer.wrap(Ints.toByteArray((int) writer.getSerializedSize()))); - writer.writeToChannel(channel, smoosher); + writer.writeTo(channel, smoosher); if (!nullRowsBitmap.isEmpty()) { - nullValueBitmapWriter.writeToChannel(channel); + nullValueBitmapWriter.writeTo(channel, smoosher); } } diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java b/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java index e37a70d02273..2f3e350852a5 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java @@ -22,11 +22,10 @@ import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.GenericColumn; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.Indexed; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.Closeable; -import java.io.File; import java.io.IOException; public class DoubleDimensionHandler implements DimensionHandler @@ -52,13 +51,16 @@ public DimensionIndexer makeIndexer() @Override public DimensionMergerV9 makeMerger( - IndexSpec indexSpec, File outDir, IOPeon ioPeon, ColumnCapabilities capabilities, ProgressIndicator progress + IndexSpec indexSpec, + SegmentWriteOutMedium segmentWriteOutMedium, + ColumnCapabilities capabilities, + ProgressIndicator progress ) throws IOException { return new DoubleDimensionMergerV9( dimensionName, indexSpec, - ioPeon + segmentWriteOutMedium ); } diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java index c1aecc2ce319..7d87798de8f8 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java @@ -21,9 +21,9 @@ import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.CompressedObjectStrategy; -import io.druid.segment.data.IOPeon; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.serde.DoubleGenericColumnPartSerdeV2; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.IOException; import java.nio.IntBuffer; @@ -33,32 +33,30 @@ public class DoubleDimensionMergerV9 implements DimensionMergerV9 { protected String dimensionName; protected final IndexSpec indexSpec; - protected IOPeon ioPeon; private DoubleColumnSerializer serializer; public DoubleDimensionMergerV9( String dimensionName, IndexSpec indexSpec, - IOPeon ioPeon + SegmentWriteOutMedium segmentWriteOutMedium ) { this.dimensionName = dimensionName; this.indexSpec = indexSpec; - this.ioPeon = ioPeon; try { - setupEncodedValueWriter(); + setupEncodedValueWriter(segmentWriteOutMedium); } catch (IOException ioe) { throw new RuntimeException(ioe); } } - protected void setupEncodedValueWriter() throws IOException + private void setupEncodedValueWriter(SegmentWriteOutMedium segmentWriteOutMedium) throws IOException { - final CompressedObjectStrategy.CompressionStrategy metCompression = indexSpec.getMetricCompression(); + final CompressionStrategy metCompression = indexSpec.getMetricCompression(); this.serializer = DoubleColumnSerializer.create( - ioPeon, + segmentWriteOutMedium, dimensionName, metCompression, indexSpec.getBitmapSerdeFactory() @@ -66,22 +64,6 @@ protected void setupEncodedValueWriter() throws IOException serializer.open(); } - @Override - public ColumnDescriptor makeColumnDescriptor() throws IOException - { - serializer.close(); - final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); - builder.setValueType(ValueType.DOUBLE); - builder.addSerde( - DoubleGenericColumnPartSerdeV2.serializerBuilder() - .withByteOrder(IndexIO.BYTE_ORDER) - .withDelegate(serializer) - .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) - .build() - ); - return builder.build(); - } - @Override public void writeMergedValueMetadata(List adapters) throws IOException { @@ -111,4 +93,19 @@ public boolean canSkip() { return false; } + + @Override + public ColumnDescriptor makeColumnDescriptor() throws IOException + { + final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); + builder.setValueType(ValueType.DOUBLE); + builder.addSerde( + DoubleGenericColumnPartSerdeV2.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withDelegate(serializer) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) + .build() + ); + return builder.build(); + } } diff --git a/processing/src/main/java/io/druid/segment/FilteredOffset.java b/processing/src/main/java/io/druid/segment/FilteredOffset.java index 685c0cf6ef8a..3becd42533ab 100644 --- a/processing/src/main/java/io/druid/segment/FilteredOffset.java +++ b/processing/src/main/java/io/druid/segment/FilteredOffset.java @@ -110,10 +110,20 @@ public ReadableOffset getBaseReadableOffset() return baseOffset.getBaseReadableOffset(); } + /** + * clone() is not supported by FilteredOffset because it's not possible to clone {@link #filterMatcher}, and + * while re-creating filterMatcher could be not very cheap for some implementations of {@link Filter}. Although this + * approach could be investigated. + * + * If clone is made possible for FilteredOffset, some improvements could become possible in {@link + * io.druid.query.topn.PooledTopNAlgorithm#computeSpecializedScanAndAggregateImplementations}. + * + * See also https://github.com/druid-io/druid/issues/5132. + */ @Override public Offset clone() { - throw new UnsupportedOperationException("FilteredOffset should not be cloned"); + throw new UnsupportedOperationException("FilteredOffset could not be cloned"); } @Override diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSelector.java b/processing/src/main/java/io/druid/segment/FloatColumnSelector.java index 92b4a4c31950..0e890138936b 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSelector.java @@ -28,9 +28,6 @@ */ public interface FloatColumnSelector extends ColumnValueSelector { - @Override - float getFloat(); - /** * @deprecated This method is marked as deprecated in FloatColumnSelector to minimize the probability of accidential * calling. "Polymorphism" of FloatColumnSelector should be used only when operating on {@link ColumnValueSelector} diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java index 0a2448787362..4790f9877495 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java @@ -26,10 +26,10 @@ import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.ByteBufferWriter; -import io.druid.segment.data.CompressedObjectStrategy; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.FloatSupplierSerializer; -import io.druid.segment.data.IOPeon; import javax.annotation.Nullable; import java.io.IOException; @@ -40,34 +40,35 @@ public class FloatColumnSerializer implements GenericColumnSerializer { public static FloatColumnSerializer create( - IOPeon ioPeon, + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, - CompressedObjectStrategy.CompressionStrategy compression, + CompressionStrategy compression, BitmapSerdeFactory bitmapSerdeFactory ) { - return new FloatColumnSerializer(ioPeon, filenameBase, IndexIO.BYTE_ORDER, compression, bitmapSerdeFactory); + return new FloatColumnSerializer(segmentWriteOutMedium, filenameBase, IndexIO.BYTE_ORDER, compression, bitmapSerdeFactory); } - private final IOPeon ioPeon; + private final SegmentWriteOutMedium segmentWriteOutMedium; private final String filenameBase; private final ByteOrder byteOrder; - private final CompressedObjectStrategy.CompressionStrategy compression; + private final CompressionStrategy compression; private final BitmapSerdeFactory bitmapSerdeFactory; + private FloatSupplierSerializer writer; private ByteBufferWriter nullValueBitmapWriter; private MutableBitmap nullRowsBitmap; private int rowCount = 0; - public FloatColumnSerializer( - IOPeon ioPeon, + private FloatColumnSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression, + CompressionStrategy compression, BitmapSerdeFactory bitmapSerdeFactory ) { - this.ioPeon = ioPeon; + this.segmentWriteOutMedium = segmentWriteOutMedium; this.filenameBase = filenameBase; this.byteOrder = byteOrder; this.compression = compression; @@ -78,15 +79,14 @@ public FloatColumnSerializer( public void open() throws IOException { writer = CompressionFactory.getFloatSerializer( - ioPeon, + segmentWriteOutMedium, StringUtils.format("%s.float_column", filenameBase), byteOrder, compression ); writer.open(); nullValueBitmapWriter = new ByteBufferWriter<>( - ioPeon, - StringUtils.format("%s.nullBitmap", filenameBase), + segmentWriteOutMedium, bitmapSerdeFactory.getObjectStrategy() ); nullValueBitmapWriter.open(); @@ -106,16 +106,10 @@ public void serialize(@Nullable Object obj) throws IOException } @Override - public void close() throws IOException - { - writer.close(); - nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); - nullValueBitmapWriter.close(); - } - @Override - public long getSerializedSize() + public long getSerializedSize() throws IOException { + nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); long bitmapSize = nullRowsBitmap.isEmpty() ? 0L : nullValueBitmapWriter.getSerializedSize(); @@ -123,13 +117,12 @@ public long getSerializedSize() } @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { channel.write(ByteBuffer.wrap(Ints.toByteArray((int) writer.getSerializedSize()))); - writer.writeToChannel(channel, smoosher); + writer.writeTo(channel, smoosher); if (!nullRowsBitmap.isEmpty()) { - nullValueBitmapWriter.writeToChannel(channel); + nullValueBitmapWriter.writeTo(channel, smoosher); } } - } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java index 99243913a6ac..a946bf9e1bcd 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java @@ -22,11 +22,10 @@ import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.GenericColumn; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.Indexed; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.Closeable; -import java.io.File; import java.io.IOException; public class FloatDimensionHandler implements DimensionHandler @@ -52,13 +51,16 @@ public DimensionIndexer makeIndexer() @Override public DimensionMergerV9 makeMerger( - IndexSpec indexSpec, File outDir, IOPeon ioPeon, ColumnCapabilities capabilities, ProgressIndicator progress + IndexSpec indexSpec, + SegmentWriteOutMedium segmentWriteOutMedium, + ColumnCapabilities capabilities, + ProgressIndicator progress ) throws IOException { return new FloatDimensionMergerV9( dimensionName, indexSpec, - ioPeon + segmentWriteOutMedium ); } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java index 45c878f08ca3..f1d41de12116 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java @@ -21,9 +21,9 @@ import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.CompressedObjectStrategy; -import io.druid.segment.data.IOPeon; import io.druid.segment.serde.FloatGenericColumnPartSerdeV2; +import io.druid.segment.data.CompressionStrategy; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.IOException; import java.nio.IntBuffer; @@ -33,32 +33,29 @@ public class FloatDimensionMergerV9 implements DimensionMergerV9 { protected String dimensionName; protected final IndexSpec indexSpec; - protected IOPeon ioPeon; - private FloatColumnSerializer serializer; public FloatDimensionMergerV9( String dimensionName, IndexSpec indexSpec, - IOPeon ioPeon + SegmentWriteOutMedium segmentWriteOutMedium ) { this.dimensionName = dimensionName; this.indexSpec = indexSpec; - this.ioPeon = ioPeon; try { - setupEncodedValueWriter(); + setupEncodedValueWriter(segmentWriteOutMedium); } catch (IOException ioe) { throw new RuntimeException(ioe); } } - protected void setupEncodedValueWriter() throws IOException + private void setupEncodedValueWriter(SegmentWriteOutMedium segmentWriteOutMedium) throws IOException { - final CompressedObjectStrategy.CompressionStrategy metCompression = indexSpec.getMetricCompression(); + final CompressionStrategy metCompression = indexSpec.getMetricCompression(); this.serializer = FloatColumnSerializer.create( - ioPeon, + segmentWriteOutMedium, dimensionName, metCompression, indexSpec.getBitmapSerdeFactory() @@ -99,7 +96,6 @@ public boolean canSkip() @Override public ColumnDescriptor makeColumnDescriptor() throws IOException { - serializer.close(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.FLOAT); builder.addSerde( diff --git a/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java b/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java index b72a4a9e181a..a9f35004fab5 100644 --- a/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/GenericColumnSerializer.java @@ -20,21 +20,14 @@ package io.druid.segment; import io.druid.guice.annotations.ExtensionPoint; -import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.Serializer; -import javax.annotation.Nullable; -import java.io.Closeable; import java.io.IOException; -import java.nio.channels.WritableByteChannel; @ExtensionPoint -public interface GenericColumnSerializer extends Closeable +public interface GenericColumnSerializer extends Serializer { void open() throws IOException; - void serialize(@Nullable Object obj) throws IOException; - - long getSerializedSize(); - - void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException; + void serialize(Object obj) throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index e90669bd6442..c0cca673781e 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -44,6 +44,7 @@ import io.druid.java.util.common.io.smoosh.Smoosh; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnCapabilities; @@ -70,6 +71,7 @@ import io.druid.segment.serde.SpatialIndexColumnPartSupplier; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -96,11 +98,13 @@ public class IndexIO private static final SerializerUtils serializerUtils = new SerializerUtils(); private final ObjectMapper mapper; + private final SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory; @Inject - public IndexIO(ObjectMapper mapper, ColumnConfig columnConfig) + public IndexIO(ObjectMapper mapper, SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory, ColumnConfig columnConfig) { this.mapper = Preconditions.checkNotNull(mapper, "null ObjectMapper"); + this.defaultSegmentWriteOutMediumFactory = Preconditions.checkNotNull(defaultSegmentWriteOutMediumFactory, "null SegmentWriteOutMediumFactory"); Preconditions.checkNotNull(columnConfig, "null ColumnConfig"); ImmutableMap.Builder indexLoadersBuilder = ImmutableMap.builder(); LegacyIndexLoader legacyIndexLoader = new LegacyIndexLoader(new DefaultIndexIOHandler(), columnConfig); @@ -223,13 +227,17 @@ public boolean convertSegment( File converted, IndexSpec indexSpec, boolean forceIfCurrent, - boolean validate + boolean validate, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) throws IOException { final int version = SegmentUtils.getVersionFromDir(toConvert); boolean current = version == CURRENT_VERSION_ID; if (!current || forceIfCurrent) { - new IndexMergerV9(mapper, this).convert(toConvert, converted, indexSpec); + if (segmentWriteOutMediumFactory == null) { + segmentWriteOutMediumFactory = this.defaultSegmentWriteOutMediumFactory; + } + new IndexMergerV9(mapper, this, segmentWriteOutMediumFactory).convert(toConvert, converted, indexSpec); if (validate) { validateTwoSegments(toConvert, converted); } @@ -340,8 +348,7 @@ public MMappedIndex mapDir(File inDir) throws IOException CompressedLongsIndexedSupplier timestamps = CompressedLongsIndexedSupplier.fromByteBuffer( smooshedFiles.mapFile(makeTimeFile(inDir, BYTE_ORDER).getName()), - BYTE_ORDER, - smooshedFiles + BYTE_ORDER ); Map metrics = Maps.newLinkedHashMap(); diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index b678cfc179ad..8fc5e5439ac9 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -19,6 +19,7 @@ package io.druid.segment; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; @@ -39,6 +40,7 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.data.Indexed; import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntSortedSet; @@ -156,7 +158,12 @@ static > ArrayList mergeIndexed(List> ArrayList mergeIndexed(List indexes, boolean rollup, @@ -208,11 +225,21 @@ File merge( // Faster than IndexMaker File convert(File inDir, File outDir, IndexSpec indexSpec) throws IOException; - File convert(File inDir, File outDir, IndexSpec indexSpec, ProgressIndicator progress) - throws IOException; + File convert( + File inDir, + File outDir, + IndexSpec indexSpec, + ProgressIndicator progress, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory + ) throws IOException; - File append(List indexes, AggregatorFactory[] aggregators, File outDir, IndexSpec indexSpec) - throws IOException; + File append( + List indexes, + AggregatorFactory[] aggregators, + File outDir, + IndexSpec indexSpec, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory + ) throws IOException; interface IndexSeeker { @@ -385,7 +412,8 @@ public Rowboat apply(Rowboat lhs, Rowboat rhs) Int2ObjectMap.Entry entry = entryIterator.next(); for (IntIterator setIterator = entry.getValue().iterator(); setIterator.hasNext(); /* NOP */) { - retVal.addRow(entry.getIntKey(), setIterator.nextInt()); + int rowNum = setIterator.nextInt(); + retVal.addRow(entry.getIntKey(), rowNum); } } } diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index 8c28ea09bf9d..0d2578063b5f 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -28,7 +28,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import com.google.common.io.ByteStreams; import com.google.common.io.Files; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; @@ -52,12 +51,10 @@ import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.Indexed; -import io.druid.segment.data.TmpFileIOPeon; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexAdapter; import io.druid.segment.loading.MMappedQueryableSegmentizerFactory; @@ -67,6 +64,8 @@ import io.druid.segment.serde.DoubleGenericColumnPartSerdeV2; import io.druid.segment.serde.FloatGenericColumnPartSerdeV2; import io.druid.segment.serde.LongGenericColumnPartSerdeV2; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntSortedSet; @@ -75,7 +74,6 @@ import org.joda.time.Interval; import javax.annotation.Nullable; -import java.io.Closeable; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -91,30 +89,22 @@ public class IndexMergerV9 implements IndexMerger { private static final Logger log = new Logger(IndexMergerV9.class); - protected final ObjectMapper mapper; - protected final IndexIO indexIO; + + private final ObjectMapper mapper; + private final IndexIO indexIO; + private final SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory; @Inject public IndexMergerV9( ObjectMapper mapper, - IndexIO indexIO + IndexIO indexIO, + SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory ) { this.mapper = Preconditions.checkNotNull(mapper, "null ObjectMapper"); this.indexIO = Preconditions.checkNotNull(indexIO, "null IndexIO"); - - } - - private static void registerDeleteDirectory(Closer closer, final File dir) - { - closer.register(new Closeable() - { - @Override - public void close() throws IOException - { - FileUtils.deleteDirectory(dir); - } - }); + this.defaultSegmentWriteOutMediumFactory = + Preconditions.checkNotNull(defaultSegmentWriteOutMediumFactory, "null SegmentWriteOutMediumFactory"); } private File makeIndexFiles( @@ -125,7 +115,8 @@ private File makeIndexFiles( final List mergedDimensions, final List mergedMetrics, final Function>, Iterable> rowMergerFn, - final IndexSpec indexSpec + final IndexSpec indexSpec, + final @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) throws IOException { progress.start(); @@ -163,21 +154,15 @@ public Metadata apply(IndexableAdapter input) Closer closer = Closer.create(); try { final FileSmoosher v9Smoosher = new FileSmoosher(outDir); - final File v9TmpDir = new File(outDir, "v9-tmp"); - FileUtils.forceMkdir(v9TmpDir); - registerDeleteDirectory(closer, v9TmpDir); - log.info("Start making v9 index files, outDir:%s", outDir); - - File tmpPeonFilesDir = new File(v9TmpDir, "tmpPeonFiles"); - FileUtils.forceMkdir(tmpPeonFilesDir); - registerDeleteDirectory(closer, tmpPeonFilesDir); - final IOPeon ioPeon = new TmpFileIOPeon(tmpPeonFilesDir, false); - closer.register(ioPeon); + FileUtils.forceMkdir(outDir); + + SegmentWriteOutMediumFactory omf = segmentWriteOutMediumFactory != null ? segmentWriteOutMediumFactory + : defaultSegmentWriteOutMediumFactory; + log.info("Using SegmentWriteOutMediumFactory[%s]", omf.getClass().getSimpleName()); + SegmentWriteOutMedium segmentWriteOutMedium = omf.makeSegmentWriteOutMedium(outDir); + closer.register(segmentWriteOutMedium); long startTime = System.currentTimeMillis(); - ByteStreams.write( - Ints.toByteArray(IndexIO.V9_VERSION), - Files.newOutputStreamSupplier(new File(outDir, "version.bin")) - ); + Files.asByteSink(new File(outDir, "version.bin")).write(Ints.toByteArray(IndexIO.V9_VERSION)); log.info("Completed version.bin in %,d millis.", System.currentTimeMillis() - startTime); progress.progress(); @@ -196,7 +181,7 @@ public Metadata apply(IndexableAdapter input) final DimensionHandler[] handlers = makeDimensionHandlers(mergedDimensions, dimCapabilities); final List mergers = new ArrayList<>(); for (int i = 0; i < mergedDimensions.size(); i++) { - mergers.add(handlers[i].makeMerger(indexSpec, v9TmpDir, ioPeon, dimCapabilities.get(i), progress)); + mergers.add(handlers[i].makeMerger(indexSpec, segmentWriteOutMedium, dimCapabilities.get(i), progress)); } /************* Setup Dim Conversions **************/ @@ -215,15 +200,17 @@ public Metadata apply(IndexableAdapter input) handlers, mergers ); - final LongColumnSerializer timeWriter = setupTimeWriter(ioPeon, indexSpec); + final LongColumnSerializer timeWriter = setupTimeWriter(segmentWriteOutMedium, indexSpec); final ArrayList metWriters = setupMetricsWriters( - ioPeon, mergedMetrics, metricsValueTypes, metricTypeNames, indexSpec + segmentWriteOutMedium, + mergedMetrics, + metricsValueTypes, + metricTypeNames, + indexSpec ); final List rowNumConversions = Lists.newArrayListWithCapacity(adapters.size()); - mergeIndexesAndWriteColumns( - adapters, progress, theRows, timeWriter, metWriters, rowNumConversions, mergers - ); + mergeIndexesAndWriteColumns(adapters, progress, theRows, timeWriter, metWriters, rowNumConversions, mergers); /************ Create Inverted Indexes and Finalize Build Columns *************/ final String section = "build inverted index and columns"; @@ -253,9 +240,7 @@ public Metadata apply(IndexableAdapter input) /************* Make index.drd & metadata.drd files **************/ progress.progress(); - makeIndexBinary( - v9Smoosher, adapters, outDir, mergedDimensions, mergedMetrics, progress, indexSpec, mergers - ); + makeIndexBinary(v9Smoosher, adapters, outDir, mergedDimensions, mergedMetrics, progress, indexSpec, mergers); makeMetadataBinary(v9Smoosher, progress, segmentMetadata); v9Smoosher.close(); @@ -320,8 +305,8 @@ private void makeIndexBinary( + serializerUtils.getSerializedStringByteSize(bitmapSerdeFactoryType); final SmooshedWriter writer = v9Smoosher.addWithSmooshedWriter("index.drd", numBytes); - cols.writeToChannel(writer); - dims.writeToChannel(writer); + cols.writeTo(writer, v9Smoosher); + dims.writeTo(writer, v9Smoosher); DateTime minTime = DateTimes.MAX; DateTime maxTime = DateTimes.MIN; @@ -335,9 +320,7 @@ private void makeIndexBinary( serializerUtils.writeLong(writer, dataInterval.getStartMillis()); serializerUtils.writeLong(writer, dataInterval.getEndMillis()); - serializerUtils.writeString( - writer, bitmapSerdeFactoryType - ); + serializerUtils.writeString(writer, bitmapSerdeFactoryType); writer.close(); IndexIO.checkFileSize(new File(outDir, "index.drd")); @@ -364,7 +347,6 @@ private void makeMetricsColumns( String metric = mergedMetrics.get(i); long metricStartTime = System.currentTimeMillis(); GenericColumnSerializer writer = metWriters.get(i); - writer.close(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); ValueType type = metricsValueTypes.get(metric); @@ -433,8 +415,6 @@ private void makeTimeColumn( progress.startSection(section); long startTime = System.currentTimeMillis(); - timeWriter.close(); - final ColumnDescriptor serdeficator = ColumnDescriptor .builder() .setValueType(ValueType.LONG) @@ -459,10 +439,11 @@ private void makeColumn( ZeroCopyByteArrayOutputStream specBytes = new ZeroCopyByteArrayOutputStream(); serializerUtils.writeString(specBytes, mapper.writeValueAsString(serdeficator)); try (SmooshedWriter channel = v9Smoosher.addWithSmooshedWriter( - columnName, serdeficator.numBytes() + specBytes.size() + columnName, + specBytes.size() + serdeficator.getSerializedSize() )) { specBytes.writeTo(channel); - serdeficator.write(channel, v9Smoosher); + serdeficator.writeTo(channel, v9Smoosher); } } @@ -535,10 +516,11 @@ private void mergeIndexesAndWriteColumns( progress.stopSection(section); } - private LongColumnSerializer setupTimeWriter(final IOPeon ioPeon, final IndexSpec indexSpec) throws IOException + private LongColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMedium, IndexSpec indexSpec) + throws IOException { LongColumnSerializer timeWriter = LongColumnSerializer.create( - ioPeon, "little_end_time", CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY, + segmentWriteOutMedium, "little_end_time", CompressionStrategy.DEFAULT_COMPRESSION_STRATEGY, indexSpec.getLongEncoding(), indexSpec.getBitmapSerdeFactory() ); @@ -548,7 +530,7 @@ private LongColumnSerializer setupTimeWriter(final IOPeon ioPeon, final IndexSpe } private ArrayList setupMetricsWriters( - final IOPeon ioPeon, + final SegmentWriteOutMedium segmentWriteOutMedium, final List mergedMetrics, final Map metricsValueTypes, final Map metricTypeNames, @@ -556,7 +538,7 @@ private ArrayList setupMetricsWriters( ) throws IOException { ArrayList metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size()); - final CompressedObjectStrategy.CompressionStrategy metCompression = indexSpec.getMetricCompression(); + final CompressionStrategy metCompression = indexSpec.getMetricCompression(); final CompressionFactory.LongEncodingStrategy longEncoding = indexSpec.getLongEncoding(); for (String metric : mergedMetrics) { ValueType type = metricsValueTypes.get(metric); @@ -564,7 +546,7 @@ private ArrayList setupMetricsWriters( switch (type) { case LONG: writer = LongColumnSerializer.create( - ioPeon, + segmentWriteOutMedium, metric, metCompression, longEncoding, @@ -572,10 +554,20 @@ private ArrayList setupMetricsWriters( ); break; case FLOAT: - writer = FloatColumnSerializer.create(ioPeon, metric, metCompression, indexSpec.getBitmapSerdeFactory()); + writer = FloatColumnSerializer.create( + segmentWriteOutMedium, + metric, + metCompression, + indexSpec.getBitmapSerdeFactory() + ); break; case DOUBLE: - writer = DoubleColumnSerializer.create(ioPeon, metric, metCompression, indexSpec.getBitmapSerdeFactory()); + writer = DoubleColumnSerializer.create( + segmentWriteOutMedium, + metric, + metCompression, + indexSpec.getBitmapSerdeFactory() + ); break; case COMPLEX: final String typeName = metricTypeNames.get(metric); @@ -583,7 +575,7 @@ private ArrayList setupMetricsWriters( if (serde == null) { throw new ISE("Unknown type[%s]", typeName); } - writer = serde.getSerializer(ioPeon, metric); + writer = serde.getSerializer(segmentWriteOutMedium, metric); break; default: throw new ISE("Unknown type[%s]", type); @@ -649,10 +641,11 @@ private void mergeCapabilities( public File persist( final IncrementalIndex index, File outDir, - IndexSpec indexSpec + IndexSpec indexSpec, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) throws IOException { - return persist(index, index.getInterval(), outDir, indexSpec); + return persist(index, index.getInterval(), outDir, indexSpec, segmentWriteOutMediumFactory); } @Override @@ -660,10 +653,11 @@ public File persist( final IncrementalIndex index, final Interval dataInterval, File outDir, - IndexSpec indexSpec + IndexSpec indexSpec, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) throws IOException { - return persist(index, dataInterval, outDir, indexSpec, new BaseProgressIndicator()); + return persist(index, dataInterval, outDir, indexSpec, new BaseProgressIndicator(), segmentWriteOutMediumFactory); } @Override @@ -672,7 +666,8 @@ public File persist( final Interval dataInterval, File outDir, IndexSpec indexSpec, - ProgressIndicator progress + ProgressIndicator progress, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) throws IOException { if (index.isEmpty()) { @@ -709,7 +704,8 @@ public File persist( index.getMetricAggs(), outDir, indexSpec, - progress + progress, + segmentWriteOutMediumFactory ); } @@ -719,10 +715,19 @@ public File mergeQueryableIndex( boolean rollup, final AggregatorFactory[] metricAggs, File outDir, - IndexSpec indexSpec + IndexSpec indexSpec, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) throws IOException { - return mergeQueryableIndex(indexes, rollup, metricAggs, outDir, indexSpec, new BaseProgressIndicator()); + return mergeQueryableIndex( + indexes, + rollup, + metricAggs, + outDir, + indexSpec, + new BaseProgressIndicator(), + segmentWriteOutMediumFactory + ); } @Override @@ -732,7 +737,8 @@ public File mergeQueryableIndex( final AggregatorFactory[] metricAggs, File outDir, IndexSpec indexSpec, - ProgressIndicator progress + ProgressIndicator progress, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) throws IOException { return merge( @@ -741,7 +747,8 @@ public File mergeQueryableIndex( metricAggs, outDir, indexSpec, - progress + progress, + segmentWriteOutMediumFactory ); } @@ -754,16 +761,17 @@ public File merge( IndexSpec indexSpec ) throws IOException { - return merge(indexes, rollup, metricAggs, outDir, indexSpec, new BaseProgressIndicator()); + return merge(indexes, rollup, metricAggs, outDir, indexSpec, new BaseProgressIndicator(), null); } - public File merge( + private File merge( List indexes, final boolean rollup, final AggregatorFactory[] metricAggs, File outDir, IndexSpec indexSpec, - ProgressIndicator progress + ProgressIndicator progress, + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) throws IOException { FileUtils.deleteDirectory(outDir); @@ -868,19 +876,25 @@ public int compare(Rowboat left, Rowboat right) mergedDimensions, mergedMetrics, rowMergerFn, - indexSpec + indexSpec, + segmentWriteOutMediumFactory ); } @Override public File convert(final File inDir, final File outDir, final IndexSpec indexSpec) throws IOException { - return convert(inDir, outDir, indexSpec, new BaseProgressIndicator()); + return convert(inDir, outDir, indexSpec, new BaseProgressIndicator(), defaultSegmentWriteOutMediumFactory); } @Override - public File convert(final File inDir, final File outDir, final IndexSpec indexSpec, final ProgressIndicator progress) - throws IOException + public File convert( + final File inDir, + final File outDir, + final IndexSpec indexSpec, + final ProgressIndicator progress, + final @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory + ) throws IOException { try (QueryableIndex index = indexIO.loadIndex(inDir)) { final IndexableAdapter adapter = new QueryableIndexIndexableAdapter(index); @@ -900,25 +914,19 @@ public Iterable apply(ArrayList> input) return input.get(0); } }, - indexSpec + indexSpec, + segmentWriteOutMediumFactory ); } } @Override - public File append( - List indexes, AggregatorFactory[] aggregators, File outDir, IndexSpec indexSpec - ) throws IOException - { - return append(indexes, aggregators, outDir, indexSpec, new BaseProgressIndicator()); - } - public File append( List indexes, AggregatorFactory[] aggregators, File outDir, IndexSpec indexSpec, - ProgressIndicator progress + @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) throws IOException { FileUtils.deleteDirectory(outDir); @@ -965,11 +973,12 @@ public Iterable apply( indexes, aggregators, outDir, - progress, + new BaseProgressIndicator(), mergedDimensions, mergedMetrics, rowMergerFn, - indexSpec + indexSpec, + segmentWriteOutMediumFactory ); } diff --git a/processing/src/main/java/io/druid/segment/IndexSpec.java b/processing/src/main/java/io/druid/segment/IndexSpec.java index 67ea64d2ac36..073a6a879af6 100644 --- a/processing/src/main/java/io/druid/segment/IndexSpec.java +++ b/processing/src/main/java/io/druid/segment/IndexSpec.java @@ -25,8 +25,8 @@ import com.google.common.collect.Sets; import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; import java.util.Arrays; @@ -41,16 +41,16 @@ */ public class IndexSpec { - public static final CompressedObjectStrategy.CompressionStrategy DEFAULT_METRIC_COMPRESSION = CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY; - public static final CompressedObjectStrategy.CompressionStrategy DEFAULT_DIMENSION_COMPRESSION = CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY; + public static final CompressionStrategy DEFAULT_METRIC_COMPRESSION = CompressionStrategy.DEFAULT_COMPRESSION_STRATEGY; + public static final CompressionStrategy DEFAULT_DIMENSION_COMPRESSION = CompressionStrategy.DEFAULT_COMPRESSION_STRATEGY; public static final CompressionFactory.LongEncodingStrategy DEFAULT_LONG_ENCODING = CompressionFactory.DEFAULT_LONG_ENCODING_STRATEGY; - private static final Set METRIC_COMPRESSION = Sets.newHashSet( - Arrays.asList(CompressedObjectStrategy.CompressionStrategy.values()) + private static final Set METRIC_COMPRESSION = Sets.newHashSet( + Arrays.asList(CompressionStrategy.values()) ); - private static final Set DIMENSION_COMPRESSION = Sets.newHashSet( - Arrays.asList(CompressedObjectStrategy.CompressionStrategy.noNoneValues()) + private static final Set DIMENSION_COMPRESSION = Sets.newHashSet( + Arrays.asList(CompressionStrategy.noNoneValues()) ); private static final Set LONG_ENCODING_NAMES = Sets.newHashSet( @@ -58,8 +58,8 @@ public class IndexSpec ); private final BitmapSerdeFactory bitmapSerdeFactory; - private final CompressedObjectStrategy.CompressionStrategy dimensionCompression; - private final CompressedObjectStrategy.CompressionStrategy metricCompression; + private final CompressionStrategy dimensionCompression; + private final CompressionStrategy metricCompression; private final CompressionFactory.LongEncodingStrategy longEncoding; @@ -80,10 +80,10 @@ public IndexSpec() * setting, or, if none was set, uses the default defined in {@link BitmapSerde} * * @param dimensionCompression compression format for dimension columns, null to use the default. - * Defaults to {@link CompressedObjectStrategy#DEFAULT_COMPRESSION_STRATEGY} + * Defaults to {@link CompressionStrategy#DEFAULT_COMPRESSION_STRATEGY} * * @param metricCompression compression format for metric columns, null to use the default. - * Defaults to {@link CompressedObjectStrategy#DEFAULT_COMPRESSION_STRATEGY} + * Defaults to {@link CompressionStrategy#DEFAULT_COMPRESSION_STRATEGY} * * @param longEncoding encoding strategy for metric and dimension columns with type long, null to use the default. * Defaults to {@link CompressionFactory#DEFAULT_LONG_ENCODING_STRATEGY} @@ -91,8 +91,8 @@ public IndexSpec() @JsonCreator public IndexSpec( @JsonProperty("bitmap") BitmapSerdeFactory bitmapSerdeFactory, - @JsonProperty("dimensionCompression") CompressedObjectStrategy.CompressionStrategy dimensionCompression, - @JsonProperty("metricCompression") CompressedObjectStrategy.CompressionStrategy metricCompression, + @JsonProperty("dimensionCompression") CompressionStrategy dimensionCompression, + @JsonProperty("metricCompression") CompressionStrategy metricCompression, @JsonProperty("longEncoding") CompressionFactory.LongEncodingStrategy longEncoding ) { @@ -118,13 +118,13 @@ public BitmapSerdeFactory getBitmapSerdeFactory() } @JsonProperty - public CompressedObjectStrategy.CompressionStrategy getDimensionCompression() + public CompressionStrategy getDimensionCompression() { return dimensionCompression; } @JsonProperty - public CompressedObjectStrategy.CompressionStrategy getMetricCompression() + public CompressionStrategy getMetricCompression() { return metricCompression; } diff --git a/processing/src/main/java/io/druid/segment/LongColumnSelector.java b/processing/src/main/java/io/druid/segment/LongColumnSelector.java index 85ba51d87f65..9f849fe9063e 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSelector.java @@ -28,9 +28,6 @@ */ public interface LongColumnSelector extends ColumnValueSelector { - @Override - long getLong(); - /** * @deprecated This method is marked as deprecated in LongColumnSelector to minimize the probability of accidential * calling. "Polymorphism" of LongColumnSelector should be used only when operating on {@link ColumnValueSelector} diff --git a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java index 26c87d7329b8..2f003d06974b 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java @@ -26,9 +26,9 @@ import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.ByteBufferWriter; -import io.druid.segment.data.CompressedObjectStrategy; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.data.CompressionFactory; -import io.druid.segment.data.IOPeon; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.LongSupplierSerializer; import javax.annotation.Nullable; @@ -43,15 +43,15 @@ public class LongColumnSerializer implements GenericColumnSerializer { public static LongColumnSerializer create( - IOPeon ioPeon, + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, - CompressedObjectStrategy.CompressionStrategy compression, + CompressionStrategy compression, CompressionFactory.LongEncodingStrategy encoding, BitmapSerdeFactory bitmapSerdeFactory ) { return new LongColumnSerializer( - ioPeon, + segmentWriteOutMedium, filenameBase, IndexIO.BYTE_ORDER, compression, @@ -60,10 +60,10 @@ public static LongColumnSerializer create( ); } - private final IOPeon ioPeon; + private final SegmentWriteOutMedium segmentWriteOutMedium; private final String filenameBase; private final ByteOrder byteOrder; - private final CompressedObjectStrategy.CompressionStrategy compression; + private final CompressionStrategy compression; private final CompressionFactory.LongEncodingStrategy encoding; private final BitmapSerdeFactory bitmapSerdeFactory; private LongSupplierSerializer writer; @@ -71,16 +71,16 @@ public static LongColumnSerializer create( private MutableBitmap nullRowsBitmap; private int rowCount = 0; - public LongColumnSerializer( - IOPeon ioPeon, + private LongColumnSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression, + CompressionStrategy compression, CompressionFactory.LongEncodingStrategy encoding, BitmapSerdeFactory bitmapSerdeFactory ) { - this.ioPeon = ioPeon; + this.segmentWriteOutMedium = segmentWriteOutMedium; this.filenameBase = filenameBase; this.byteOrder = byteOrder; this.compression = compression; @@ -92,7 +92,7 @@ public LongColumnSerializer( public void open() throws IOException { writer = CompressionFactory.getLongSerializer( - ioPeon, + segmentWriteOutMedium, StringUtils.format("%s.long_column", filenameBase), byteOrder, encoding, @@ -100,8 +100,7 @@ public void open() throws IOException ); writer.open(); nullValueBitmapWriter = new ByteBufferWriter<>( - ioPeon, - StringUtils.format("%s.nullBitmap", filenameBase), + segmentWriteOutMedium, bitmapSerdeFactory.getObjectStrategy() ); nullValueBitmapWriter.open(); @@ -121,16 +120,9 @@ public void serialize(@Nullable Object obj) throws IOException } @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { - writer.close(); nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); - nullValueBitmapWriter.close(); - } - - @Override - public long getSerializedSize() - { long bitmapSize = nullRowsBitmap.isEmpty() ? 0L : nullValueBitmapWriter.getSerializedSize(); @@ -138,13 +130,12 @@ public long getSerializedSize() } @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { channel.write(ByteBuffer.wrap(Ints.toByteArray((int) writer.getSerializedSize()))); - writer.writeToChannel(channel, smoosher); + writer.writeTo(channel, smoosher); if (!nullRowsBitmap.isEmpty()) { - nullValueBitmapWriter.writeToChannel(channel); + nullValueBitmapWriter.writeTo(channel, smoosher); } } - } diff --git a/processing/src/main/java/io/druid/segment/LongDimensionHandler.java b/processing/src/main/java/io/druid/segment/LongDimensionHandler.java index 5782972520bf..19796815169d 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionHandler.java @@ -22,11 +22,10 @@ import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.GenericColumn; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.Indexed; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.Closeable; -import java.io.File; import java.io.IOException; public class LongDimensionHandler implements DimensionHandler @@ -52,13 +51,16 @@ public DimensionIndexer makeIndexer() @Override public DimensionMergerV9 makeMerger( - IndexSpec indexSpec, File outDir, IOPeon ioPeon, ColumnCapabilities capabilities, ProgressIndicator progress + IndexSpec indexSpec, + SegmentWriteOutMedium segmentWriteOutMedium, + ColumnCapabilities capabilities, + ProgressIndicator progress ) throws IOException { return new LongDimensionMergerV9( dimensionName, indexSpec, - ioPeon + segmentWriteOutMedium ); } diff --git a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java index a7645aa03a61..a3a6d94ea4fb 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java @@ -22,10 +22,10 @@ import com.google.common.base.Throwables; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; -import io.druid.segment.data.IOPeon; import io.druid.segment.serde.LongGenericColumnPartSerdeV2; +import io.druid.segment.data.CompressionStrategy; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.IOException; import java.nio.IntBuffer; @@ -36,33 +36,31 @@ public class LongDimensionMergerV9 implements DimensionMergerV9 protected String dimensionName; protected final IndexSpec indexSpec; - protected IOPeon ioPeon; protected LongColumnSerializer serializer; - public LongDimensionMergerV9( + LongDimensionMergerV9( String dimensionName, IndexSpec indexSpec, - IOPeon ioPeon + SegmentWriteOutMedium segmentWriteOutMedium ) { this.dimensionName = dimensionName; this.indexSpec = indexSpec; - this.ioPeon = ioPeon; try { - setupEncodedValueWriter(); + setupEncodedValueWriter(segmentWriteOutMedium); } catch (IOException ioe) { Throwables.propagate(ioe); } } - protected void setupEncodedValueWriter() throws IOException + protected void setupEncodedValueWriter(SegmentWriteOutMedium segmentWriteOutMedium) throws IOException { - final CompressedObjectStrategy.CompressionStrategy metCompression = indexSpec.getMetricCompression(); + final CompressionStrategy metCompression = indexSpec.getMetricCompression(); final CompressionFactory.LongEncodingStrategy longEncoding = indexSpec.getLongEncoding(); this.serializer = LongColumnSerializer.create( - ioPeon, + segmentWriteOutMedium, dimensionName, metCompression, longEncoding, @@ -104,7 +102,6 @@ public boolean canSkip() @Override public ColumnDescriptor makeColumnDescriptor() throws IOException { - serializer.close(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.LONG); builder.addSerde( diff --git a/processing/src/main/java/io/druid/segment/MetricHolder.java b/processing/src/main/java/io/druid/segment/MetricHolder.java index 63954a66107b..226e6a0c8848 100644 --- a/processing/src/main/java/io/druid/segment/MetricHolder.java +++ b/processing/src/main/java/io/druid/segment/MetricHolder.java @@ -40,7 +40,6 @@ public class MetricHolder private static final byte[] version = new byte[]{0x0}; private static final SerializerUtils serializerUtils = new SerializerUtils(); - public static MetricHolder fromByteBuffer(ByteBuffer buf, SmooshedFileMapper mapper) throws IOException { return fromByteBuffer(buf, null, mapper); @@ -60,7 +59,7 @@ public static MetricHolder fromByteBuffer(ByteBuffer buf, ObjectStrategy strateg switch (holder.type) { case FLOAT: - holder.floatType = CompressedFloatsIndexedSupplier.fromByteBuffer(buf, ByteOrder.nativeOrder(), mapper); + holder.floatType = CompressedFloatsIndexedSupplier.fromByteBuffer(buf, ByteOrder.nativeOrder()); break; case COMPLEX: if (strategy != null) { diff --git a/processing/src/main/java/io/druid/segment/Rowboat.java b/processing/src/main/java/io/druid/segment/Rowboat.java index 1ef2dbf3aa19..c047fd1c567b 100644 --- a/processing/src/main/java/io/druid/segment/Rowboat.java +++ b/processing/src/main/java/io/druid/segment/Rowboat.java @@ -51,7 +51,7 @@ public Rowboat( this.rowNum = rowNum; this.handlers = handlers; - this.comprisedRows = new Int2ObjectOpenHashMap<>(); + this.comprisedRows = new Int2ObjectOpenHashMap<>(1); } public long getTimestamp() diff --git a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java index 55e9cd395277..fe3387f5548b 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java @@ -21,15 +21,14 @@ import com.google.common.primitives.Ints; import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.DictionaryEncodedColumn; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedInts; import java.io.Closeable; -import java.io.File; import java.lang.reflect.Array; import java.util.Arrays; @@ -213,13 +212,12 @@ public DimensionIndexer makeIndexer() @Override public DimensionMergerV9 makeMerger( IndexSpec indexSpec, - File outDir, - IOPeon ioPeon, + SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, ProgressIndicator progress ) { - return new StringDimensionMergerV9(dimensionName, indexSpec, outDir, ioPeon, capabilities, progress); + return new StringDimensionMergerV9(dimensionName, indexSpec, segmentWriteOutMedium, capabilities, progress); } } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index d369143d69d2..44031413cd37 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -21,17 +21,13 @@ import com.google.common.base.Splitter; import com.google.common.base.Strings; -import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.google.common.io.ByteStreams; -import com.google.common.io.Files; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.MutableBitmap; import io.druid.collections.spatial.ImmutableRTree; import io.druid.collections.spatial.RTree; import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; -import io.druid.java.util.common.ByteBufferUtils; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; @@ -42,28 +38,24 @@ import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.BitmapValues; import io.druid.segment.data.ByteBufferWriter; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressedVSizeIndexedV3Writer; import io.druid.segment.data.CompressedVSizeIntsIndexedWriter; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.GenericIndexed; import io.druid.segment.data.GenericIndexedWriter; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.ImmutableRTreeObjectStrategy; import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedIntsWriter; import io.druid.segment.data.VSizeIndexedIntsWriter; import io.druid.segment.data.VSizeIndexedWriter; import io.druid.segment.serde.DictionaryEncodedColumnPartSerde; +import io.druid.segment.writeout.SegmentWriteOutMedium; import it.unimi.dsi.fastutil.ints.IntIterable; import it.unimi.dsi.fastutil.ints.IntIterator; import javax.annotation.Nonnull; -import java.io.Closeable; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.IntBuffer; -import java.nio.MappedByteBuffer; import java.util.ArrayList; import java.util.List; @@ -76,33 +68,33 @@ public class StringDimensionMergerV9 implements DimensionMergerV9 String.class ); protected static final int[] NULL_STR_DIM_ARRAY = new int[]{0}; - protected static final Splitter SPLITTER = Splitter.on(","); + private static final Splitter SPLITTER = Splitter.on(","); private IndexedIntsWriter encodedValueWriter; - protected String dimensionName; - protected GenericIndexedWriter dictionaryWriter; - protected GenericIndexedWriter bitmapWriter; - protected ByteBufferWriter spatialWriter; - protected ArrayList dimConversions; - protected int cardinality = 0; - protected boolean convertMissingValues = false; - protected boolean hasNull = false; - protected MutableBitmap nullRowsBitmap; - protected IOPeon ioPeon; - protected int rowCount = 0; - protected ColumnCapabilities capabilities; - protected final File outDir; - protected List adapters; - protected ProgressIndicator progress; - protected final IndexSpec indexSpec; - protected IndexMerger.DictionaryMergeIterator dictionaryMergeIterator; + private String dimensionName; + private GenericIndexedWriter dictionaryWriter; + private String firstDictionaryValue; + private int dictionarySize; + private GenericIndexedWriter bitmapWriter; + private ByteBufferWriter spatialWriter; + private ArrayList dimConversions; + private int cardinality = 0; + private boolean convertMissingValues = false; + private boolean hasNull = false; + private MutableBitmap nullRowsBitmap; + private final SegmentWriteOutMedium segmentWriteOutMedium; + private int rowCount = 0; + private ColumnCapabilities capabilities; + private List adapters; + private ProgressIndicator progress; + private final IndexSpec indexSpec; + private IndexMerger.DictionaryMergeIterator dictionaryMergeIterator; public StringDimensionMergerV9( String dimensionName, IndexSpec indexSpec, - File outDir, - IOPeon ioPeon, + SegmentWriteOutMedium segmentWriteOutMedium, ColumnCapabilities capabilities, ProgressIndicator progress ) @@ -110,8 +102,7 @@ public StringDimensionMergerV9( this.dimensionName = dimensionName; this.indexSpec = indexSpec; this.capabilities = capabilities; - this.outDir = outDir; - this.ioPeon = ioPeon; + this.segmentWriteOutMedium = segmentWriteOutMedium; this.progress = progress; nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); } @@ -162,21 +153,15 @@ public void writeMergedValueMetadata(List adapters) throws IOE } String dictFilename = StringUtils.format("%s.dim_values", dimensionName); - dictionaryWriter = new GenericIndexedWriter<>( - ioPeon, - dictFilename, - GenericIndexed.STRING_STRATEGY - ); + dictionaryWriter = new GenericIndexedWriter<>(segmentWriteOutMedium, dictFilename, GenericIndexed.STRING_STRATEGY); + firstDictionaryValue = null; + dictionarySize = 0; dictionaryWriter.open(); cardinality = 0; if (numMergeIndex > 1) { dictionaryMergeIterator = new IndexMerger.DictionaryMergeIterator(dimValueLookups, true); - - while (dictionaryMergeIterator.hasNext()) { - dictionaryWriter.write(dictionaryMergeIterator.next()); - } - + writeDictionary(() -> dictionaryMergeIterator); for (int i = 0; i < adapters.size(); i++) { if (dimValueLookups[i] != null && dictionaryMergeIterator.needConversion(i)) { dimConversions.set(i, dictionaryMergeIterator.conversions[i]); @@ -184,9 +169,7 @@ public void writeMergedValueMetadata(List adapters) throws IOE } cardinality = dictionaryMergeIterator.counter; } else if (numMergeIndex == 1) { - for (String value : dimValueLookup) { - dictionaryWriter.write(value); - } + writeDictionary(dimValueLookup); cardinality = dimValueLookup.size(); } @@ -196,34 +179,49 @@ public void writeMergedValueMetadata(List adapters) throws IOE cardinality, System.currentTimeMillis() - dimStartTime ); - dictionaryWriter.close(); setupEncodedValueWriter(); } + private void writeDictionary(Iterable dictionaryValues) throws IOException + { + for (String value : dictionaryValues) { + dictionaryWriter.write(value); + value = Strings.emptyToNull(value); + if (dictionarySize == 0) { + firstDictionaryValue = value; + } + dictionarySize++; + } + } + protected void setupEncodedValueWriter() throws IOException { - final CompressedObjectStrategy.CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); + final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); String filenameBase = StringUtils.format("%s.forward_dim", dimensionName); if (capabilities.hasMultipleValues()) { - encodedValueWriter = (compressionStrategy != CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED) - ? CompressedVSizeIndexedV3Writer.create( - ioPeon, - filenameBase, - cardinality, - compressionStrategy - ) - : new VSizeIndexedWriter(ioPeon, filenameBase, cardinality); + if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { + encodedValueWriter = CompressedVSizeIndexedV3Writer.create( + segmentWriteOutMedium, + filenameBase, + cardinality, + compressionStrategy + ); + } else { + encodedValueWriter = new VSizeIndexedWriter(segmentWriteOutMedium, cardinality); + } } else { - encodedValueWriter = (compressionStrategy != CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED) - ? CompressedVSizeIntsIndexedWriter.create( - ioPeon, - filenameBase, - cardinality, - compressionStrategy - ) - : new VSizeIndexedIntsWriter(ioPeon, filenameBase, cardinality); + if (compressionStrategy != CompressionStrategy.UNCOMPRESSED) { + encodedValueWriter = CompressedVSizeIntsIndexedWriter.create( + segmentWriteOutMedium, + filenameBase, + cardinality, + compressionStrategy + ); + } else { + encodedValueWriter = new VSizeIndexedIntsWriter(segmentWriteOutMedium, cardinality); + } } encodedValueWriter.open(); } @@ -281,87 +279,65 @@ public void writeIndexes(List segmentRowNumConversions) throws IOExce final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); String bmpFilename = StringUtils.format("%s.inverted", dimensionName); - bitmapWriter = new GenericIndexedWriter<>(ioPeon, bmpFilename, bitmapSerdeFactory.getObjectStrategy()); + bitmapWriter = new GenericIndexedWriter<>( + segmentWriteOutMedium, + bmpFilename, + indexSpec.getBitmapSerdeFactory().getObjectStrategy() + ); bitmapWriter.open(); bitmapWriter.setObjectsNotSorted(); - // write dim values to one single file because we need to read it - File dimValueFile = IndexIO.makeDimFile(outDir, dimensionName); - try (FileOutputStream fos = new FileOutputStream(dimValueFile)) { - ByteStreams.copy(dictionaryWriter.combineStreams(), fos); + BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory(); + + RTree tree = null; + boolean hasSpatial = capabilities.hasSpatialIndexes(); + if (hasSpatial) { + spatialWriter = new ByteBufferWriter<>( + segmentWriteOutMedium, + new ImmutableRTreeObjectStrategy(bitmapFactory) + ); + spatialWriter.open(); + tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bitmapFactory), bitmapFactory); } - final MappedByteBuffer dimValsMapped = Files.map(dimValueFile); - try ( - Closeable toCloseEncodedValueWriter = encodedValueWriter; - Closeable toCloseBitmapWriter = bitmapWriter; - // We need to free the ByteBuffers allocated by the dictionary merge iterator here, - // these buffers are used by dictIdSeeker in mergeBitmaps() below. The iterator is created and only used - // in writeMergedValueMetadata(), but the buffers are still used until after mergeBitmaps(). - Closeable toCloseDictionaryMergeIterator = dictionaryMergeIterator; - Closeable dimValsMappedUnmapper = () -> ByteBufferUtils.unmap(dimValsMapped) - ) { - Indexed dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY); - BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory(); - - RTree tree = null; - boolean hasSpatial = capabilities.hasSpatialIndexes(); - if (hasSpatial) { - spatialWriter = new ByteBufferWriter<>( - ioPeon, - StringUtils.format("%s.spatial", dimensionName), - new ImmutableRTreeObjectStrategy(bmpFactory) - ); - spatialWriter.open(); - tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bmpFactory), bmpFactory); - } + IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName); + + //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result. + for (int dictId = 0; dictId < dictionarySize; dictId++) { + progress.progress(); + mergeBitmaps( + segmentRowNumConversions, + bitmapFactory, + tree, + hasSpatial, + dictIdSeeker, + dictId + ); + } - IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName); - - //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result. - for (int dictId = 0; dictId < dimVals.size(); dictId++) { - progress.progress(); - mergeBitmaps( - segmentRowNumConversions, - dimVals, - bmpFactory, - tree, - hasSpatial, - dictIdSeeker, - dictId, - adapters, - dimensionName, - nullRowsBitmap, - bitmapWriter - ); - } + if (hasSpatial) { + spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree)); + } - if (hasSpatial) { - spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree)); - spatialWriter.close(); - } + log.info( + "Completed dim[%s] inverted with cardinality[%,d] in %,d millis.", + dimensionName, + dictionarySize, + System.currentTimeMillis() - dimStartTime + ); - log.info( - "Completed dim[%s] inverted with cardinality[%,d] in %,d millis.", - dimensionName, - dimVals.size(), - System.currentTimeMillis() - dimStartTime - ); + if (dictionaryMergeIterator != null) { + dictionaryMergeIterator.close(); } } - static void mergeBitmaps( + void mergeBitmaps( List segmentRowNumConversions, - Indexed dimVals, BitmapFactory bmpFactory, RTree tree, boolean hasSpatial, IndexSeeker[] dictIdSeeker, - int dictId, - List adapters, - String dimensionName, - MutableBitmap nullRowsBitmap, - GenericIndexedWriter bitmapWriter + int dictId ) throws IOException { List convertedInvertedIndexesToMerge = Lists.newArrayListWithCapacity(adapters.size()); @@ -396,15 +372,16 @@ static void mergeBitmaps( prevRow = row; } - if ((dictId == 0) && Iterables.getFirst(dimVals, null) == null) { + if (dictId == 0 && firstDictionaryValue == null) { mergedIndexes.or(nullRowsBitmap); } bitmapWriter.write(bmpFactory.makeImmutableBitmap(mergedIndexes)); if (hasSpatial) { - String dimVal = dimVals.get(dictId); - if (!Strings.isNullOrEmpty(dimVal)) { + + String dimVal = dictionaryWriter.get(dictId); + if (dimVal != null) { List stringCoords = Lists.newArrayList(SPLITTER.split(dimVal)); float[] coords = new float[stringCoords.size()]; for (int j = 0; j < coords.length; j++) { @@ -426,7 +403,7 @@ public ColumnDescriptor makeColumnDescriptor() { // Now write everything boolean hasMultiValue = capabilities.hasMultipleValues(); - final CompressedObjectStrategy.CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); + final CompressionStrategy compressionStrategy = indexSpec.getDimensionCompression(); final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); @@ -438,7 +415,7 @@ public ColumnDescriptor makeColumnDescriptor() .withValue( encodedValueWriter, hasMultiValue, - compressionStrategy != CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED + compressionStrategy != CompressionStrategy.UNCOMPRESSED ) .withBitmapSerdeFactory(bitmapSerdeFactory) .withBitmapIndex(bitmapWriter) diff --git a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java index 277b990ce8c2..7f100f02d388 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java @@ -27,6 +27,7 @@ import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.segment.serde.ColumnPartSerde; +import io.druid.segment.serde.Serializer; import java.io.IOException; import java.nio.ByteBuffer; @@ -35,7 +36,7 @@ /** */ -public class ColumnDescriptor +public class ColumnDescriptor implements Serializer { public static Builder builder() { @@ -76,21 +77,21 @@ public List getParts() return parts; } - public long numBytes() + @Override + public long getSerializedSize() throws IOException { - long retVal = 0; - + long size = 0; for (ColumnPartSerde part : parts) { - retVal += part.getSerializer().numBytes(); + size += part.getSerializer().getSerializedSize(); } - - return retVal; + return size; } - public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { for (ColumnPartSerde part : parts) { - part.getSerializer().write(channel, smoosher); + part.getSerializer().writeTo(channel, smoosher); } } diff --git a/processing/src/main/java/io/druid/segment/data/BlockLayoutDoubleSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/BlockLayoutDoubleSupplierSerializer.java index 6069000176ea..287deafe4d54 100644 --- a/processing/src/main/java/io/druid/segment/data/BlockLayoutDoubleSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/BlockLayoutDoubleSupplierSerializer.java @@ -19,52 +19,49 @@ package io.druid.segment.data; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import com.google.common.primitives.Doubles; -import com.google.common.primitives.Ints; -import io.druid.collections.ResourceHolder; -import io.druid.collections.StupidResourceHolder; +import io.druid.java.util.common.io.Closer; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.CompressedPools; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.IOException; -import java.io.InputStream; +import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.DoubleBuffer; -import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; public class BlockLayoutDoubleSupplierSerializer implements DoubleSupplierSerializer { - private final IOPeon ioPeon; - private final int sizePer; - private final GenericIndexedWriter> flattener; - private final CompressedObjectStrategy.CompressionStrategy compression; - private final String metaFile; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((BlockLayoutDoubleSupplierSerializer x) -> CompressedDoublesIndexedSupplier.VERSION) + .writeInt(x -> x.numInserted) + .writeInt(x -> CompressedPools.BUFFER_SIZE / Double.BYTES) + .writeByte(x -> x.compression.getId()); + + private final GenericIndexedWriter flattener; + private final CompressionStrategy compression; - private long metaCount = 0; private int numInserted = 0; - private DoubleBuffer endBuffer; + private ByteBuffer endBuffer; - public BlockLayoutDoubleSupplierSerializer( - IOPeon ioPeon, + BlockLayoutDoubleSupplierSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, - ByteOrder order, - CompressedObjectStrategy.CompressionStrategy compression + ByteOrder byteOrder, + CompressionStrategy compression ) { - this.ioPeon = ioPeon; - this.sizePer = CompressedPools.BUFFER_SIZE / Doubles.BYTES; - this.flattener = new GenericIndexedWriter<>( - ioPeon, filenameBase, CompressedDoubleBufferObjectStrategy.getBufferForOrder(order, compression, sizePer) + this.flattener = GenericIndexedWriter.ofCompressedByteBuffers( + segmentWriteOutMedium, + filenameBase, + compression, + CompressedPools.BUFFER_SIZE ); - this.metaFile = filenameBase + ".format"; this.compression = compression; - - endBuffer = DoubleBuffer.allocate(sizePer); - endBuffer.mark(); + CompressionStrategy.Compressor compressor = compression.getCompressor(); + Closer closer = segmentWriteOutMedium.getCloser(); + this.endBuffer = compressor.allocateInBuffer(CompressedPools.BUFFER_SIZE, closer).order(byteOrder); } @Override @@ -76,48 +73,42 @@ public void open() throws IOException @Override public void add(double value) throws IOException { + if (endBuffer == null) { + throw new IllegalStateException("written out already"); + } if (!endBuffer.hasRemaining()) { endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - endBuffer = DoubleBuffer.allocate(sizePer); - endBuffer.mark(); + flattener.write(endBuffer); + endBuffer.clear(); } - endBuffer.put(value); + endBuffer.putDouble(value); ++numInserted; } @Override - public long getSerializedSize() + public long getSerializedSize() throws IOException { - return metaCount + flattener.getSerializedSize(); + writeEndBuffer(); + return metaSerdeHelper.size(this) + flattener.getSerializedSize(); } @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - try (InputStream meta = ioPeon.makeInputStream(metaFile)) { - ByteStreams.copy(Channels.newChannel(meta), channel); - flattener.writeToChannel(channel, smoosher); - } + writeEndBuffer(); + metaSerdeHelper.writeTo(channel, this); + flattener.writeTo(channel, smoosher); } - @Override - public void close() throws IOException + private void writeEndBuffer() throws IOException { - endBuffer.limit(endBuffer.position()); - endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - endBuffer = null; - flattener.close(); - - try (CountingOutputStream metaOut = new CountingOutputStream(ioPeon.makeOutputStream(metaFile))) { - metaOut.write(CompressedDoublesIndexedSupplier.version); - metaOut.write(Ints.toByteArray(numInserted)); - metaOut.write(Ints.toByteArray(sizePer)); - metaOut.write(compression.getId()); - metaOut.close(); - metaCount = metaOut.getCount(); + if (endBuffer != null) { + endBuffer.flip(); + if (endBuffer.remaining() > 0) { + flattener.write(endBuffer); + } + endBuffer = null; } } } diff --git a/processing/src/main/java/io/druid/segment/data/BlockLayoutFloatSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/BlockLayoutFloatSupplierSerializer.java index ddb112b5063e..d0961c801ab7 100644 --- a/processing/src/main/java/io/druid/segment/data/BlockLayoutFloatSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/BlockLayoutFloatSupplierSerializer.java @@ -19,53 +19,48 @@ package io.druid.segment.data; -import com.google.common.io.ByteSink; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import com.google.common.primitives.Floats; -import com.google.common.primitives.Ints; -import io.druid.collections.ResourceHolder; -import io.druid.collections.StupidResourceHolder; +import io.druid.java.util.common.io.Closer; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.CompressedPools; +import io.druid.segment.serde.MetaSerdeHelper; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.FloatBuffer; -import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; public class BlockLayoutFloatSupplierSerializer implements FloatSupplierSerializer { - private final IOPeon ioPeon; - private final int sizePer; - private final GenericIndexedWriter> flattener; - private final CompressedObjectStrategy.CompressionStrategy compression; - private final String metaFile; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((BlockLayoutFloatSupplierSerializer x) -> CompressedFloatsIndexedSupplier.VERSION) + .writeInt(x -> x.numInserted) + .writeInt(x -> CompressedPools.BUFFER_SIZE / Float.BYTES) + .writeByte(x -> x.compression.getId()); + + private final GenericIndexedWriter flattener; + private final CompressionStrategy compression; - private long metaCount = 0; private int numInserted = 0; - private FloatBuffer endBuffer; + private ByteBuffer endBuffer; - public BlockLayoutFloatSupplierSerializer( - IOPeon ioPeon, + BlockLayoutFloatSupplierSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, - ByteOrder order, - CompressedObjectStrategy.CompressionStrategy compression + ByteOrder byteOrder, + CompressionStrategy compression ) { - this.ioPeon = ioPeon; - this.sizePer = CompressedPools.BUFFER_SIZE / Floats.BYTES; - this.flattener = new GenericIndexedWriter<>( - ioPeon, filenameBase, CompressedFloatBufferObjectStrategy.getBufferForOrder(order, compression, sizePer) + this.flattener = GenericIndexedWriter.ofCompressedByteBuffers( + segmentWriteOutMedium, + filenameBase, + compression, + CompressedPools.BUFFER_SIZE ); - this.metaFile = filenameBase + ".format"; this.compression = compression; - - endBuffer = FloatBuffer.allocate(sizePer); - endBuffer.mark(); + CompressionStrategy.Compressor compressor = compression.getCompressor(); + Closer closer = segmentWriteOutMedium.getCloser(); + this.endBuffer = compressor.allocateInBuffer(CompressedPools.BUFFER_SIZE, closer).order(byteOrder); } @Override @@ -83,59 +78,41 @@ public int size() @Override public void add(float value) throws IOException { + if (endBuffer == null) { + throw new IllegalStateException("written out already"); + } if (!endBuffer.hasRemaining()) { endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - endBuffer = FloatBuffer.allocate(sizePer); - endBuffer.mark(); + flattener.write(endBuffer); + endBuffer.clear(); } - - endBuffer.put(value); + endBuffer.putFloat(value); ++numInserted; } @Override - public void closeAndConsolidate(ByteSink consolidatedOut) throws IOException - { - close(); - try (OutputStream out = consolidatedOut.openStream(); - InputStream meta = ioPeon.makeInputStream(metaFile)) { - ByteStreams.copy(meta, out); - ByteStreams.copy(flattener.combineStreams(), out); - } - } - - @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { - endBuffer.limit(endBuffer.position()); - endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - endBuffer = null; - flattener.close(); - - try (CountingOutputStream metaOut = new CountingOutputStream(ioPeon.makeOutputStream(metaFile))) { - metaOut.write(CompressedFloatsIndexedSupplier.version); - metaOut.write(Ints.toByteArray(numInserted)); - metaOut.write(Ints.toByteArray(sizePer)); - metaOut.write(compression.getId()); - metaOut.close(); - metaCount = metaOut.getCount(); - } + writeEndBuffer(); + return metaSerdeHelper.size(this) + flattener.getSerializedSize(); } @Override - public long getSerializedSize() + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - return metaCount + flattener.getSerializedSize(); + writeEndBuffer(); + metaSerdeHelper.writeTo(channel, this); + flattener.writeTo(channel, smoosher); } - @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + private void writeEndBuffer() throws IOException { - try (InputStream meta = ioPeon.makeInputStream(metaFile)) { - ByteStreams.copy(Channels.newChannel(meta), channel); - flattener.writeToChannel(channel, smoosher); + if (endBuffer != null) { + endBuffer.flip(); + if (endBuffer.remaining() > 0) { + flattener.write(endBuffer); + } + endBuffer = null; } } } diff --git a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedDoubleSupplier.java b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedDoubleSupplier.java index 779665b41a50..85bdebd11f0b 100644 --- a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedDoubleSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedDoubleSupplier.java @@ -20,10 +20,8 @@ package io.druid.segment.data; import com.google.common.base.Supplier; -import com.google.common.primitives.Doubles; import io.druid.collections.ResourceHolder; import io.druid.java.util.common.guava.CloseQuietly; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -41,18 +39,13 @@ public BlockLayoutIndexedDoubleSupplier( int sizePer, ByteBuffer fromBuffer, ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper fileMapper + CompressionStrategy strategy ) { baseDoubleBuffers = GenericIndexed.read( fromBuffer, - VSizeCompressedObjectStrategy.getBufferForOrder(byteOrder, - strategy, - sizePer * Doubles.BYTES - ), - fileMapper + new DecompressingByteBufferObjectStrategy(byteOrder, strategy) ); this.totalSize = totalSize; diff --git a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedFloatSupplier.java b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedFloatSupplier.java index 972da01b8290..77f249ce4e5c 100644 --- a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedFloatSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedFloatSupplier.java @@ -20,11 +20,9 @@ package io.druid.segment.data; import com.google.common.base.Supplier; -import com.google.common.primitives.Floats; import io.druid.collections.ResourceHolder; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -41,18 +39,12 @@ public BlockLayoutIndexedFloatSupplier( int sizePer, ByteBuffer fromBuffer, ByteOrder order, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper mapper + CompressionStrategy strategy ) { baseFloatBuffers = GenericIndexed.read( fromBuffer, - VSizeCompressedObjectStrategy.getBufferForOrder( - order, - strategy, - sizePer * Floats.BYTES - ), - mapper + new DecompressingByteBufferObjectStrategy(order, strategy) ); this.totalSize = totalSize; this.sizePer = sizePer; diff --git a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedLongSupplier.java b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedLongSupplier.java index 7202b5bc02f8..54f2ef62d9eb 100644 --- a/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedLongSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/BlockLayoutIndexedLongSupplier.java @@ -23,7 +23,6 @@ import io.druid.collections.ResourceHolder; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -43,19 +42,10 @@ public BlockLayoutIndexedLongSupplier( ByteBuffer fromBuffer, ByteOrder order, CompressionFactory.LongEncodingReader reader, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper fileMapper + CompressionStrategy strategy ) { - baseLongBuffers = GenericIndexed.read( - fromBuffer, - VSizeCompressedObjectStrategy.getBufferForOrder( - order, - strategy, - reader.getNumBytes(sizePer) - ), - fileMapper - ); + baseLongBuffers = GenericIndexed.read(fromBuffer, new DecompressingByteBufferObjectStrategy(order, strategy)); this.totalSize = totalSize; this.sizePer = sizePer; this.baseReader = reader; diff --git a/processing/src/main/java/io/druid/segment/data/BlockLayoutLongSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/BlockLayoutLongSupplierSerializer.java index abdae5da7a97..f44a91633d83 100644 --- a/processing/src/main/java/io/druid/segment/data/BlockLayoutLongSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/BlockLayoutLongSupplierSerializer.java @@ -19,60 +19,50 @@ package io.druid.segment.data; -import com.google.common.io.ByteSink; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import com.google.common.primitives.Ints; -import io.druid.collections.ResourceHolder; -import io.druid.collections.StupidResourceHolder; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.CompressedPools; +import io.druid.segment.serde.MetaSerdeHelper; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; public class BlockLayoutLongSupplierSerializer implements LongSupplierSerializer { + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((BlockLayoutLongSupplierSerializer x) -> CompressedLongsIndexedSupplier.VERSION) + .writeInt(x -> x.numInserted) + .writeInt(x -> x.sizePer) + .writeSomething(CompressionFactory.longEncodingWriter(x -> x.writer, x -> x.compression)); - private final IOPeon ioPeon; private final int sizePer; private final CompressionFactory.LongEncodingWriter writer; - private final GenericIndexedWriter> flattener; - private final CompressedObjectStrategy.CompressionStrategy compression; - private final String metaFile; - private long metaCount = 0; - + private final GenericIndexedWriter flattener; + private final CompressionStrategy compression; private int numInserted = 0; + private int numInsertedForNextFlush; private ByteBuffer endBuffer = null; - public BlockLayoutLongSupplierSerializer( - IOPeon ioPeon, + BlockLayoutLongSupplierSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, - ByteOrder order, + ByteOrder byteOrder, CompressionFactory.LongEncodingWriter writer, - CompressedObjectStrategy.CompressionStrategy compression + CompressionStrategy compression ) { - this.ioPeon = ioPeon; this.sizePer = writer.getBlockSize(CompressedPools.BUFFER_SIZE); - this.flattener = new GenericIndexedWriter<>( - ioPeon, - filenameBase, - VSizeCompressedObjectStrategy.getBufferForOrder( - order, - compression, - writer.getNumBytes(sizePer) - ) - ); - this.metaFile = filenameBase + ".format"; + int bufferSize = writer.getNumBytes(sizePer); + this.flattener = GenericIndexedWriter.ofCompressedByteBuffers(segmentWriteOutMedium, filenameBase, compression, bufferSize); this.writer = writer; this.compression = compression; + CompressionStrategy.Compressor compressor = compression.getCompressor(); + endBuffer = compressor.allocateInBuffer(writer.getNumBytes(sizePer), segmentWriteOutMedium.getCloser()).order(byteOrder); + writer.setBuffer(endBuffer); + numInsertedForNextFlush = sizePer; } @Override @@ -90,15 +80,15 @@ public int size() @Override public void add(long value) throws IOException { - if (numInserted % sizePer == 0) { - if (endBuffer != null) { - writer.flush(); - endBuffer.limit(endBuffer.position()); - endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - } - endBuffer = ByteBuffer.allocate(writer.getNumBytes(sizePer)); - writer.setBuffer(endBuffer); + if (endBuffer == null) { + throw new IllegalStateException("written out already"); + } + if (numInserted == numInsertedForNextFlush) { + numInsertedForNextFlush += sizePer; + writer.flush(); + endBuffer.flip(); + flattener.write(endBuffer); + endBuffer.clear(); } writer.write(value); @@ -106,50 +96,29 @@ public void add(long value) throws IOException } @Override - public void closeAndConsolidate(ByteSink consolidatedOut) throws IOException - { - close(); - try (OutputStream out = consolidatedOut.openStream(); - InputStream meta = ioPeon.makeInputStream(metaFile)) { - ByteStreams.copy(meta, out); - ByteStreams.copy(flattener.combineStreams(), out); - } - } - - @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { - if (endBuffer != null) { - writer.flush(); - endBuffer.limit(endBuffer.position()); - endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - } - endBuffer = null; - flattener.close(); - - try (CountingOutputStream metaOut = new CountingOutputStream(ioPeon.makeOutputStream(metaFile))) { - metaOut.write(CompressedLongsIndexedSupplier.version); - metaOut.write(Ints.toByteArray(numInserted)); - metaOut.write(Ints.toByteArray(sizePer)); - writer.putMeta(metaOut, compression); - metaOut.close(); - metaCount = metaOut.getCount(); - } + writeEndBuffer(); + return metaSerdeHelper.size(this) + flattener.getSerializedSize(); } @Override - public long getSerializedSize() + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - return metaCount + flattener.getSerializedSize(); + writeEndBuffer(); + metaSerdeHelper.writeTo(channel, this); + flattener.writeTo(channel, smoosher); } - @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + private void writeEndBuffer() throws IOException { - try (InputStream meta = ioPeon.makeInputStream(metaFile)) { - ByteStreams.copy(Channels.newChannel(meta), channel); - flattener.writeToChannel(channel, smoosher); + if (endBuffer != null) { + writer.flush(); + endBuffer.flip(); + if (endBuffer.remaining() > 0) { + flattener.write(endBuffer); + } + endBuffer = null; } } } diff --git a/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java b/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java index 7ccb99ebc0f0..e655f7fa92b0 100644 --- a/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java @@ -25,7 +25,7 @@ /** */ -public class ByteBufferSerializer +public class ByteBufferSerializer { @Nullable public static T read(ByteBuffer buffer, ObjectStrategy strategy) @@ -41,5 +41,4 @@ public static T read(ByteBuffer buffer, ObjectStrategy strategy) return strategy.fromByteBuffer(bufferToUse, size); } - } diff --git a/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java b/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java index 5b5fde997e71..27c5a08dd41f 100644 --- a/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java +++ b/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java @@ -19,114 +19,60 @@ package io.druid.segment.data; -import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import com.google.common.io.InputSupplier; import com.google.common.primitives.Ints; -import io.druid.common.utils.SerializerUtils; -import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.Serializer; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.writeout.WriteOutBytes; -import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; -import java.util.Arrays; /** */ -public class ByteBufferWriter implements Closeable +public class ByteBufferWriter implements Serializer { - private final IOPeon ioPeon; - private final String filenameBase; + private final SegmentWriteOutMedium segmentWriteOutMedium; private final ObjectStrategy strategy; - private CountingOutputStream headerOut = null; - private CountingOutputStream valueOut = null; - private final ByteBuffer helperBuffer = ByteBuffer.allocate(Ints.BYTES); + private WriteOutBytes headerOut = null; + private WriteOutBytes valueOut = null; - public ByteBufferWriter( - IOPeon ioPeon, - String filenameBase, - ObjectStrategy strategy - ) + public ByteBufferWriter(SegmentWriteOutMedium segmentWriteOutMedium, ObjectStrategy strategy) { - this.ioPeon = ioPeon; - this.filenameBase = filenameBase; + this.segmentWriteOutMedium = segmentWriteOutMedium; this.strategy = strategy; } public void open() throws IOException { - headerOut = new CountingOutputStream(ioPeon.makeOutputStream(makeFilename("header"))); - valueOut = new CountingOutputStream(ioPeon.makeOutputStream(makeFilename("value"))); + headerOut = segmentWriteOutMedium.makeWriteOutBytes(); + valueOut = segmentWriteOutMedium.makeWriteOutBytes(); } public void write(T objectToWrite) throws IOException { - byte[] bytesToWrite = strategy.toBytes(objectToWrite); - int length = bytesToWrite == null ? -1 : bytesToWrite.length; - SerializerUtils.writeBigEndianIntToOutputStream(headerOut, length, helperBuffer); - if (bytesToWrite != null) { - valueOut.write(bytesToWrite); - } + long sizeBefore = valueOut.size(); + strategy.writeTo(objectToWrite, valueOut); + headerOut.writeInt(Ints.checkedCast(valueOut.size() - sizeBefore)); } - private String makeFilename(String suffix) + @Override + public long getSerializedSize() throws IOException { - return StringUtils.format("%s.%s", filenameBase, suffix); + return headerOut.size() + valueOut.size(); } @Override - public void close() throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - headerOut.close(); - valueOut.close(); - - final long numBytesWritten = headerOut.getCount() + valueOut.getCount(); + final long numBytesWritten = headerOut.size() + valueOut.size(); Preconditions.checkState( numBytesWritten < Integer.MAX_VALUE, "Wrote[%s] bytes, which is too many.", numBytesWritten ); - } - - public long getSerializedSize() - { - return headerOut.getCount() + valueOut.getCount(); - } - public InputSupplier combineStreams() - { - return ByteStreams.join( - Iterables.transform( - Arrays.asList("header", "value"), - new Function>() - { - @Override - public InputSupplier apply(final String input) - { - return new InputSupplier() - { - @Override - public InputStream getInput() throws IOException - { - return ioPeon.makeInputStream(makeFilename(input)); - } - }; - } - } - ) - ); - } - - public void writeToChannel(WritableByteChannel channel) throws IOException - { - try (final ReadableByteChannel from = Channels.newChannel(combineStreams().getInput())) { - ByteStreams.copy(from, channel); - } + headerOut.writeTo(channel); + valueOut.writeTo(channel); } } diff --git a/processing/src/main/java/io/druid/segment/data/CompressedByteBufferObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedByteBufferObjectStrategy.java deleted file mode 100644 index 96afb159c784..000000000000 --- a/processing/src/main/java/io/druid/segment/data/CompressedByteBufferObjectStrategy.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - -import io.druid.java.util.common.guava.Comparators; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -public class CompressedByteBufferObjectStrategy extends FixedSizeCompressedObjectStrategy -{ - - public static CompressedByteBufferObjectStrategy getBufferForOrder(final ByteOrder order, final CompressionStrategy compression, final int sizePer) - { - return new CompressedByteBufferObjectStrategy(order, compression, sizePer); - } - - public CompressedByteBufferObjectStrategy( - ByteOrder order, - CompressionStrategy compression, - final int sizePer - ) - { - super( - order, new BufferConverter() - { - @Override - public ByteBuffer convert(ByteBuffer buf) - { - return buf; - } - - @Override - public int compare(ByteBuffer lhs, ByteBuffer rhs) - { - return Comparators.naturalNullsFirst().compare(lhs, rhs); - } - - @Override - public int sizeOf(int count) - { - return count; // 1 byte per element - } - - @Override - public ByteBuffer combine(ByteBuffer into, ByteBuffer from) - { - return into.put(from); - } - }, compression, sizePer - ); - } -} diff --git a/processing/src/main/java/io/druid/segment/data/CompressedDoubleBufferObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedDoubleBufferObjectStrategy.java deleted file mode 100644 index 5e7778056ba9..000000000000 --- a/processing/src/main/java/io/druid/segment/data/CompressedDoubleBufferObjectStrategy.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - - -import com.google.common.primitives.Doubles; -import io.druid.java.util.common.guava.Comparators; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.DoubleBuffer; - -public class CompressedDoubleBufferObjectStrategy extends FixedSizeCompressedObjectStrategy -{ - public static CompressedDoubleBufferObjectStrategy getBufferForOrder( - final ByteOrder order, - final CompressionStrategy compression, - final int size - ) - { - return new CompressedDoubleBufferObjectStrategy(order, compression, size); - } - private CompressedDoubleBufferObjectStrategy( - ByteOrder order, - CompressionStrategy compression, - int sizePer - ) - { - super(order, new BufferConverter() - { - @Override - public DoubleBuffer convert(ByteBuffer buf) - { - return buf.asDoubleBuffer(); - } - - @Override - public int compare(DoubleBuffer lhs, DoubleBuffer rhs) - { - return Comparators.naturalNullsFirst().compare(lhs, rhs); - } - - @Override - public int sizeOf(int count) - { - return count * Doubles.BYTES; - } - - @Override - public DoubleBuffer combine(ByteBuffer into, DoubleBuffer from) - { - return into.asDoubleBuffer().put(from); - } - }, compression, sizePer); - } -} diff --git a/processing/src/main/java/io/druid/segment/data/CompressedDoublesIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedDoublesIndexedSupplier.java index 5297f4cb5ba3..3023e42a03fe 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedDoublesIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedDoublesIndexedSupplier.java @@ -21,7 +21,6 @@ import com.google.common.base.Supplier; import io.druid.java.util.common.IAE; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -29,7 +28,7 @@ public class CompressedDoublesIndexedSupplier { public static final byte LZF_VERSION = 0x1; - public static final byte version = 0x2; + public static final byte VERSION = 0x2; private CompressedDoublesIndexedSupplier() { @@ -37,30 +36,27 @@ private CompressedDoublesIndexedSupplier() public static Supplier fromByteBuffer( ByteBuffer buffer, - ByteOrder order, - SmooshedFileMapper mapper + ByteOrder order ) { byte versionFromBuffer = buffer.get(); - if (versionFromBuffer == LZF_VERSION || versionFromBuffer == version) { + if (versionFromBuffer == LZF_VERSION || versionFromBuffer == VERSION) { final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); - CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.LZF; - if (versionFromBuffer == version) { + CompressionStrategy compression = CompressionStrategy.LZF; + if (versionFromBuffer == VERSION) { byte compressionId = buffer.get(); - compression = CompressedObjectStrategy.CompressionStrategy.forId(compressionId); + compression = CompressionStrategy.forId(compressionId); } return CompressionFactory.getDoubleSupplier( totalSize, sizePer, buffer.asReadOnlyBuffer(), order, - compression, - mapper + compression ); } throw new IAE("Unknown version[%s]", versionFromBuffer); } - } diff --git a/processing/src/main/java/io/druid/segment/data/CompressedFloatBufferObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedFloatBufferObjectStrategy.java deleted file mode 100644 index 09bb6b5819d8..000000000000 --- a/processing/src/main/java/io/druid/segment/data/CompressedFloatBufferObjectStrategy.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - -import com.google.common.primitives.Floats; -import io.druid.java.util.common.guava.Comparators; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.FloatBuffer; - -/** -*/ -public class CompressedFloatBufferObjectStrategy extends FixedSizeCompressedObjectStrategy -{ - public static CompressedFloatBufferObjectStrategy getBufferForOrder(final ByteOrder order, final CompressionStrategy compression, final int sizePer) - { - return new CompressedFloatBufferObjectStrategy(order, compression, sizePer); - } - - private CompressedFloatBufferObjectStrategy(final ByteOrder order, final CompressionStrategy compression, final int sizePer) - { - super( - order, - new BufferConverter() - { - @Override - public FloatBuffer convert(ByteBuffer buf) - { - return buf.asFloatBuffer(); - } - - @Override - public int compare(FloatBuffer lhs, FloatBuffer rhs) - { - return Comparators.naturalNullsFirst().compare(lhs, rhs); - } - - @Override - public int sizeOf(int count) - { - return count * Floats.BYTES; - } - - @Override - public FloatBuffer combine(ByteBuffer into, FloatBuffer from) - { - return into.asFloatBuffer().put(from); - } - }, - compression, - sizePer - ); - } -} diff --git a/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java index 8fb80677a9a2..db0477d66536 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java @@ -20,32 +20,40 @@ package io.druid.segment.data; import com.google.common.base.Supplier; -import com.google.common.primitives.Ints; +import io.druid.io.Channels; import io.druid.java.util.common.IAE; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.serde.Serializer; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; -public class CompressedFloatsIndexedSupplier implements Supplier +public class CompressedFloatsIndexedSupplier implements Supplier, Serializer { public static final byte LZF_VERSION = 0x1; - public static final byte version = 0x2; + public static final byte VERSION = 0x2; + + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((CompressedFloatsIndexedSupplier x) -> VERSION) + .writeInt(x -> x.totalSize) + .writeInt(x -> x.sizePer) + .writeByte(x -> x.compression.getId()); private final int totalSize; private final int sizePer; private final ByteBuffer buffer; private final Supplier supplier; - private final CompressedObjectStrategy.CompressionStrategy compression; + private final CompressionStrategy compression; CompressedFloatsIndexedSupplier( int totalSize, int sizePer, ByteBuffer buffer, Supplier supplier, - CompressedObjectStrategy.CompressionStrategy compression + CompressionStrategy compression ) { this.totalSize = totalSize; @@ -61,43 +69,37 @@ public IndexedFloats get() return supplier.get(); } - public long getSerializedSize() + @Override + public long getSerializedSize() throws IOException { - return buffer.remaining() + 1 + 4 + 4 + 1; + return metaSerdeHelper.size(this) + (long) buffer.remaining(); } - public void writeToChannel(WritableByteChannel channel) throws IOException + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{version})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(totalSize))); - channel.write(ByteBuffer.wrap(Ints.toByteArray(sizePer))); - channel.write(ByteBuffer.wrap(new byte[]{compression.getId()})); - channel.write(buffer.asReadOnlyBuffer()); + metaSerdeHelper.writeTo(channel, this); + Channels.writeFully(channel, buffer.asReadOnlyBuffer()); } - public static CompressedFloatsIndexedSupplier fromByteBuffer( - ByteBuffer buffer, - ByteOrder order, - SmooshedFileMapper mapper - ) + public static CompressedFloatsIndexedSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); - if (versionFromBuffer == LZF_VERSION || versionFromBuffer == version) { + if (versionFromBuffer == LZF_VERSION || versionFromBuffer == VERSION) { final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); - CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.LZF; - if (versionFromBuffer == version) { + CompressionStrategy compression = CompressionStrategy.LZF; + if (versionFromBuffer == VERSION) { byte compressionId = buffer.get(); - compression = CompressedObjectStrategy.CompressionStrategy.forId(compressionId); + compression = CompressionStrategy.forId(compressionId); } Supplier supplier = CompressionFactory.getFloatSupplier( totalSize, sizePer, buffer.asReadOnlyBuffer(), order, - compression, - mapper + compression ); return new CompressedFloatsIndexedSupplier( totalSize, diff --git a/processing/src/main/java/io/druid/segment/data/CompressedIntBufferObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedIntBufferObjectStrategy.java deleted file mode 100644 index b3fd010d0ee8..000000000000 --- a/processing/src/main/java/io/druid/segment/data/CompressedIntBufferObjectStrategy.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - -import com.google.common.primitives.Ints; -import io.druid.java.util.common.guava.Comparators; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.IntBuffer; - -public class CompressedIntBufferObjectStrategy extends FixedSizeCompressedObjectStrategy -{ - - public static CompressedIntBufferObjectStrategy getBufferForOrder(final ByteOrder order, final CompressionStrategy compression, final int sizePer) - { - return new CompressedIntBufferObjectStrategy(order, compression, sizePer); - } - - private CompressedIntBufferObjectStrategy(final ByteOrder order, final CompressionStrategy compression, final int sizePer) - { - super( - order, - new BufferConverter() - { - @Override - public IntBuffer convert(ByteBuffer buf) - { - return buf.asIntBuffer(); - } - - @Override - public int compare(IntBuffer lhs, IntBuffer rhs) - { - return Comparators.naturalNullsFirst().compare(lhs, rhs); - } - - @Override - public int sizeOf(int count) - { - return count * Ints.BYTES; - } - - @Override - public IntBuffer combine(ByteBuffer into, IntBuffer from) - { - return into.asIntBuffer().put(from); - } - }, - compression, - sizePer - ); - } -} diff --git a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java index b4ead1f97ddf..888a0bcdd719 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedSupplier.java @@ -19,16 +19,19 @@ package io.druid.segment.data; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.io.Closeables; import com.google.common.primitives.Ints; import io.druid.collections.ResourceHolder; -import io.druid.collections.StupidResourceHolder; import io.druid.java.util.common.IAE; import io.druid.java.util.common.guava.CloseQuietly; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.Closer; +import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.CompressedPools; +import io.druid.segment.serde.MetaSerdeHelper; +import it.unimi.dsi.fastutil.ints.IntArrayList; import java.io.IOException; import java.nio.ByteBuffer; @@ -36,24 +39,28 @@ import java.nio.IntBuffer; import java.nio.channels.WritableByteChannel; import java.util.Iterator; -import java.util.List; public class CompressedIntsIndexedSupplier implements WritableSupplier { public static final byte VERSION = 0x2; public static final int MAX_INTS_IN_BUFFER = CompressedPools.BUFFER_SIZE / Ints.BYTES; + private static MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((CompressedIntsIndexedSupplier x) -> VERSION) + .writeInt(x -> x.totalSize) + .writeInt(x -> x.sizePer) + .writeByte(x -> x.compression.getId()); private final int totalSize; private final int sizePer; - private final GenericIndexed> baseIntBuffers; - private final CompressedObjectStrategy.CompressionStrategy compression; + private final GenericIndexed> baseIntBuffers; + private final CompressionStrategy compression; - CompressedIntsIndexedSupplier( + private CompressedIntsIndexedSupplier( int totalSize, int sizePer, - GenericIndexed> baseIntBuffers, - CompressedObjectStrategy.CompressionStrategy compression + GenericIndexed> baseIntBuffers, + CompressionStrategy compression ) { this.totalSize = totalSize; @@ -91,55 +98,36 @@ public int get(int index) } @Override - public long getSerializedSize() + public long getSerializedSize() throws IOException { - return 1 + // version - 4 + // totalSize - 4 + // sizePer - 1 + // compressionId - baseIntBuffers.getSerializedSize(); // data + return metaSerdeHelper.size(this) + baseIntBuffers.getSerializedSize(); } @Override - public void writeToChannel(WritableByteChannel channel) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{VERSION})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(totalSize))); - channel.write(ByteBuffer.wrap(Ints.toByteArray(sizePer))); - channel.write(ByteBuffer.wrap(new byte[]{compression.getId()})); - baseIntBuffers.writeToChannel(channel); + metaSerdeHelper.writeTo(channel, this); + baseIntBuffers.writeTo(channel, smoosher); } - /** - * For testing. Do not use unless you like things breaking - */ - GenericIndexed> getBaseIntBuffers() + @VisibleForTesting + GenericIndexed getBaseIntBuffers() { return baseIntBuffers; } - public static CompressedIntsIndexedSupplier fromByteBuffer( - ByteBuffer buffer, - ByteOrder order, - SmooshedFileMapper fileMapper - ) + public static CompressedIntsIndexedSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); if (versionFromBuffer == VERSION) { final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); - final CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.forId( - buffer.get() - ); + final CompressionStrategy compression = CompressionStrategy.forId(buffer.get()); return new CompressedIntsIndexedSupplier( totalSize, sizePer, - GenericIndexed.read( - buffer, - CompressedIntBufferObjectStrategy.getBufferForOrder(order, compression, sizePer), - fileMapper - ), + GenericIndexed.read(buffer, new DecompressingByteBufferObjectStrategy(order, compression)), compression ); } @@ -147,8 +135,13 @@ public static CompressedIntsIndexedSupplier fromByteBuffer( throw new IAE("Unknown version[%s]", versionFromBuffer); } - public static CompressedIntsIndexedSupplier fromIntBuffer( - final IntBuffer buffer, final int chunkFactor, final ByteOrder byteOrder, CompressedObjectStrategy.CompressionStrategy compression + @VisibleForTesting + static CompressedIntsIndexedSupplier fromIntBuffer( + final IntBuffer buffer, + final int chunkFactor, + final ByteOrder byteOrder, + final CompressionStrategy compression, + final Closer closer ) { Preconditions.checkArgument( @@ -158,15 +151,20 @@ public static CompressedIntsIndexedSupplier fromIntBuffer( return new CompressedIntsIndexedSupplier( buffer.remaining(), chunkFactor, - GenericIndexed.fromIterable( - new Iterable>() + GenericIndexed.ofCompressedByteBuffers( + new Iterable() { @Override - public Iterator> iterator() + public Iterator iterator() { - return new Iterator>() + return new Iterator() { - IntBuffer myBuffer = buffer.asReadOnlyBuffer(); + final IntBuffer myBuffer = buffer.asReadOnlyBuffer(); + final ByteBuffer retVal = compression + .getCompressor() + .allocateInBuffer(chunkFactor * Ints.BYTES, closer) + .order(byteOrder); + final IntBuffer retValAsIntBuffer = retVal.asIntBuffer(); @Override public boolean hasNext() @@ -175,16 +173,17 @@ public boolean hasNext() } @Override - public ResourceHolder next() + public ByteBuffer next() { - IntBuffer retVal = myBuffer.asReadOnlyBuffer(); - + int initialLimit = myBuffer.limit(); if (chunkFactor < myBuffer.remaining()) { - retVal.limit(retVal.position() + chunkFactor); + myBuffer.limit(myBuffer.position() + chunkFactor); } - myBuffer.position(myBuffer.position() + retVal.remaining()); - - return StupidResourceHolder.create(retVal); + retValAsIntBuffer.clear(); + retValAsIntBuffer.put(myBuffer); + myBuffer.limit(initialLimit); + retVal.clear().limit(retValAsIntBuffer.position() * Ints.BYTES); + return retVal; } @Override @@ -195,17 +194,22 @@ public void remove() }; } }, - CompressedIntBufferObjectStrategy.getBufferForOrder(byteOrder, compression, chunkFactor) + compression, + chunkFactor * Ints.BYTES, + byteOrder, + closer ), compression ); } + @VisibleForTesting public static CompressedIntsIndexedSupplier fromList( - final List list, + final IntArrayList list, final int chunkFactor, final ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression + final CompressionStrategy compression, + final Closer closer ) { Preconditions.checkArgument( @@ -215,14 +219,18 @@ public static CompressedIntsIndexedSupplier fromList( return new CompressedIntsIndexedSupplier( list.size(), chunkFactor, - GenericIndexed.fromIterable( - new Iterable>() + GenericIndexed.ofCompressedByteBuffers( + new Iterable() { @Override - public Iterator> iterator() + public Iterator iterator() { - return new Iterator>() + return new Iterator() { + private final ByteBuffer retVal = compression + .getCompressor() + .allocateInBuffer(chunkFactor * Ints.BYTES, closer) + .order(byteOrder); int position = 0; @Override @@ -232,21 +240,15 @@ public boolean hasNext() } @Override - public ResourceHolder next() + public ByteBuffer next() { - IntBuffer retVal = IntBuffer.allocate(chunkFactor); - - if (chunkFactor > list.size() - position) { - retVal.limit(list.size() - position); + int blockSize = Math.min(list.size() - position, chunkFactor); + retVal.clear(); + for (int limit = position + blockSize; position < limit; position++) { + retVal.putInt(list.getInt(position)); } - final List ints = list.subList(position, position + retVal.remaining()); - for (int value : ints) { - retVal.put(value); - } - retVal.rewind(); - position += retVal.remaining(); - - return StupidResourceHolder.create(retVal); + retVal.flip(); + return retVal; } @Override @@ -257,7 +259,10 @@ public void remove() }; } }, - CompressedIntBufferObjectStrategy.getBufferForOrder(byteOrder, compression, chunkFactor) + compression, + chunkFactor * Ints.BYTES, + byteOrder, + closer ), compression ); @@ -265,10 +270,10 @@ public void remove() private class CompressedIndexedInts implements IndexedInts { - final Indexed> singleThreadedIntBuffers = baseIntBuffers.singleThreaded(); + final Indexed> singleThreadedIntBuffers = baseIntBuffers.singleThreaded(); int currIndex = -1; - ResourceHolder holder; + ResourceHolder holder; IntBuffer buffer; @Override @@ -294,7 +299,7 @@ protected void loadBuffer(int bufferNum) { CloseQuietly.close(holder); holder = singleThreadedIntBuffers.get(bufferNum); - buffer = holder.get(); + buffer = holder.get().asIntBuffer(); currIndex = bufferNum; } diff --git a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedWriter.java index a95bd5673aee..2feacc73d05a 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedIntsIndexedWriter.java @@ -19,15 +19,14 @@ package io.druid.segment.data; -import com.google.common.primitives.Ints; -import io.druid.collections.ResourceHolder; -import io.druid.collections.StupidResourceHolder; +import io.druid.java.util.common.io.Closer; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.IntBuffer; import java.nio.channels.WritableByteChannel; /** @@ -37,36 +36,55 @@ public class CompressedIntsIndexedWriter extends SingleValueIndexedIntsWriter { private static final byte VERSION = CompressedIntsIndexedSupplier.VERSION; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((CompressedIntsIndexedWriter x) -> VERSION) + .writeInt(x -> x.numInserted) + .writeInt(x -> x.chunkFactor) + .writeByte(x -> x.compression.getId()); + private final int chunkFactor; - private final CompressedObjectStrategy.CompressionStrategy compression; - private final GenericIndexedWriter> flattener; - private IntBuffer endBuffer; + private final CompressionStrategy compression; + private final GenericIndexedWriter flattener; + private ByteBuffer endBuffer; private int numInserted; - public CompressedIntsIndexedWriter( - final IOPeon ioPeon, + CompressedIntsIndexedWriter( + final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int chunkFactor, final ByteOrder byteOrder, - final CompressedObjectStrategy.CompressionStrategy compression + final CompressionStrategy compression ) { - this(chunkFactor, compression, new GenericIndexedWriter<>( - ioPeon, filenameBase, CompressedIntBufferObjectStrategy.getBufferForOrder(byteOrder, compression, chunkFactor) - )); + this( + segmentWriteOutMedium, + chunkFactor, + byteOrder, + compression, + GenericIndexedWriter.ofCompressedByteBuffers( + segmentWriteOutMedium, + filenameBase, + compression, + chunkFactor * Integer.BYTES + ) + ); } - public CompressedIntsIndexedWriter( + CompressedIntsIndexedWriter( + final SegmentWriteOutMedium segmentWriteOutMedium, final int chunkFactor, - final CompressedObjectStrategy.CompressionStrategy compression, - GenericIndexedWriter> flattener + final ByteOrder byteOrder, + final CompressionStrategy compression, + final GenericIndexedWriter flattener ) { this.chunkFactor = chunkFactor; this.compression = compression; - this.endBuffer = IntBuffer.allocate(chunkFactor); - this.numInserted = 0; this.flattener = flattener; + CompressionStrategy.Compressor compressor = compression.getCompressor(); + Closer closer = segmentWriteOutMedium.getCloser(); + this.endBuffer = compressor.allocateInBuffer(chunkFactor * Integer.BYTES, closer).order(byteOrder); + this.numInserted = 0; } @Override @@ -78,48 +96,41 @@ public void open() throws IOException @Override protected void addValue(int val) throws IOException { + if (endBuffer == null) { + throw new IllegalStateException("written out already"); + } if (!endBuffer.hasRemaining()) { endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - endBuffer = IntBuffer.allocate(chunkFactor); + flattener.write(endBuffer); + endBuffer.clear(); } - endBuffer.put(val); + endBuffer.putInt(val); numInserted++; } @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { - try { - if (numInserted > 0) { - endBuffer.limit(endBuffer.position()); - endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - } - endBuffer = null; - } - finally { - flattener.close(); - } + writeEndBuffer(); + return metaSerdeHelper.size(this) + flattener.getSerializedSize(); } @Override - public long getSerializedSize() + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - return 1 + // version - Ints.BYTES + // numInserted - Ints.BYTES + // chunkFactor - 1 + // compression id - flattener.getSerializedSize(); + writeEndBuffer(); + metaSerdeHelper.writeTo(channel, this); + flattener.writeTo(channel, smoosher); } - @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + private void writeEndBuffer() throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{VERSION})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(numInserted))); - channel.write(ByteBuffer.wrap(Ints.toByteArray(chunkFactor))); - channel.write(ByteBuffer.wrap(new byte[]{compression.getId()})); - flattener.writeToChannel(channel, smoosher); + if (endBuffer != null) { + endBuffer.flip(); + if (endBuffer.remaining() > 0) { + flattener.write(endBuffer); + } + endBuffer = null; + } } } diff --git a/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java index 8d9edcdb9ffa..b408b54de6ae 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java @@ -20,9 +20,11 @@ package io.druid.segment.data; import com.google.common.base.Supplier; -import com.google.common.primitives.Ints; +import io.druid.io.Channels; import io.druid.java.util.common.IAE; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.serde.Serializer; import java.io.IOException; import java.nio.ByteBuffer; @@ -31,17 +33,32 @@ /** */ -public class CompressedLongsIndexedSupplier implements Supplier +public class CompressedLongsIndexedSupplier implements Supplier, Serializer { public static final byte LZF_VERSION = 0x1; - public static final byte version = 0x2; + public static final byte VERSION = 0x2; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((CompressedLongsIndexedSupplier x) -> VERSION) + .writeInt(x -> x.totalSize) + .writeInt(x -> x.sizePer) + .maybeWriteByte( + x -> x.encoding != CompressionFactory.LEGACY_LONG_ENCODING_FORMAT, + x -> CompressionFactory.setEncodingFlag(x.compression.getId()) + ) + .writeByte(x -> { + if (x.encoding != CompressionFactory.LEGACY_LONG_ENCODING_FORMAT) { + return x.encoding.getId(); + } else { + return x.compression.getId(); + } + }); private final int totalSize; private final int sizePer; private final ByteBuffer buffer; private final Supplier supplier; - private final CompressedObjectStrategy.CompressionStrategy compression; + private final CompressionStrategy compression; private final CompressionFactory.LongEncodingFormat encoding; CompressedLongsIndexedSupplier( @@ -49,7 +66,7 @@ public class CompressedLongsIndexedSupplier implements Supplier int sizePer, ByteBuffer buffer, Supplier supplier, - CompressedObjectStrategy.CompressionStrategy compression, + CompressionStrategy compression, CompressionFactory.LongEncodingFormat encoding ) { @@ -67,45 +84,35 @@ public IndexedLongs get() return supplier.get(); } - public long getSerializedSize() + @Override + public long getSerializedSize() throws IOException { - return buffer.remaining() + 1 + 4 + 4 + 1 + (encoding == CompressionFactory.LEGACY_LONG_ENCODING_FORMAT ? 0 : 1); + return metaSerdeHelper.size(this) + (long) buffer.remaining(); } - public void writeToChannel(WritableByteChannel channel) throws IOException + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{version})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(totalSize))); - channel.write(ByteBuffer.wrap(Ints.toByteArray(sizePer))); - if (encoding == CompressionFactory.LEGACY_LONG_ENCODING_FORMAT) { - channel.write(ByteBuffer.wrap(new byte[]{compression.getId()})); - } else { - channel.write(ByteBuffer.wrap(new byte[]{CompressionFactory.setEncodingFlag(compression.getId())})); - channel.write(ByteBuffer.wrap(new byte[]{encoding.getId()})); - } - channel.write(buffer.asReadOnlyBuffer()); + metaSerdeHelper.writeTo(channel, this); + Channels.writeFully(channel, buffer.asReadOnlyBuffer()); } - public static CompressedLongsIndexedSupplier fromByteBuffer( - ByteBuffer buffer, - ByteOrder order, - SmooshedFileMapper fileMapper - ) + public static CompressedLongsIndexedSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order) { byte versionFromBuffer = buffer.get(); - if (versionFromBuffer == LZF_VERSION || versionFromBuffer == version) { + if (versionFromBuffer == LZF_VERSION || versionFromBuffer == VERSION) { final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); - CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.LZF; + CompressionStrategy compression = CompressionStrategy.LZF; CompressionFactory.LongEncodingFormat encoding = CompressionFactory.LEGACY_LONG_ENCODING_FORMAT; - if (versionFromBuffer == version) { + if (versionFromBuffer == VERSION) { byte compressionId = buffer.get(); if (CompressionFactory.hasEncodingFlag(compressionId)) { encoding = CompressionFactory.LongEncodingFormat.forId(buffer.get()); compressionId = CompressionFactory.clearEncodingFlag(compressionId); } - compression = CompressedObjectStrategy.CompressionStrategy.forId(compressionId); + compression = CompressionStrategy.forId(compressionId); } Supplier supplier = CompressionFactory.getLongSupplier( totalSize, @@ -113,8 +120,7 @@ public static CompressedLongsIndexedSupplier fromByteBuffer( buffer.asReadOnlyBuffer(), order, encoding, - compression, - fileMapper + compression ); return new CompressedLongsIndexedSupplier( totalSize, diff --git a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java deleted file mode 100644 index 92b6736927f1..000000000000 --- a/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java +++ /dev/null @@ -1,402 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; -import com.google.common.collect.Maps; -import com.ning.compress.BufferRecycler; -import com.ning.compress.lzf.LZFDecoder; -import com.ning.compress.lzf.LZFEncoder; -import io.druid.collections.ResourceHolder; -import io.druid.java.util.common.StringUtils; -import io.druid.java.util.common.logger.Logger; -import io.druid.segment.CompressedPools; -import net.jpountz.lz4.LZ4Factory; -import net.jpountz.lz4.LZ4SafeDecompressor; -import org.apache.commons.lang.ArrayUtils; - -import java.io.IOException; -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.Map; - -/** - */ -public class CompressedObjectStrategy implements ObjectStrategy> -{ - private static final Logger log = new Logger(CompressedObjectStrategy.class); - public static final CompressionStrategy DEFAULT_COMPRESSION_STRATEGY = CompressionStrategy.LZ4; - - /** - * Compression strategy is used to compress block of bytes without knowledge of what data the bytes represents. - * - * When adding compression strategy, do not use id in the range [0x7C, 0xFD] (greater than 123 or less than -2), since - * a flag mechanism is used in CompressionFactory that involves subtracting the value 126 from the compression id - * (see {@link CompressionFactory#FLAG_BOUND}) - */ - public enum CompressionStrategy - { - LZF((byte) 0x0) { - @Override - public Decompressor getDecompressor() - { - return LZFDecompressor.defaultDecompressor; - } - - @Override - public Compressor getCompressor() - { - return LZFCompressor.defaultCompressor; - } - }, - - LZ4((byte) 0x1) { - @Override - public Decompressor getDecompressor() - { - return LZ4Decompressor.defaultDecompressor; - } - - @Override - public Compressor getCompressor() - { - return LZ4Compressor.defaultCompressor; - } - }, - UNCOMPRESSED((byte) 0xFF) { - @Override - public Decompressor getDecompressor() - { - return UncompressedDecompressor.defaultDecompressor; - } - - @Override - public Compressor getCompressor() - { - return UncompressedCompressor.defaultCompressor; - } - }, - /* - This value indicate no compression strategy should be used, and compression should not be block based - Currently only IndexedLong support non block based compression, and other types treat this as UNCOMPRESSED - */ - NONE((byte) 0xFE) { - @Override - public Decompressor getDecompressor() - { - throw new UnsupportedOperationException("NONE compression strategy shouldn't use any decompressor"); - } - - @Override - public Compressor getCompressor() - { - throw new UnsupportedOperationException("NONE compression strategy shouldn't use any compressor"); - } - }; - - final byte id; - - CompressionStrategy(byte id) - { - this.id = id; - } - - public byte getId() - { - return id; - } - - public abstract Compressor getCompressor(); - - public abstract Decompressor getDecompressor(); - - @JsonValue - @Override - public String toString() - { - return StringUtils.toLowerCase(this.name()); - } - - @JsonCreator - public static CompressionStrategy fromString(String name) - { - return valueOf(StringUtils.toUpperCase(name)); - } - - static final Map idMap = Maps.newHashMap(); - - static { - for (CompressionStrategy strategy : CompressionStrategy.values()) { - idMap.put(strategy.getId(), strategy); - } - } - - public static CompressionStrategy forId(byte id) - { - return idMap.get(id); - } - - // TODO remove this method and change all its callers to use all CompressionStrategy values when NONE type is supported by all types - public static CompressionStrategy[] noNoneValues() - { - return (CompressionStrategy[]) ArrayUtils.removeElement(CompressionStrategy.values(), NONE); - } - } - - public interface Decompressor - { - /** - * Implementations of this method are expected to call out.flip() after writing to the output buffer - * - * @param in - * @param numBytes - * @param out - */ - void decompress(ByteBuffer in, int numBytes, ByteBuffer out); - - void decompress(ByteBuffer in, int numBytes, ByteBuffer out, int decompressedSize); - } - - public interface Compressor - { - /** - * Currently assumes buf is an array backed ByteBuffer - * - * @param bytes - * - * @return - */ - byte[] compress(byte[] bytes); - } - - public static class UncompressedCompressor implements Compressor - { - private static final UncompressedCompressor defaultCompressor = new UncompressedCompressor(); - - @Override - public byte[] compress(byte[] bytes) - { - return bytes; - } - } - - public static class UncompressedDecompressor implements Decompressor - { - private static final UncompressedDecompressor defaultDecompressor = new UncompressedDecompressor(); - - @Override - public void decompress(ByteBuffer in, int numBytes, ByteBuffer out) - { - final ByteBuffer copyBuffer = in.duplicate(); - copyBuffer.limit(copyBuffer.position() + numBytes); - out.put(copyBuffer).flip(); - in.position(in.position() + numBytes); - } - - @Override - public void decompress(ByteBuffer in, int numBytes, ByteBuffer out, int decompressedSize) - { - decompress(in, numBytes, out); - } - } - - public static class LZFDecompressor implements Decompressor - { - private static final LZFDecompressor defaultDecompressor = new LZFDecompressor(); - - @Override - public void decompress(ByteBuffer in, int numBytes, ByteBuffer out) - { - final byte[] bytes = new byte[numBytes]; - in.get(bytes); - - try (final ResourceHolder outputBytesHolder = CompressedPools.getOutputBytes()) { - final byte[] outputBytes = outputBytesHolder.get(); - final int numDecompressedBytes = LZFDecoder.decode(bytes, outputBytes); - out.put(outputBytes, 0, numDecompressedBytes); - out.flip(); - } - catch (IOException e) { - log.error(e, "Error decompressing data"); - } - } - - @Override - public void decompress(ByteBuffer in, int numBytes, ByteBuffer out, int decompressedSize) - { - decompress(in, numBytes, out); - } - } - - public static class LZFCompressor implements Compressor - { - private static final LZFCompressor defaultCompressor = new LZFCompressor(); - - @Override - public byte[] compress(byte[] bytes) - { - try (final ResourceHolder bufferRecycler = CompressedPools.getBufferRecycler()) { - return LZFEncoder.encode(bytes, 0, bytes.length, bufferRecycler.get()); - } - } - } - - public static class LZ4Decompressor implements Decompressor - { - private static final LZ4SafeDecompressor lz4Safe = LZ4Factory.fastestInstance().safeDecompressor(); - private static final LZ4Decompressor defaultDecompressor = new LZ4Decompressor(); - - @Override - public void decompress(ByteBuffer in, int numBytes, ByteBuffer out) - { - // Since decompressed size is NOT known, must use lz4Safe - // lz4Safe.decompress does not modify buffer positions - final int numDecompressedBytes = lz4Safe.decompress( - in, - in.position(), - numBytes, - out, - out.position(), - out.remaining() - ); - out.limit(out.position() + numDecompressedBytes); - } - - @Override - public void decompress(ByteBuffer in, int numBytes, ByteBuffer out, int decompressedSize) - { - // lz4Safe.decompress does not modify buffer positions. - // Using lz4Safe API for forward-compatibility with https://github.com/druid-io/druid/pull/4762, which doesn't - // always compressed blocks of the same size. - lz4Safe.decompress( - in, - in.position(), - numBytes, - out, - out.position(), - decompressedSize - ); - out.limit(out.position() + decompressedSize); - } - } - - public static class LZ4Compressor implements Compressor - { - private static final LZ4Compressor defaultCompressor = new LZ4Compressor(); - private static final net.jpountz.lz4.LZ4Compressor lz4High = LZ4Factory.fastestInstance().highCompressor(); - - @Override - public byte[] compress(byte[] bytes) - { - return lz4High.compress(bytes); - } - } - - protected final ByteOrder order; - protected final BufferConverter converter; - protected final Decompressor decompressor; - private final Compressor compressor; - - protected CompressedObjectStrategy( - final ByteOrder order, - final BufferConverter converter, - final CompressionStrategy compression - ) - { - this.order = order; - this.converter = converter; - this.decompressor = compression.getDecompressor(); - this.compressor = compression.getCompressor(); - } - - @Override - @SuppressWarnings("unchecked") - public Class> getClazz() - { - return (Class) ResourceHolder.class; - } - - @Override - public ResourceHolder fromByteBuffer(ByteBuffer buffer, int numBytes) - { - final ResourceHolder bufHolder = CompressedPools.getByteBuf(order); - final ByteBuffer buf = bufHolder.get(); - buf.position(0); - buf.limit(buf.capacity()); - - decompress(buffer, numBytes, buf); - return new ResourceHolder() - { - @Override - public T get() - { - return converter.convert(buf); - } - - @Override - public void close() - { - bufHolder.close(); - } - }; - } - - protected void decompress( - ByteBuffer buffer, - int numBytes, - ByteBuffer buf - ) - { - decompressor.decompress(buffer, numBytes, buf); - } - - @Override - public byte[] toBytes(ResourceHolder holder) - { - T val = holder.get(); - ByteBuffer buf = bufferFor(val); - converter.combine(buf, val); - return compressor.compress(buf.array()); - } - - protected ByteBuffer bufferFor(T val) - { - return ByteBuffer.allocate(converter.sizeOf(val.remaining())).order(order); - } - - @Override - public int compare(ResourceHolder o1, ResourceHolder o2) - { - return converter.compare(o1.get(), o2.get()); - } - - public interface BufferConverter - { - T convert(ByteBuffer buf); - - int compare(T lhs, T rhs); - - int sizeOf(int count); - - T combine(ByteBuffer into, T from); - } -} diff --git a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIndexedV3Writer.java b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIndexedV3Writer.java index 660127cdeb14..b5b2574f7fa4 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIndexedV3Writer.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIndexedV3Writer.java @@ -17,46 +17,46 @@ * under the License. */ -/** - * Streams array of integers out in the binary format described by CompressedVSizeIndexedV3Supplier - */ + package io.druid.segment.data; -import io.druid.java.util.common.StringUtils; +import io.druid.io.Channels; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.CompressedVSizeIndexedV3Supplier; import io.druid.segment.IndexIO; +import it.unimi.dsi.fastutil.ints.IntList; +import it.unimi.dsi.fastutil.ints.IntLists; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; -import java.util.ArrayList; -import java.util.List; +/** + * Streams array of integers out in the binary format described by CompressedVSizeIndexedV3Supplier + */ public class CompressedVSizeIndexedV3Writer extends MultiValueIndexedIntsWriter { private static final byte VERSION = CompressedVSizeIndexedV3Supplier.VERSION; - private static final List EMPTY_LIST = new ArrayList<>(); - public static CompressedVSizeIndexedV3Writer create( - final IOPeon ioPeon, + final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, - final CompressedObjectStrategy.CompressionStrategy compression + final CompressionStrategy compression ) { return new CompressedVSizeIndexedV3Writer( new CompressedIntsIndexedWriter( - ioPeon, - StringUtils.format("%s.offsets", filenameBase), + segmentWriteOutMedium, + filenameBase, CompressedIntsIndexedSupplier.MAX_INTS_IN_BUFFER, IndexIO.BYTE_ORDER, compression ), new CompressedVSizeIntsIndexedWriter( - ioPeon, - StringUtils.format("%s.values", filenameBase), + segmentWriteOutMedium, + filenameBase, maxValue, CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForValue(maxValue), IndexIO.BYTE_ORDER, @@ -68,8 +68,9 @@ public static CompressedVSizeIndexedV3Writer create( private final CompressedIntsIndexedWriter offsetWriter; private final CompressedVSizeIntsIndexedWriter valueWriter; private int offset; + private boolean lastOffsetWritten = false; - public CompressedVSizeIndexedV3Writer( + CompressedVSizeIndexedV3Writer( CompressedIntsIndexedWriter offsetWriter, CompressedVSizeIntsIndexedWriter valueWriter ) @@ -87,43 +88,42 @@ public void open() throws IOException } @Override - protected void addValues(List vals) throws IOException + protected void addValues(IntList vals) throws IOException { + if (lastOffsetWritten) { + throw new IllegalStateException("written out already"); + } if (vals == null) { - vals = EMPTY_LIST; + vals = IntLists.EMPTY_LIST; } offsetWriter.add(offset); - for (Integer val : vals) { - valueWriter.add(val); + for (int i = 0; i < vals.size(); i++) { + valueWriter.add(vals.getInt(i)); } offset += vals.size(); } @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { - try { - offsetWriter.add(offset); - } - finally { - offsetWriter.close(); - valueWriter.close(); - } + writeLastOffset(); + return 1 + offsetWriter.getSerializedSize() + valueWriter.getSerializedSize(); } @Override - public long getSerializedSize() + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - return 1 + // version - offsetWriter.getSerializedSize() + - valueWriter.getSerializedSize(); + writeLastOffset(); + Channels.writeFully(channel, ByteBuffer.wrap(new byte[]{VERSION})); + offsetWriter.writeTo(channel, smoosher); + valueWriter.writeTo(channel, smoosher); } - @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + private void writeLastOffset() throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{VERSION})); - offsetWriter.writeToChannel(channel, smoosher); - valueWriter.writeToChannel(channel, smoosher); + if (!lastOffsetWritten) { + offsetWriter.add(offset); + lastOffsetWritten = true; + } } } diff --git a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java index 41b407eeabe5..625351374288 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplier.java @@ -25,12 +25,15 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Shorts; import io.druid.collections.ResourceHolder; -import io.druid.collections.StupidResourceHolder; +import io.druid.common.utils.ByteUtils; import io.druid.java.util.common.IAE; import io.druid.java.util.common.guava.CloseQuietly; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.java.util.common.io.Closer; +import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.CompressedPools; +import io.druid.segment.serde.MetaSerdeHelper; +import it.unimi.dsi.fastutil.ints.IntList; import java.io.IOException; import java.nio.ByteBuffer; @@ -39,26 +42,32 @@ import java.nio.ShortBuffer; import java.nio.channels.WritableByteChannel; import java.util.Iterator; -import java.util.List; public class CompressedVSizeIntsIndexedSupplier implements WritableSupplier { public static final byte VERSION = 0x2; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((CompressedVSizeIntsIndexedSupplier x) -> VERSION) + .writeByte(x -> ByteUtils.checkedCast(x.numBytes)) + .writeInt(x -> x.totalSize) + .writeInt(x -> x.sizePer) + .writeByte(x -> x.compression.getId()); + private final int totalSize; private final int sizePer; private final int numBytes; private final int bigEndianShift; private final int littleEndianMask; private final GenericIndexed> baseBuffers; - private final CompressedObjectStrategy.CompressionStrategy compression; + private final CompressionStrategy compression; - CompressedVSizeIntsIndexedSupplier( + private CompressedVSizeIntsIndexedSupplier( int totalSize, int sizePer, int numBytes, GenericIndexed> baseBuffers, - CompressedObjectStrategy.CompressionStrategy compression + CompressionStrategy compression ) { Preconditions.checkArgument( @@ -79,10 +88,10 @@ public static int maxIntsInBufferForBytes(int numBytes) { int maxSizePer = (CompressedPools.BUFFER_SIZE - bufferPadding(numBytes)) / numBytes; // round down to the nearest power of 2 - return 1 << (Integer.SIZE - 1 - Integer.numberOfLeadingZeros(maxSizePer)); + return Integer.highestOneBit(maxSizePer); } - public static int bufferPadding(int numBytes) + static int bufferPadding(int numBytes) { // when numBytes == 3 we need to pad the buffer to allow reading an extra byte // beyond the end of the last value, since we use buffer.getInt() to read values. @@ -117,26 +126,17 @@ public IndexedInts get() } } - @Override - public long getSerializedSize() + public long getSerializedSize() throws IOException { - return 1 + // version - 1 + // numBytes - Ints.BYTES + // totalSize - Ints.BYTES + // sizePer - 1 + // compression id - baseBuffers.getSerializedSize(); // data + return metaSerdeHelper.size(this) + baseBuffers.getSerializedSize(); } @Override - public void writeToChannel(WritableByteChannel channel) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{VERSION, (byte) numBytes})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(totalSize))); - channel.write(ByteBuffer.wrap(Ints.toByteArray(sizePer))); - channel.write(ByteBuffer.wrap(new byte[]{compression.getId()})); - baseBuffers.writeToChannel(channel); + metaSerdeHelper.writeTo(channel, this); + baseBuffers.writeTo(channel, smoosher); } @VisibleForTesting @@ -147,8 +147,7 @@ GenericIndexed> getBaseBuffers() public static CompressedVSizeIntsIndexedSupplier fromByteBuffer( ByteBuffer buffer, - ByteOrder order, - SmooshedFileMapper fileMapper + ByteOrder order ) { byte versionFromBuffer = buffer.get(); @@ -157,21 +156,14 @@ public static CompressedVSizeIntsIndexedSupplier fromByteBuffer( final int numBytes = buffer.get(); final int totalSize = buffer.getInt(); final int sizePer = buffer.getInt(); - final int chunkBytes = sizePer * numBytes + bufferPadding(numBytes); - final CompressedObjectStrategy.CompressionStrategy compression = CompressedObjectStrategy.CompressionStrategy.forId( - buffer.get() - ); + final CompressionStrategy compression = CompressionStrategy.forId(buffer.get()); return new CompressedVSizeIntsIndexedSupplier( totalSize, sizePer, numBytes, - GenericIndexed.read( - buffer, - CompressedByteBufferObjectStrategy.getBufferForOrder(order, compression, chunkBytes), - fileMapper - ), + GenericIndexed.read(buffer, new DecompressingByteBufferObjectStrategy(order, compression)), compression ); @@ -180,16 +172,18 @@ public static CompressedVSizeIntsIndexedSupplier fromByteBuffer( throw new IAE("Unknown version[%s]", versionFromBuffer); } + @VisibleForTesting public static CompressedVSizeIntsIndexedSupplier fromList( - final List list, + final IntList list, final int maxValue, final int chunkFactor, final ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression + final CompressionStrategy compression, + final Closer closer ) { final int numBytes = VSizeIndexedInts.getNumBytesForMax(maxValue); - final int chunkBytes = chunkFactor * numBytes + bufferPadding(numBytes); + final int chunkBytes = chunkFactor * numBytes; Preconditions.checkArgument( chunkFactor <= maxIntsInBufferForBytes(numBytes), @@ -201,15 +195,19 @@ public static CompressedVSizeIntsIndexedSupplier fromList( list.size(), chunkFactor, numBytes, - GenericIndexed.fromIterable( - new Iterable>() + GenericIndexed.ofCompressedByteBuffers( + new Iterable() { @Override - public Iterator> iterator() + public Iterator iterator() { - return new Iterator>() + return new Iterator() { int position = 0; + private final ByteBuffer retVal = + compression.getCompressor().allocateInBuffer(chunkBytes, closer).order(byteOrder); + private final boolean isBigEndian = byteOrder.equals(ByteOrder.BIG_ENDIAN); + private final ByteBuffer helperBuf = ByteBuffer.allocate(Ints.BYTES).order(byteOrder); @Override public boolean hasNext() @@ -218,35 +216,27 @@ public boolean hasNext() } @Override - public ResourceHolder next() + public ByteBuffer next() { - ByteBuffer retVal = ByteBuffer - .allocate(chunkBytes) - .order(byteOrder); + retVal.clear(); + int elementCount = Math.min(list.size() - position, chunkFactor); + retVal.limit(numBytes * elementCount); - if (chunkFactor > list.size() - position) { - retVal.limit((list.size() - position) * numBytes); - } else { - retVal.limit(chunkFactor * numBytes); - } - - final List ints = list.subList(position, position + retVal.remaining() / numBytes); - final ByteBuffer buf = ByteBuffer - .allocate(Ints.BYTES) - .order(byteOrder); - final boolean bigEndian = byteOrder.equals(ByteOrder.BIG_ENDIAN); - for (int value : ints) { - buf.putInt(0, value); - if (bigEndian) { - retVal.put(buf.array(), Ints.BYTES - numBytes, numBytes); - } else { - retVal.put(buf.array(), 0, numBytes); - } + for (int limit = position + elementCount; position < limit; position++) { + writeIntToRetVal(list.getInt(position)); } retVal.rewind(); - position += retVal.remaining() / numBytes; + return retVal; + } - return StupidResourceHolder.create(retVal); + private void writeIntToRetVal(int value) + { + helperBuf.putInt(0, value); + if (isBigEndian) { + retVal.put(helperBuf.array(), Ints.BYTES - numBytes, numBytes); + } else { + retVal.put(helperBuf.array(), 0, numBytes); + } } @Override @@ -257,7 +247,10 @@ public void remove() }; } }, - CompressedByteBufferObjectStrategy.getBufferForOrder(byteOrder, compression, chunkBytes) + compression, + chunkBytes, + byteOrder, + closer ), compression ); diff --git a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriter.java index 0a8e9de60b31..f00541aa9547 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriter.java @@ -20,10 +20,11 @@ package io.druid.segment.data; import com.google.common.primitives.Ints; -import io.druid.collections.ResourceHolder; -import io.druid.collections.StupidResourceHolder; +import io.druid.common.utils.ByteUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.IndexIO; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.IOException; import java.nio.ByteBuffer; @@ -37,79 +38,85 @@ public class CompressedVSizeIntsIndexedWriter extends SingleValueIndexedIntsWrit { private static final byte VERSION = CompressedVSizeIntsIndexedSupplier.VERSION; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((CompressedVSizeIntsIndexedWriter x) -> VERSION) + .writeByte(x -> ByteUtils.checkedCast(x.numBytes)) + .writeInt(x -> x.numInserted) + .writeInt(x -> x.chunkFactor) + .writeByte(x -> x.compression.getId()); + + public static CompressedVSizeIntsIndexedWriter create( + final SegmentWriteOutMedium segmentWriteOutMedium, + final String filenameBase, + final int maxValue, + final CompressionStrategy compression + ) + { + return new CompressedVSizeIntsIndexedWriter( + segmentWriteOutMedium, + filenameBase, + maxValue, + CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForValue(maxValue), + IndexIO.BYTE_ORDER, + compression + ); + } + private final int numBytes; private final int chunkFactor; - private final int chunkBytes; - private final ByteOrder byteOrder; - private final CompressedObjectStrategy.CompressionStrategy compression; - private final GenericIndexedWriter> flattener; + private final boolean isBigEndian; + private final CompressionStrategy compression; + private final GenericIndexedWriter flattener; private final ByteBuffer intBuffer; private ByteBuffer endBuffer; private int numInserted; - public CompressedVSizeIntsIndexedWriter( - final IOPeon ioPeon, + CompressedVSizeIntsIndexedWriter( + final SegmentWriteOutMedium segmentWriteOutMedium, final String filenameBase, final int maxValue, final int chunkFactor, final ByteOrder byteOrder, - final CompressedObjectStrategy.CompressionStrategy compression + final CompressionStrategy compression ) { this( + segmentWriteOutMedium, maxValue, chunkFactor, byteOrder, compression, - new GenericIndexedWriter<>( - ioPeon, + GenericIndexedWriter.ofCompressedByteBuffers( + segmentWriteOutMedium, filenameBase, - CompressedByteBufferObjectStrategy.getBufferForOrder( - byteOrder, - compression, - sizePer(maxValue, chunkFactor) - ) + compression, + sizePer(maxValue, chunkFactor) ) ); } - public CompressedVSizeIntsIndexedWriter( + CompressedVSizeIntsIndexedWriter( + final SegmentWriteOutMedium segmentWriteOutMedium, final int maxValue, final int chunkFactor, final ByteOrder byteOrder, - final CompressedObjectStrategy.CompressionStrategy compression, - final GenericIndexedWriter writer + final CompressionStrategy compression, + final GenericIndexedWriter flattener ) { this.numBytes = VSizeIndexedInts.getNumBytesForMax(maxValue); this.chunkFactor = chunkFactor; - this.chunkBytes = chunkFactor * numBytes + CompressedVSizeIntsIndexedSupplier.bufferPadding(numBytes); - this.byteOrder = byteOrder; + int chunkBytes = chunkFactor * numBytes; + this.isBigEndian = byteOrder.equals(ByteOrder.BIG_ENDIAN); this.compression = compression; - this.flattener = writer; + this.flattener = flattener; this.intBuffer = ByteBuffer.allocate(Ints.BYTES).order(byteOrder); - this.endBuffer = ByteBuffer.allocate(chunkBytes).order(byteOrder); - this.endBuffer.limit(numBytes * chunkFactor); + CompressionStrategy.Compressor compressor = compression.getCompressor(); + this.endBuffer = compressor.allocateInBuffer(chunkBytes, segmentWriteOutMedium.getCloser()).order(byteOrder); this.numInserted = 0; } - public static CompressedVSizeIntsIndexedWriter create( - final IOPeon ioPeon, - final String filenameBase, - final int maxValue, - final CompressedObjectStrategy.CompressionStrategy compression - ) - { - return new CompressedVSizeIntsIndexedWriter( - ioPeon, - filenameBase, - maxValue, - CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForValue(maxValue), - IndexIO.BYTE_ORDER, compression - ); - } - private static int sizePer(int maxValue, int chunkFactor) { return chunkFactor * VSizeIndexedInts.getNumBytesForMax(maxValue) @@ -125,14 +132,16 @@ public void open() throws IOException @Override protected void addValue(int val) throws IOException { + if (endBuffer == null) { + throw new IllegalStateException("written out already"); + } if (!endBuffer.hasRemaining()) { endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - endBuffer = ByteBuffer.allocate(chunkBytes).order(byteOrder); - endBuffer.limit(numBytes * chunkFactor); + flattener.write(endBuffer); + endBuffer.clear(); } intBuffer.putInt(0, val); - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) { + if (isBigEndian) { endBuffer.put(intBuffer.array(), Ints.BYTES - numBytes, numBytes); } else { endBuffer.put(intBuffer.array(), 0, numBytes); @@ -141,39 +150,28 @@ protected void addValue(int val) throws IOException } @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { - try { - if (numInserted > 0) { - endBuffer.limit(endBuffer.position()); - endBuffer.rewind(); - flattener.write(StupidResourceHolder.create(endBuffer)); - } - endBuffer = null; - } - finally { - flattener.close(); - } + writeEndBuffer(); + return metaSerdeHelper.size(this) + flattener.getSerializedSize(); } @Override - public long getSerializedSize() + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - return 1 + // version - 1 + // numBytes - Ints.BYTES + // numInserted - Ints.BYTES + // chunkFactor - 1 + // compression id - flattener.getSerializedSize(); + writeEndBuffer(); + metaSerdeHelper.writeTo(channel, this); + flattener.writeTo(channel, smoosher); } - @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + private void writeEndBuffer() throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{VERSION, (byte) numBytes})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(numInserted))); - channel.write(ByteBuffer.wrap(Ints.toByteArray(chunkFactor))); - channel.write(ByteBuffer.wrap(new byte[]{compression.getId()})); - flattener.writeToChannel(channel, smoosher); + if (endBuffer != null) { + endBuffer.flip(); + if (endBuffer.remaining() > 0) { + flattener.write(endBuffer); + } + endBuffer = null; + } } } diff --git a/processing/src/main/java/io/druid/segment/data/CompressionFactory.java b/processing/src/main/java/io/druid/segment/data/CompressionFactory.java index 2eae56128a99..1e6a2ea8a007 100644 --- a/processing/src/main/java/io/druid/segment/data/CompressionFactory.java +++ b/processing/src/main/java/io/druid/segment/data/CompressionFactory.java @@ -25,16 +25,18 @@ import com.google.common.collect.Maps; import io.druid.java.util.common.IAE; import io.druid.java.util.common.StringUtils; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.segment.writeout.WriteOutBytes; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.serde.MetaSerdeHelper; import java.io.IOException; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Map; +import java.util.function.Function; /** - * Compression of metrics is done by using a combination of {@link CompressedObjectStrategy.CompressionStrategy} + * Compression of metrics is done by using a combination of {@link CompressionStrategy} * and Encoding(such as {@link LongEncodingStrategy} for type Long). CompressionStrategy is unaware of the data type * and is based on byte operations. It must compress and decompress in block of bytes. Encoding refers to compression * method relies on data format, so a different set of Encodings exist for each data type. @@ -215,7 +217,7 @@ public static LongEncodingFormat forId(byte id) /** * This writer output encoded values to the given ByteBuffer or OutputStream. {@link #setBuffer(ByteBuffer)} or - * {@link #setOutputStream(OutputStream)} must be called before any value is written, and {@link #flush()} must + * {@link #setOutputStream(WriteOutBytes)} must be called before any value is written, and {@link #flush()} must * be called before calling setBuffer or setOutputStream again to set another output. */ public interface LongEncodingWriter @@ -226,7 +228,7 @@ public interface LongEncodingWriter */ void setBuffer(ByteBuffer buffer); - void setOutputStream(OutputStream output); + void setOutputStream(WriteOutBytes output); void write(long value) throws IOException; @@ -239,7 +241,9 @@ public interface LongEncodingWriter * Output the header values of the associating encoding format to the given outputStream. The header also include * bytes for compression strategy and encoding format(optional) as described above in Compression Storage Format. */ - void putMeta(OutputStream metaOut, CompressedObjectStrategy.CompressionStrategy strategy) throws IOException; + void putMeta(ByteBuffer metaOut, CompressionStrategy strategy) throws IOException; + + int metaSize(); /** * Get the number of values that can be encoded into each block for the given block size in bytes @@ -252,14 +256,33 @@ public interface LongEncodingWriter int getNumBytes(int values); } + static MetaSerdeHelper.FieldWriter longEncodingWriter( + Function getWriter, + Function getCompressionStrategy + ) + { + return new MetaSerdeHelper.FieldWriter() + { + @Override + public void writeTo(ByteBuffer buffer, T x) throws IOException + { + getWriter.apply(x).putMeta(buffer, getCompressionStrategy.apply(x)); + } + + @Override + public int size(T x) + { + return getWriter.apply(x).metaSize(); + } + }; + } + public interface LongEncodingReader { void setBuffer(ByteBuffer buffer); long read(int index); - int getNumBytes(int values); - LongEncodingReader duplicate(); } @@ -269,11 +292,10 @@ public static Supplier getLongSupplier( ByteBuffer fromBuffer, ByteOrder order, LongEncodingFormat encodingFormat, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper fileMapper + CompressionStrategy strategy ) { - if (strategy == CompressedObjectStrategy.CompressionStrategy.NONE) { + if (strategy == CompressionStrategy.NONE) { return new EntireLayoutIndexedLongSupplier(totalSize, encodingFormat.getReader(fromBuffer, order)); } else { return new BlockLayoutIndexedLongSupplier( @@ -282,28 +304,31 @@ public static Supplier getLongSupplier( fromBuffer, order, encodingFormat.getReader(fromBuffer, order), - strategy, - fileMapper + strategy ); } } public static LongSupplierSerializer getLongSerializer( - IOPeon ioPeon, String filenameBase, ByteOrder order, + SegmentWriteOutMedium segmentWriteOutMedium, + String filenameBase, + ByteOrder order, LongEncodingStrategy encodingStrategy, - CompressedObjectStrategy.CompressionStrategy compressionStrategy + CompressionStrategy compressionStrategy ) { if (encodingStrategy == LongEncodingStrategy.AUTO) { - return new IntermediateLongSupplierSerializer(ioPeon, filenameBase, order, compressionStrategy); + return new IntermediateLongSupplierSerializer(segmentWriteOutMedium, filenameBase, order, compressionStrategy); } else if (encodingStrategy == LongEncodingStrategy.LONGS) { - if (compressionStrategy == CompressedObjectStrategy.CompressionStrategy.NONE) { - return new EntireLayoutLongSupplierSerializer( - ioPeon, filenameBase, new LongsLongEncodingWriter(order) - ); + if (compressionStrategy == CompressionStrategy.NONE) { + return new EntireLayoutLongSupplierSerializer(segmentWriteOutMedium, new LongsLongEncodingWriter(order)); } else { return new BlockLayoutLongSupplierSerializer( - ioPeon, filenameBase, order, new LongsLongEncodingWriter(order), compressionStrategy + segmentWriteOutMedium, + filenameBase, + order, + new LongsLongEncodingWriter(order), + compressionStrategy ); } } else { @@ -318,30 +343,27 @@ public static Supplier getFloatSupplier( int sizePer, ByteBuffer fromBuffer, ByteOrder order, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper fileMapper + CompressionStrategy strategy ) { - if (strategy == CompressedObjectStrategy.CompressionStrategy.NONE) { + if (strategy == CompressionStrategy.NONE) { return new EntireLayoutIndexedFloatSupplier(totalSize, fromBuffer, order); } else { - return new BlockLayoutIndexedFloatSupplier(totalSize, sizePer, fromBuffer, order, strategy, fileMapper); + return new BlockLayoutIndexedFloatSupplier(totalSize, sizePer, fromBuffer, order, strategy); } } public static FloatSupplierSerializer getFloatSerializer( - IOPeon ioPeon, String filenameBase, ByteOrder order, - CompressedObjectStrategy.CompressionStrategy compressionStrategy + SegmentWriteOutMedium segmentWriteOutMedium, + String filenameBase, + ByteOrder order, + CompressionStrategy compressionStrategy ) { - if (compressionStrategy == CompressedObjectStrategy.CompressionStrategy.NONE) { - return new EntireLayoutFloatSupplierSerializer( - ioPeon, filenameBase, order - ); + if (compressionStrategy == CompressionStrategy.NONE) { + return new EntireLayoutFloatSupplierSerializer(segmentWriteOutMedium, order); } else { - return new BlockLayoutFloatSupplierSerializer( - ioPeon, filenameBase, order, compressionStrategy - ); + return new BlockLayoutFloatSupplierSerializer(segmentWriteOutMedium, filenameBase, order, compressionStrategy); } } @@ -350,29 +372,29 @@ public static Supplier getDoubleSupplier( int sizePer, ByteBuffer fromBuffer, ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy strategy, - SmooshedFileMapper fileMapper + CompressionStrategy strategy ) { switch (strategy) { case NONE: return new EntireLayoutIndexedDoubleSupplier(totalSize, fromBuffer, byteOrder); default: - return new BlockLayoutIndexedDoubleSupplier(totalSize, sizePer, fromBuffer, byteOrder, strategy, fileMapper); + return new BlockLayoutIndexedDoubleSupplier(totalSize, sizePer, fromBuffer, byteOrder, strategy); } } + public static DoubleSupplierSerializer getDoubleSerializer( - IOPeon ioPeon, + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, - CompressedObjectStrategy.CompressionStrategy compression + CompressionStrategy compression ) { - if (compression == CompressedObjectStrategy.CompressionStrategy.NONE) { - return new EntireLayoutDoubleSupplierSerializer(ioPeon, filenameBase, byteOrder); + if (compression == CompressionStrategy.NONE) { + return new EntireLayoutDoubleSupplierSerializer(segmentWriteOutMedium, byteOrder); } else { - return new BlockLayoutDoubleSupplierSerializer(ioPeon, filenameBase, byteOrder, compression); + return new BlockLayoutDoubleSupplierSerializer(segmentWriteOutMedium, filenameBase, byteOrder, compression); } } } diff --git a/processing/src/main/java/io/druid/segment/data/CompressionStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressionStrategy.java new file mode 100644 index 000000000000..c4a6fd742be5 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/data/CompressionStrategy.java @@ -0,0 +1,338 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.data; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +import com.google.common.collect.Maps; +import com.ning.compress.BufferRecycler; +import com.ning.compress.lzf.LZFDecoder; +import com.ning.compress.lzf.LZFEncoder; +import io.druid.collections.ResourceHolder; +import io.druid.java.util.common.ByteBufferUtils; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.io.Closer; +import io.druid.segment.CompressedPools; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4SafeDecompressor; +import org.apache.commons.lang.ArrayUtils; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; + +/** + * Compression strategy is used to compress block of bytes without knowledge of what data the bytes represents. + * + * When adding compression strategy, do not use id in the range [0x7C, 0xFD] (greater than 123 or less than -2), since + * a flag mechanism is used in CompressionFactory that involves subtracting the value 126 from the compression id + * (see {@link CompressionFactory#FLAG_BOUND}) + */ +public enum CompressionStrategy +{ + LZF((byte) 0x0) { + @Override + public Decompressor getDecompressor() + { + return LZFDecompressor.defaultDecompressor; + } + + @Override + public Compressor getCompressor() + { + return LZFCompressor.defaultCompressor; + } + }, + + LZ4((byte) 0x1) { + @Override + public Decompressor getDecompressor() + { + return LZ4Decompressor.defaultDecompressor; + } + + @Override + public Compressor getCompressor() + { + return LZ4Compressor.defaultCompressor; + } + }, + UNCOMPRESSED((byte) 0xFF) { + @Override + public Decompressor getDecompressor() + { + return UncompressedDecompressor.defaultDecompressor; + } + + @Override + public Compressor getCompressor() + { + return UncompressedCompressor.defaultCompressor; + } + }, + /* + This value indicate no compression strategy should be used, and compression should not be block based + Currently only IndexedLong support non block based compression, and other types treat this as UNCOMPRESSED + */ + NONE((byte) 0xFE) { + @Override + public Decompressor getDecompressor() + { + throw new UnsupportedOperationException("NONE compression strategy shouldn't use any decompressor"); + } + + @Override + public Compressor getCompressor() + { + throw new UnsupportedOperationException("NONE compression strategy shouldn't use any compressor"); + } + }; + public static final CompressionStrategy DEFAULT_COMPRESSION_STRATEGY = LZ4; + + final byte id; + + CompressionStrategy(byte id) + { + this.id = id; + } + + public byte getId() + { + return id; + } + + public abstract Compressor getCompressor(); + + public abstract Decompressor getDecompressor(); + + @JsonValue + @Override + public String toString() + { + return StringUtils.toLowerCase(this.name()); + } + + @JsonCreator + public static CompressionStrategy fromString(String name) + { + return valueOf(StringUtils.toUpperCase(name)); + } + + static final Map idMap = Maps.newHashMap(); + + static { + for (CompressionStrategy strategy : CompressionStrategy.values()) { + idMap.put(strategy.getId(), strategy); + } + } + + public static CompressionStrategy forId(byte id) + { + return idMap.get(id); + } + + // TODO remove this method and change all its callers to use all CompressionStrategy values when NONE type is supported by all types + public static CompressionStrategy[] noNoneValues() + { + return (CompressionStrategy[]) ArrayUtils.removeElement(CompressionStrategy.values(), NONE); + } + + public interface Decompressor + { + /** + * Implementations of this method are expected to call out.flip() after writing to the output buffer + */ + void decompress(ByteBuffer in, int numBytes, ByteBuffer out); + } + + public static abstract class Compressor + { + /** + * Allocates a buffer that should be passed to {@link #compress} method as input buffer. Different Compressors + * require (or work more efficiently with) different kinds of buffers. + * + * If the allocated buffer is a direct buffer, it should be registered to be freed with the given Closer. + */ + ByteBuffer allocateInBuffer(int inputSize, Closer closer) + { + return ByteBuffer.allocate(inputSize); + } + + /** + * Allocates a buffer that should be passed to {@link #compress} method as output buffer. Different Compressors + * require (or work more efficiently with) different kinds of buffers. + * + * Allocates a buffer that is always enough to compress a byte sequence of the given size. + * + * If the allocated buffer is a direct buffer, it should be registered to be freed with the given Closer. + */ + abstract ByteBuffer allocateOutBuffer(int inputSize, Closer closer); + + /** + * Returns a ByteBuffer with compressed contents of in between it's position and limit. It may be the provided out + * ByteBuffer, or the in ByteBuffer, depending on the implementation. {@code out}'s position and limit + * are not respected and could be discarded. + * + *

Contents of {@code in} between it's position and limit are compressed. It's contents, position and limit + * shouldn't be changed in compress() method. + */ + public abstract ByteBuffer compress(ByteBuffer in, ByteBuffer out); + } + + public static class UncompressedCompressor extends Compressor + { + private static final UncompressedCompressor defaultCompressor = new UncompressedCompressor(); + + @Override + ByteBuffer allocateOutBuffer(int inputSize, Closer closer) + { + return ByteBuffer.allocate(inputSize); + } + + @Override + public ByteBuffer compress(ByteBuffer in, ByteBuffer out) + { + return in; + } + } + + public static class UncompressedDecompressor implements Decompressor + { + private static final UncompressedDecompressor defaultDecompressor = new UncompressedDecompressor(); + + @Override + public void decompress(ByteBuffer in, int numBytes, ByteBuffer out) + { + final ByteBuffer copyBuffer = in.duplicate(); + copyBuffer.limit(copyBuffer.position() + numBytes); + out.put(copyBuffer).flip(); + in.position(in.position() + numBytes); + } + + } + + public static class LZFDecompressor implements Decompressor + { + private static final LZFDecompressor defaultDecompressor = new LZFDecompressor(); + + @Override + public void decompress(ByteBuffer in, int numBytes, ByteBuffer out) + { + final byte[] bytes = new byte[numBytes]; + in.get(bytes); + + try (final ResourceHolder outputBytesHolder = CompressedPools.getOutputBytes()) { + final byte[] outputBytes = outputBytesHolder.get(); + final int numDecompressedBytes = LZFDecoder.decode(bytes, outputBytes); + out.put(outputBytes, 0, numDecompressedBytes); + out.flip(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + } + + public static class LZFCompressor extends Compressor + { + private static final LZFCompressor defaultCompressor = new LZFCompressor(); + + @Override + public ByteBuffer allocateOutBuffer(int inputSize, Closer closer) + { + return ByteBuffer.allocate(LZFEncoder.estimateMaxWorkspaceSize(inputSize)); + } + + @Override + public ByteBuffer compress(ByteBuffer in, ByteBuffer out) + { + try (final ResourceHolder bufferRecycler = CompressedPools.getBufferRecycler()) { + int encodedLen = LZFEncoder.appendEncoded( + in.array(), + in.arrayOffset() + in.position(), + in.remaining(), + out.array(), + out.arrayOffset(), + bufferRecycler.get() + ); + out.clear(); + out.limit(encodedLen); + return out; + } + } + } + + public static class LZ4Decompressor implements Decompressor + { + private static final LZ4SafeDecompressor lz4Safe = LZ4Factory.fastestInstance().safeDecompressor(); + private static final LZ4Decompressor defaultDecompressor = new LZ4Decompressor(); + + @Override + public void decompress(ByteBuffer in, int numBytes, ByteBuffer out) + { + // Since decompressed size is NOT known, must use lz4Safe + // lz4Safe.decompress does not modify buffer positions + final int numDecompressedBytes = lz4Safe.decompress( + in, + in.position(), + numBytes, + out, + out.position(), + out.remaining() + ); + out.limit(out.position() + numDecompressedBytes); + } + + } + + public static class LZ4Compressor extends Compressor + { + private static final LZ4Compressor defaultCompressor = new LZ4Compressor(); + private static final net.jpountz.lz4.LZ4Compressor lz4High = LZ4Factory.fastestInstance().highCompressor(); + + @Override + ByteBuffer allocateInBuffer(int inputSize, Closer closer) + { + ByteBuffer inBuffer = ByteBuffer.allocateDirect(inputSize); + closer.register(() -> ByteBufferUtils.free(inBuffer)); + return inBuffer; + } + + @Override + ByteBuffer allocateOutBuffer(int inputSize, Closer closer) + { + ByteBuffer outBuffer = ByteBuffer.allocateDirect(lz4High.maxCompressedLength(inputSize)); + closer.register(() -> ByteBufferUtils.free(outBuffer)); + return outBuffer; + } + + @Override + public ByteBuffer compress(ByteBuffer in, ByteBuffer out) + { + out.clear(); + int position = in.position(); + lz4High.compress(in, out); + in.position(position); + out.flip(); + return out; + } + } +} diff --git a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java index fe9b2eceef99..044892d9f24a 100644 --- a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java @@ -46,8 +46,7 @@ public BitmapFactory getBitmapFactory() return bitmapFactory; } - private static class ImmutableConciseSetObjectStrategy - implements ObjectStrategy + private static class ImmutableConciseSetObjectStrategy implements ObjectStrategy { @Override public Class getClazz() diff --git a/processing/src/main/java/io/druid/segment/data/DecompressingByteBufferObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/DecompressingByteBufferObjectStrategy.java new file mode 100644 index 000000000000..ff7a1426d17c --- /dev/null +++ b/processing/src/main/java/io/druid/segment/data/DecompressingByteBufferObjectStrategy.java @@ -0,0 +1,84 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.data; + +import io.druid.collections.ResourceHolder; +import io.druid.segment.CompressedPools; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class DecompressingByteBufferObjectStrategy implements ObjectStrategy> +{ + private final ByteOrder order; + private final CompressionStrategy.Decompressor decompressor; + + DecompressingByteBufferObjectStrategy(ByteOrder order, CompressionStrategy compression) + { + this.order = order; + this.decompressor = compression.getDecompressor(); + } + + @Override + @SuppressWarnings("unchecked") + public Class> getClazz() + { + return (Class) ResourceHolder.class; + } + + @Override + public ResourceHolder fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ResourceHolder bufHolder = CompressedPools.getByteBuf(order); + final ByteBuffer buf = bufHolder.get(); + buf.clear(); + + decompressor.decompress(buffer, numBytes, buf); + // Needed, because if e. g. if this compressed buffer contains 3-byte integers, it should be possible to getInt() + // from the buffer, including padding. See CompressedVSizeIntsIndexedSupplier.bufferPadding(). + buf.limit(buf.capacity()); + return new ResourceHolder() + { + @Override + public ByteBuffer get() + { + return buf; + } + + @Override + public void close() + { + bufHolder.close(); + } + }; + } + + @Override + public int compare(ResourceHolder o1, ResourceHolder o2) + { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] toBytes(ResourceHolder holder) + { + throw new UnsupportedOperationException(); + } +} diff --git a/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingReader.java b/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingReader.java index 5c21615c9141..a04b1b9b62fb 100644 --- a/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingReader.java +++ b/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingReader.java @@ -65,12 +65,6 @@ public long read(int index) return base + deserializer.get(index); } - @Override - public int getNumBytes(int values) - { - return VSizeLongSerde.getSerializedSize(bitsPerValue, values); - } - @Override public CompressionFactory.LongEncodingReader duplicate() { diff --git a/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingWriter.java b/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingWriter.java index e73bd72f4722..7cefc0589fa0 100644 --- a/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingWriter.java +++ b/processing/src/main/java/io/druid/segment/data/DeltaLongEncodingWriter.java @@ -21,9 +21,9 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; +import io.druid.segment.writeout.WriteOutBytes; import java.io.IOException; -import java.io.OutputStream; import java.nio.ByteBuffer; public class DeltaLongEncodingWriter implements CompressionFactory.LongEncodingWriter @@ -46,7 +46,7 @@ public void setBuffer(ByteBuffer buffer) } @Override - public void setOutputStream(OutputStream output) + public void setOutputStream(WriteOutBytes output) { serializer = VSizeLongSerde.getSerializer(bitsPerValue, output); } @@ -58,13 +58,19 @@ public void write(long value) throws IOException } @Override - public void putMeta(OutputStream metaOut, CompressedObjectStrategy.CompressionStrategy strategy) throws IOException + public void putMeta(ByteBuffer metaOut, CompressionStrategy strategy) throws IOException { - metaOut.write(CompressionFactory.setEncodingFlag(strategy.getId())); - metaOut.write(CompressionFactory.LongEncodingFormat.DELTA.getId()); - metaOut.write(CompressionFactory.DELTA_ENCODING_VERSION); - metaOut.write(Longs.toByteArray(base)); - metaOut.write(Ints.toByteArray(bitsPerValue)); + metaOut.put(CompressionFactory.setEncodingFlag(strategy.getId())); + metaOut.put(CompressionFactory.LongEncodingFormat.DELTA.getId()); + metaOut.put(CompressionFactory.DELTA_ENCODING_VERSION); + metaOut.putLong(base); + metaOut.putInt(bitsPerValue); + } + + @Override + public int metaSize() + { + return 1 + 1 + 1 + Longs.BYTES + Ints.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/segment/data/DoubleSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/DoubleSupplierSerializer.java index 8589ca90ad0f..fe804330ed66 100644 --- a/processing/src/main/java/io/druid/segment/data/DoubleSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/DoubleSupplierSerializer.java @@ -19,17 +19,12 @@ package io.druid.segment.data; +import io.druid.segment.serde.Serializer; -import io.druid.java.util.common.io.smoosh.FileSmoosher; - -import java.io.Closeable; import java.io.IOException; -import java.nio.channels.WritableByteChannel; -public interface DoubleSupplierSerializer extends Closeable +public interface DoubleSupplierSerializer extends Serializer { void open() throws IOException; void add(double value) throws IOException; - long getSerializedSize(); - void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/data/EntireLayoutDoubleSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/EntireLayoutDoubleSupplierSerializer.java index 198fffd98354..563eff608b66 100644 --- a/processing/src/main/java/io/druid/segment/data/EntireLayoutDoubleSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/EntireLayoutDoubleSupplierSerializer.java @@ -19,45 +19,42 @@ package io.druid.segment.data; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import com.google.common.primitives.Doubles; -import com.google.common.primitives.Ints; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.writeout.WriteOutBytes; import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; public class EntireLayoutDoubleSupplierSerializer implements DoubleSupplierSerializer { - private final IOPeon ioPeon; - private final String valueFile; - private final String metaFile; - private CountingOutputStream valuesOut; - private long metaCount = 0; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((EntireLayoutDoubleSupplierSerializer x) -> CompressedDoublesIndexedSupplier.VERSION) + .writeInt(x -> x.numInserted) + .writeInt(x -> 0) + .writeByte(x -> CompressionStrategy.NONE.getId()); + private final SegmentWriteOutMedium segmentWriteOutMedium; private final ByteBuffer orderBuffer; + private WriteOutBytes valuesOut; private int numInserted = 0; - public EntireLayoutDoubleSupplierSerializer(IOPeon ioPeon, String filenameBase, ByteOrder order) + public EntireLayoutDoubleSupplierSerializer(SegmentWriteOutMedium segmentWriteOutMedium, ByteOrder order) { - this.ioPeon = ioPeon; - this.valueFile = filenameBase + ".value"; - this.metaFile = filenameBase + ".format"; - this.orderBuffer = ByteBuffer.allocate(Doubles.BYTES); + this.segmentWriteOutMedium = segmentWriteOutMedium; + this.orderBuffer = ByteBuffer.allocate(Double.BYTES); orderBuffer.order(order); } @Override public void open() throws IOException { - valuesOut = new CountingOutputStream(ioPeon.makeOutputStream(valueFile)); + valuesOut = segmentWriteOutMedium.makeWriteOutBytes(); } @Override @@ -67,38 +64,18 @@ public void add(double value) throws IOException orderBuffer.putDouble(value); valuesOut.write(orderBuffer.array()); ++numInserted; - - } - - @Override - public long getSerializedSize() - { - return metaCount + valuesOut.getCount(); } @Override - public void writeToChannel( - WritableByteChannel channel, FileSmoosher smoosher - ) throws IOException + public long getSerializedSize() throws IOException { - try (InputStream meta = ioPeon.makeInputStream(metaFile); - InputStream value = ioPeon.makeInputStream(valueFile)) { - ByteStreams.copy(Channels.newChannel(meta), channel); - ByteStreams.copy(Channels.newChannel(value), channel); - } + return metaSerdeHelper.size(this) + valuesOut.size(); } @Override - public void close() throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - valuesOut.close(); - try (CountingOutputStream metaOut = new CountingOutputStream(ioPeon.makeOutputStream(metaFile))) { - metaOut.write(CompressedDoublesIndexedSupplier.version); - metaOut.write(Ints.toByteArray(numInserted)); - metaOut.write(Ints.toByteArray(0)); - metaOut.write(CompressedObjectStrategy.CompressionStrategy.NONE.getId()); - metaOut.close(); - metaCount = metaOut.getCount(); - } + metaSerdeHelper.writeTo(channel, this); + valuesOut.writeTo(channel); } } diff --git a/processing/src/main/java/io/druid/segment/data/EntireLayoutFloatSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/EntireLayoutFloatSupplierSerializer.java index e49c32f9f691..8595d0121d7a 100644 --- a/processing/src/main/java/io/druid/segment/data/EntireLayoutFloatSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/EntireLayoutFloatSupplierSerializer.java @@ -19,49 +19,39 @@ package io.druid.segment.data; -import com.google.common.io.ByteSink; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import com.google.common.primitives.Floats; -import com.google.common.primitives.Ints; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.WriteOutBytes; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.serde.MetaSerdeHelper; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; public class EntireLayoutFloatSupplierSerializer implements FloatSupplierSerializer { - private final IOPeon ioPeon; - private final String valueFile; - private final String metaFile; - private CountingOutputStream valuesOut; - private long metaCount = 0; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((EntireLayoutFloatSupplierSerializer x) -> CompressedFloatsIndexedSupplier.VERSION) + .writeInt(x -> x.numInserted) + .writeInt(x -> 0) + .writeByte(x -> CompressionStrategy.NONE.getId()); - private final ByteBuffer orderBuffer; + private final boolean isLittleEndian; + private final SegmentWriteOutMedium segmentWriteOutMedium; + private WriteOutBytes valuesOut; private int numInserted = 0; - public EntireLayoutFloatSupplierSerializer( - IOPeon ioPeon, String filenameBase, ByteOrder order - ) + EntireLayoutFloatSupplierSerializer(SegmentWriteOutMedium segmentWriteOutMedium, ByteOrder order) { - this.ioPeon = ioPeon; - this.valueFile = filenameBase + ".value"; - this.metaFile = filenameBase + ".format"; - - orderBuffer = ByteBuffer.allocate(Floats.BYTES); - orderBuffer.order(order); + this.segmentWriteOutMedium = segmentWriteOutMedium; + isLittleEndian = order.equals(ByteOrder.LITTLE_ENDIAN); } @Override public void open() throws IOException { - valuesOut = new CountingOutputStream(ioPeon.makeOutputStream(valueFile)); + valuesOut = segmentWriteOutMedium.makeWriteOutBytes(); } @Override @@ -73,51 +63,25 @@ public int size() @Override public void add(float value) throws IOException { - orderBuffer.rewind(); - orderBuffer.putFloat(value); - valuesOut.write(orderBuffer.array()); - ++numInserted; - } - - @Override - public void closeAndConsolidate(ByteSink consolidatedOut) throws IOException - { - close(); - try (OutputStream out = consolidatedOut.openStream(); - InputStream meta = ioPeon.makeInputStream(metaFile); - InputStream value = ioPeon.makeInputStream(valueFile)) { - ByteStreams.copy(meta, out); - ByteStreams.copy(value, out); - } - } - - @Override - public void close() throws IOException - { - valuesOut.close(); - try (CountingOutputStream metaOut = new CountingOutputStream(ioPeon.makeOutputStream(metaFile))) { - metaOut.write(CompressedFloatsIndexedSupplier.version); - metaOut.write(Ints.toByteArray(numInserted)); - metaOut.write(Ints.toByteArray(0)); - metaOut.write(CompressedObjectStrategy.CompressionStrategy.NONE.getId()); - metaOut.close(); - metaCount = metaOut.getCount(); + int valueBits = Float.floatToRawIntBits(value); + // WriteOutBytes are always big-endian, so need to reverse bytes + if (isLittleEndian) { + valueBits = Integer.reverseBytes(valueBits); } + valuesOut.writeInt(valueBits); + ++numInserted; } @Override - public long getSerializedSize() + public long getSerializedSize() throws IOException { - return metaCount + valuesOut.getCount(); + return metaSerdeHelper.size(this) + valuesOut.size(); } @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - try (InputStream meta = ioPeon.makeInputStream(metaFile); - InputStream value = ioPeon.makeInputStream(valueFile)) { - ByteStreams.copy(Channels.newChannel(meta), channel); - ByteStreams.copy(Channels.newChannel(value), channel); - } + metaSerdeHelper.writeTo(channel, this); + valuesOut.writeTo(channel); } } diff --git a/processing/src/main/java/io/druid/segment/data/EntireLayoutLongSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/EntireLayoutLongSupplierSerializer.java index 2804fed9bf9e..847fbcd89ad0 100644 --- a/processing/src/main/java/io/druid/segment/data/EntireLayoutLongSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/EntireLayoutLongSupplierSerializer.java @@ -19,46 +19,41 @@ package io.druid.segment.data; -import com.google.common.io.ByteSink; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import com.google.common.primitives.Ints; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.writeout.WriteOutBytes; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; public class EntireLayoutLongSupplierSerializer implements LongSupplierSerializer { + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((EntireLayoutLongSupplierSerializer x) -> CompressedLongsIndexedSupplier.VERSION) + .writeInt(x -> x.numInserted) + .writeInt(x -> 0) + .writeSomething(CompressionFactory.longEncodingWriter(x -> x.writer, x -> CompressionStrategy.NONE)); - private final IOPeon ioPeon; - private final String valueFile; - private final String metaFile; - private CountingOutputStream valuesOut; private final CompressionFactory.LongEncodingWriter writer; - private long metaCount = 0; + private final SegmentWriteOutMedium segmentWriteOutMedium; + private WriteOutBytes valuesOut; private int numInserted = 0; - public EntireLayoutLongSupplierSerializer( - IOPeon ioPeon, - String filenameBase, + EntireLayoutLongSupplierSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, CompressionFactory.LongEncodingWriter writer ) { - this.ioPeon = ioPeon; - this.valueFile = filenameBase + ".value"; - this.metaFile = filenameBase + ".format"; + this.segmentWriteOutMedium = segmentWriteOutMedium; this.writer = writer; } @Override public void open() throws IOException { - valuesOut = new CountingOutputStream(ioPeon.makeOutputStream(valueFile)); + valuesOut = segmentWriteOutMedium.makeWriteOutBytes(); writer.setOutputStream(valuesOut); } @@ -76,45 +71,17 @@ public void add(long value) throws IOException } @Override - public void closeAndConsolidate(ByteSink consolidatedOut) throws IOException - { - close(); - try (OutputStream out = consolidatedOut.openStream(); - InputStream meta = ioPeon.makeInputStream(metaFile); - InputStream value = ioPeon.makeInputStream(valueFile)) { - ByteStreams.copy(meta, out); - ByteStreams.copy(value, out); - } - } - - @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { writer.flush(); - valuesOut.close(); - try (CountingOutputStream metaOut = new CountingOutputStream(ioPeon.makeOutputStream(metaFile))) { - metaOut.write(CompressedLongsIndexedSupplier.version); - metaOut.write(Ints.toByteArray(numInserted)); - metaOut.write(Ints.toByteArray(0)); - writer.putMeta(metaOut, CompressedObjectStrategy.CompressionStrategy.NONE); - metaOut.close(); - metaCount = metaOut.getCount(); - } + return metaSerdeHelper.size(this) + valuesOut.size(); } @Override - public long getSerializedSize() + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - return metaCount + valuesOut.getCount(); - } - - @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException - { - try (InputStream meta = ioPeon.makeInputStream(metaFile); - InputStream value = ioPeon.makeInputStream(valueFile)) { - ByteStreams.copy(Channels.newChannel(meta), channel); - ByteStreams.copy(Channels.newChannel(value), channel); - } + writer.flush(); + metaSerdeHelper.writeTo(channel, this); + valuesOut.writeTo(channel); } } diff --git a/processing/src/main/java/io/druid/segment/data/FloatSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/FloatSupplierSerializer.java index d6cf71157b62..993ab2c43ce2 100644 --- a/processing/src/main/java/io/druid/segment/data/FloatSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/FloatSupplierSerializer.java @@ -19,19 +19,13 @@ package io.druid.segment.data; -import com.google.common.io.ByteSink; -import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.Serializer; -import java.io.Closeable; import java.io.IOException; -import java.nio.channels.WritableByteChannel; -public interface FloatSupplierSerializer extends Closeable +public interface FloatSupplierSerializer extends Serializer { void open() throws IOException; int size(); void add(float value) throws IOException; - void closeAndConsolidate(ByteSink consolidatedOut) throws IOException; - long getSerializedSize(); - void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index f6768227b851..2f9b74faf0a9 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -19,17 +19,22 @@ package io.druid.segment.data; -import com.google.common.io.CountingOutputStream; import com.google.common.primitives.Ints; +import io.druid.collections.ResourceHolder; import io.druid.common.utils.SerializerUtils; -import io.druid.io.ZeroCopyByteArrayOutputStream; +import io.druid.io.Channels; import io.druid.java.util.common.IAE; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.java.util.common.guava.Comparators; +import io.druid.java.util.common.io.Closer; +import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.NullHandlingHelper; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.serde.Serializer; +import io.druid.segment.writeout.HeapByteBufferWriteOutBytes; import javax.annotation.Nullable; import java.io.Closeable; @@ -69,12 +74,19 @@ * value files are identified as: StringUtils.format("%s_value_%d", columnName, fileNumber) * number of value files == numElements/numberOfElementsPerValueFile */ -public class GenericIndexed implements Indexed +public class GenericIndexed implements Indexed, Serializer { static final byte VERSION_ONE = 0x1; static final byte VERSION_TWO = 0x2; static final byte REVERSE_LOOKUP_ALLOWED = 0x1; static final byte REVERSE_LOOKUP_DISALLOWED = 0x0; + + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((GenericIndexed x) -> VERSION_ONE) + .writeByte(x -> x.allowReverseLookup ? REVERSE_LOOKUP_ALLOWED : REVERSE_LOOKUP_DISALLOWED) + .writeInt(x -> Ints.checkedCast(x.theBuffer.remaining() + (long) Integer.BYTES)) + .writeInt(x -> x.size); + private static final SerializerUtils SERIALIZER_UTILS = new SerializerUtils(); public static final ObjectStrategy STRING_STRATEGY = new CacheableObjectStrategy() @@ -99,7 +111,7 @@ public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) @Override public byte[] toBytes(String val) { - return StringUtils.toUtf8Nullable(val); + return StringUtils.toUtf8Nullable(NullHandlingHelper.nullToEmptyIfNeeded(val)); } @Override @@ -142,9 +154,25 @@ public static GenericIndexed fromArray(T[] objects, ObjectStrategy str return fromIterable(Arrays.asList(objects), strategy); } + static GenericIndexed> ofCompressedByteBuffers( + Iterable buffers, + CompressionStrategy compression, + int bufferSize, + ByteOrder order, + Closer closer + ) + { + return fromIterableVersionOne( + buffers, + GenericIndexedWriter.compressedByteBuffersWriteObjectStrategy(compression, bufferSize, closer), + false, + new DecompressingByteBufferObjectStrategy(order, compression) + ); + } + public static GenericIndexed fromIterable(Iterable objectsIterable, ObjectStrategy strategy) { - return fromIterableVersionOne(objectsIterable, strategy); + return fromIterableVersionOne(objectsIterable, strategy, true, strategy); } static int getNumberOfFilesRequired(int bagSize, long numWritten) @@ -310,6 +338,7 @@ public Iterator iterator() return IndexedIterable.create(this).iterator(); } + @Override public long getSerializedSize() { if (!versionOne) { @@ -318,10 +347,11 @@ public long getSerializedSize() return getSerializedSizeVersionOne(); } - public void writeToChannel(WritableByteChannel channel) throws IOException + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { if (versionOne) { - writeToChannelVersionOne(channel); + writeToVersionOne(channel); } else { throw new UnsupportedOperationException( "GenericIndexed serialization for V2 is unsupported. Use GenericIndexedWriter instead."); @@ -436,22 +466,24 @@ private static GenericIndexed createGenericIndexedVersionOne(ByteBuffer b ); } - private static GenericIndexed fromIterableVersionOne(Iterable objectsIterable, ObjectStrategy strategy) + private static GenericIndexed fromIterableVersionOne( + Iterable objectsIterable, + ObjectStrategy strategy, + boolean allowReverseLookup, + ObjectStrategy resultObjectStrategy + ) { Iterator objects = objectsIterable.iterator(); if (!objects.hasNext()) { final ByteBuffer buffer = ByteBuffer.allocate(Ints.BYTES).putInt(0); buffer.flip(); - return new GenericIndexed<>(buffer, strategy, true); + return new GenericIndexed<>(buffer, resultObjectStrategy, allowReverseLookup); } - boolean allowReverseLookup = true; int count = 0; - ZeroCopyByteArrayOutputStream headerBytes = new ZeroCopyByteArrayOutputStream(); - ZeroCopyByteArrayOutputStream valueBytes = new ZeroCopyByteArrayOutputStream(); - CountingOutputStream valueCountingBytes = new CountingOutputStream(valueBytes); - ByteBuffer helperBuffer = ByteBuffer.allocate(Ints.BYTES); + HeapByteBufferWriteOutBytes headerOut = new HeapByteBufferWriteOutBytes(); + HeapByteBufferWriteOutBytes valuesOut = new HeapByteBufferWriteOutBytes(); try { T prevVal = null; do { @@ -461,20 +493,9 @@ private static GenericIndexed fromIterableVersionOne(Iterable objectsI allowReverseLookup = false; } - final byte[] bytes = strategy.toBytes(next); - - - int size = bytes == null ? -1 : bytes.length; - SerializerUtils.writeBigEndianIntToOutputStream(valueCountingBytes, size, helperBuffer); - if (bytes != null) { - valueCountingBytes.write(bytes); - } - - SerializerUtils.writeBigEndianIntToOutputStream( - headerBytes, - Ints.checkedCast(valueCountingBytes.getCount()), - helperBuffer - ); + valuesOut.writeInt(next == null ? -1 : 0); + strategy.writeTo(next, valuesOut); + headerOut.writeInt(Ints.checkedCast(valuesOut.size())); if (prevVal instanceof Closeable) { CloseQuietly.close((Closeable) prevVal); @@ -490,22 +511,18 @@ private static GenericIndexed fromIterableVersionOne(Iterable objectsI throw new RuntimeException(e); } - ByteBuffer theBuffer = ByteBuffer.allocate(Ints.BYTES + headerBytes.size() + valueBytes.size()); + ByteBuffer theBuffer = ByteBuffer.allocate(Ints.checkedCast(Ints.BYTES + headerOut.size() + valuesOut.size())); theBuffer.putInt(count); - headerBytes.writeTo(theBuffer); - valueBytes.writeTo(theBuffer); + headerOut.writeTo(theBuffer); + valuesOut.writeTo(theBuffer); theBuffer.flip(); - return new GenericIndexed<>(theBuffer.asReadOnlyBuffer(), strategy, allowReverseLookup); + return new GenericIndexed<>(theBuffer.asReadOnlyBuffer(), resultObjectStrategy, allowReverseLookup); } private long getSerializedSizeVersionOne() { - return theBuffer.remaining() - + 1 // version byte - + 1 // allowReverseLookup flag - + Ints.BYTES // numBytesUsed - + Ints.BYTES; // numElements + return metaSerdeHelper.size(this) + (long) theBuffer.remaining(); } private T getVersionOne(int index) @@ -560,15 +577,10 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) }; } - private void writeToChannelVersionOne(WritableByteChannel channel) throws IOException + private void writeToVersionOne(WritableByteChannel channel) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{ - VERSION_ONE, - allowReverseLookup ? REVERSE_LOOKUP_ALLOWED : REVERSE_LOOKUP_DISALLOWED - })); - channel.write(ByteBuffer.wrap(Ints.toByteArray(theBuffer.remaining() + Ints.BYTES))); - channel.write(ByteBuffer.wrap(Ints.toByteArray(size))); - channel.write(theBuffer.asReadOnlyBuffer()); + metaSerdeHelper.writeTo(channel, this); + Channels.writeFully(channel, theBuffer.asReadOnlyBuffer()); } diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java index bf6fe15ea677..10fcffe4659a 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java @@ -19,80 +19,151 @@ package io.druid.segment.data; -import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import com.google.common.io.InputSupplier; import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; -import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.io.Closer; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.java.util.common.io.smoosh.SmooshedWriter; - -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; +import io.druid.segment.writeout.WriteOutBytes; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.serde.Serializer; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongList; + +import javax.annotation.Nullable; +import java.io.DataInput; +import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; -import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; -import java.util.Arrays; /** * Streams arrays of objects out in the binary format described by GenericIndexed */ -public class GenericIndexedWriter implements Closeable +public class GenericIndexedWriter implements Serializer { private static int PAGE_SIZE = 4096; - private final IOPeon ioPeon; + + private static final MetaSerdeHelper singleFileMetaSerdeHelper = MetaSerdeHelper + .firstWriteByte((GenericIndexedWriter x) -> GenericIndexed.VERSION_ONE) + .writeByte( + x -> x.objectsSorted ? GenericIndexed.REVERSE_LOOKUP_ALLOWED : GenericIndexed.REVERSE_LOOKUP_DISALLOWED + ) + .writeInt(x -> Ints.checkedCast(x.headerOut.size() + x.valuesOut.size() + Integer.BYTES)) + .writeInt(x -> x.numWritten); + + private static final MetaSerdeHelper multiFileMetaSerdeHelper = MetaSerdeHelper + .firstWriteByte((GenericIndexedWriter x) -> GenericIndexed.VERSION_TWO) + .writeByte( + x -> x.objectsSorted ? GenericIndexed.REVERSE_LOOKUP_ALLOWED : GenericIndexed.REVERSE_LOOKUP_DISALLOWED + ) + .writeInt(GenericIndexedWriter::bagSizePower) + .writeInt(x -> x.numWritten) + .writeInt(x -> x.fileNameByteArray.length) + .writeByteArray(x -> x.fileNameByteArray); + + + static GenericIndexedWriter ofCompressedByteBuffers( + final SegmentWriteOutMedium segmentWriteOutMedium, + final String filenameBase, + final CompressionStrategy compressionStrategy, + final int bufferSize + ) + { + GenericIndexedWriter writer = new GenericIndexedWriter<>( + segmentWriteOutMedium, + filenameBase, + compressedByteBuffersWriteObjectStrategy(compressionStrategy, bufferSize, segmentWriteOutMedium.getCloser()) + ); + writer.objectsSorted = false; + return writer; + } + + static ObjectStrategy compressedByteBuffersWriteObjectStrategy( + final CompressionStrategy compressionStrategy, + final int bufferSize, + final Closer closer + ) + { + return new ObjectStrategy() + { + private final CompressionStrategy.Compressor compressor = compressionStrategy.getCompressor(); + private final ByteBuffer compressedDataBuffer = compressor.allocateOutBuffer(bufferSize, closer); + + @Override + public Class getClazz() + { + return ByteBuffer.class; + } + + @Override + public ByteBuffer fromByteBuffer(ByteBuffer buffer, int numBytes) + { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] toBytes(ByteBuffer val) + { + throw new UnsupportedOperationException(); + } + + @Override + public void writeTo(ByteBuffer val, WriteOutBytes out) throws IOException + { + compressedDataBuffer.clear(); + int valPos = val.position(); + out.write(compressor.compress(val, compressedDataBuffer)); + val.position(valPos); + } + + @Override + public int compare(ByteBuffer o1, ByteBuffer o2) + { + throw new UnsupportedOperationException(); + } + }; + } + + private final SegmentWriteOutMedium segmentWriteOutMedium; private final String filenameBase; private final ObjectStrategy strategy; private final int fileSizeLimit; private final byte[] fileNameByteArray; private boolean objectsSorted = true; private T prevObject = null; - private CountingOutputStream headerOut = null; - private CountingOutputStream valuesOut = null; - private CountingOutputStream headerOutLong = null; - private long numWritten = 0; + private WriteOutBytes headerOut = null; + private WriteOutBytes valuesOut = null; + private int numWritten = 0; private boolean requireMultipleFiles = false; - private ByteBuffer buf; - private final ByteBuffer sizeHelperBuffer = ByteBuffer.allocate(Ints.BYTES); + private LongList headerOutLong; + private final ByteBuffer getOffsetBuffer = ByteBuffer.allocate(Integer.BYTES); - public GenericIndexedWriter( - IOPeon ioPeon, - String filenameBase, - ObjectStrategy strategy - ) + public GenericIndexedWriter(SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy) { - this(ioPeon, filenameBase, strategy, Integer.MAX_VALUE & ~PAGE_SIZE); + this(segmentWriteOutMedium, filenameBase, strategy, Integer.MAX_VALUE & ~PAGE_SIZE); } public GenericIndexedWriter( - IOPeon ioPeon, + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy, int fileSizeLimit ) { - this.ioPeon = ioPeon; + this.segmentWriteOutMedium = segmentWriteOutMedium; this.filenameBase = filenameBase; this.strategy = strategy; this.fileSizeLimit = fileSizeLimit; fileNameByteArray = StringUtils.toUtf8(filenameBase); - buf = ByteBuffer.allocate(Ints.BYTES); } public static String generateValueFileName(String fileNameBase, int fileNum) @@ -127,8 +198,8 @@ private static void writeBytesIntoSmooshedChannel( public void open() throws IOException { - headerOut = new CountingOutputStream(ioPeon.makeOutputStream(makeFilename("header"))); - valuesOut = new CountingOutputStream(ioPeon.makeOutputStream(makeFilename("values"))); + headerOut = segmentWriteOutMedium.makeWriteOutBytes(); + valuesOut = segmentWriteOutMedium.makeWriteOutBytes(); } public void setObjectsNotSorted() @@ -142,87 +213,148 @@ public void write(T objectToWrite) throws IOException objectsSorted = false; } - byte[] bytesToWrite = strategy.toBytes(objectToWrite); - int size = bytesToWrite == null ? -1 : bytesToWrite.length; ++numWritten; - SerializerUtils.writeBigEndianIntToOutputStream(valuesOut, size, sizeHelperBuffer); - if (bytesToWrite != null) { - valuesOut.write(bytesToWrite); - } + + valuesOut.writeInt(objectToWrite == null ? -1 : 0); + strategy.writeTo(objectToWrite, valuesOut); if (!requireMultipleFiles) { - SerializerUtils.writeBigEndianIntToOutputStream(headerOut, Ints.checkedCast(valuesOut.getCount()), buf); + headerOut.writeInt(Ints.checkedCast(valuesOut.size())); } else { - SerializerUtils.writeNativeOrderedLongToOutputStream(headerOutLong, valuesOut.getCount(), buf); + headerOutLong.add(valuesOut.size()); } if (!requireMultipleFiles && getSerializedSize() > fileSizeLimit) { requireMultipleFiles = true; initializeHeaderOutLong(); - buf = ByteBuffer.allocate(Longs.BYTES).order(ByteOrder.nativeOrder()); } - prevObject = objectToWrite; + if (objectsSorted) { + prevObject = objectToWrite; + } } - private String makeFilename(String suffix) + @Nullable + public T get(int index) throws IOException { - return StringUtils.format("%s.%s", filenameBase, suffix); + long startOffset; + if (index == 0) { + startOffset = Integer.BYTES; + } else { + startOffset = getOffset(index - 1) + Integer.BYTES; + } + long endOffset = getOffset(index); + int valueSize = Ints.checkedCast(endOffset - startOffset); + if (valueSize == 0) { + return null; + } + ByteBuffer bb = ByteBuffer.allocate(valueSize); + valuesOut.readFully(startOffset, bb); + bb.clear(); + return strategy.fromByteBuffer(bb, valueSize); + } + + private long getOffset(int index) throws IOException + { + if (!requireMultipleFiles) { + getOffsetBuffer.clear(); + headerOut.readFully(index * (long) Integer.BYTES, getOffsetBuffer); + return getOffsetBuffer.getInt(0); + } else { + return headerOutLong.getLong(index); + } } @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { - valuesOut.close(); if (requireMultipleFiles) { - closeMultiFiles(); + // for multi-file version (version 2), getSerializedSize() returns number of bytes in meta file. + return multiFileMetaSerdeHelper.size(this); } else { - closeSingleFile(); + return singleFileMetaSerdeHelper.size(this) + headerOut.size() + valuesOut.size(); } } - private void closeSingleFile() throws IOException + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - headerOut.close(); - final long numBytesWritten = headerOut.getCount() + valuesOut.getCount(); + if (requireMultipleFiles) { + writeToMultiFiles(channel, smoosher); + } else { + writeToSingleFile(channel); + } + } + + private void writeToSingleFile(WritableByteChannel channel) throws IOException + { + final long numBytesWritten = headerOut.size() + valuesOut.size(); Preconditions.checkState( - headerOut.getCount() == (numWritten * 4), + headerOut.size() == (numWritten * 4), "numWritten[%s] number of rows should have [%s] bytes written to headerOut, had[%s]", numWritten, numWritten * 4, - headerOut.getCount() + headerOut.size() ); Preconditions.checkState( - numBytesWritten < fileSizeLimit, "Wrote[%s] bytes to base file %s, which is too many.", - numBytesWritten, - filenameBase + numBytesWritten < fileSizeLimit, "Wrote[%s] bytes, which is too many.", + numBytesWritten ); - try (OutputStream metaOut = ioPeon.makeOutputStream(makeFilename("meta"))) { - metaOut.write(GenericIndexed.VERSION_ONE); - metaOut.write(objectsSorted ? GenericIndexed.REVERSE_LOOKUP_ALLOWED : GenericIndexed.REVERSE_LOOKUP_DISALLOWED); - metaOut.write(Ints.toByteArray(Ints.checkedCast(numBytesWritten + 4))); - metaOut.write(Ints.toByteArray(Ints.checkedCast(numWritten))); - } + singleFileMetaSerdeHelper.writeTo(channel, this); + headerOut.writeTo(channel); + valuesOut.writeTo(channel); } - private void closeMultiFiles() throws IOException + private void writeToMultiFiles(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - headerOutLong.close(); Preconditions.checkState( - headerOutLong.getCount() == (numWritten * Longs.BYTES), - "numWritten[%s] number of rows should have [%s] bytes written to headerOutLong, had[%s]", + headerOutLong.size() == numWritten, + "numWritten[%s] number of rows doesn't match headerOutLong's size[%s]", numWritten, - numWritten * Longs.BYTES, - headerOutLong.getCount() + headerOutLong.size() ); Preconditions.checkState( - headerOutLong.getCount() < (Integer.MAX_VALUE & ~PAGE_SIZE), - "Wrote[%s] bytes in header file of base file %s, which is too many.", - headerOutLong.getCount(), - filenameBase + (((long) headerOutLong.size()) * Long.BYTES) < (Integer.MAX_VALUE & ~PAGE_SIZE), + "Wrote[%s] bytes in header, which is too many.", + (((long) headerOutLong.size()) * Long.BYTES) ); + + if (smoosher == null) { + throw new IAE("version 2 GenericIndexedWriter requires FileSmoosher."); + } + + int bagSizePower = bagSizePower(); + multiFileMetaSerdeHelper.writeTo(channel, this); + + long previousValuePosition = 0; + int bagSize = 1 << bagSizePower; + + int numberOfFilesRequired = GenericIndexed.getNumberOfFilesRequired(bagSize, numWritten); + byte[] buffer = new byte[1 << 16]; + + try (InputStream is = valuesOut.asInputStream()) { + int counter = -1; + for (int i = 0; i < numberOfFilesRequired; i++) { + long valuePosition; + if (i != numberOfFilesRequired - 1) { + valuePosition = headerOutLong.getLong(bagSize + counter); + counter = counter + bagSize; + } else { + valuePosition = headerOutLong.getLong(numWritten - 1); + } + + long numBytesToPutInFile = valuePosition - previousValuePosition; + + try (SmooshedWriter smooshChannel = smoosher + .addWithSmooshedWriter(generateValueFileName(filenameBase, i), numBytesToPutInFile)) { + writeBytesIntoSmooshedChannel(numBytesToPutInFile, buffer, smooshChannel, is); + previousValuePosition = valuePosition; + } + } + } + writeHeaderLong(smoosher, bagSizePower); } /** @@ -234,25 +366,19 @@ private void closeMultiFiles() throws IOException */ private int bagSizePower() throws IOException { - long avgObjectSize = (valuesOut.getCount() + numWritten - 1) / numWritten; + long avgObjectSize = (valuesOut.size() + numWritten - 1) / numWritten; - File f = ioPeon.getFile(makeFilename("headerLong")); - Preconditions.checkNotNull(f, "header file missing."); - - try (RandomAccessFile headerFile = new RandomAccessFile(f, "r")) { - for (int i = 31; i >= 0; --i) { - if ((1L << i) * avgObjectSize <= fileSizeLimit) { - if (actuallyFits(i, headerFile)) { - return i; - } + for (int i = 31; i >= 0; --i) { + if ((1L << i) * avgObjectSize <= fileSizeLimit) { + if (actuallyFits(i)) { + return i; } } } throw new ISE( - "no value split found with fileSizeLimit [%d], avgObjectSize [%d] while serializing [%s]", + "no value split found with fileSizeLimit [%d], avgObjectSize [%d]", fileSizeLimit, - avgObjectSize, - filenameBase + avgObjectSize ); } @@ -260,17 +386,16 @@ private int bagSizePower() throws IOException * Checks if candidate value splits can divide value file in such a way no object/element crosses the value splits. * * @param powerTwo candidate value split expressed as power of 2. - * @param headerFile header file. * * @return true if candidate value split can hold all splits. * * @throws IOException */ - private boolean actuallyFits(int powerTwo, RandomAccessFile headerFile) throws IOException + private boolean actuallyFits(int powerTwo) throws IOException { long lastValueOffset = 0; long currentValueOffset = 0; - long valueBytesWritten = valuesOut.getCount(); + long valueBytesWritten = valuesOut.size(); long headerIndex = 0; long bagSize = 1L << powerTwo; @@ -279,11 +404,9 @@ private boolean actuallyFits(int powerTwo, RandomAccessFile headerFile) throws I if (headerIndex >= numWritten) { return true; } else if (headerIndex + bagSize <= numWritten) { - headerFile.seek((headerIndex + bagSize - 1) * Longs.BYTES); - currentValueOffset = Long.reverseBytes(headerFile.readLong()); + currentValueOffset = headerOutLong.getLong(Ints.checkedCast(headerIndex + bagSize - 1)); } else if (numWritten < headerIndex + bagSize) { - headerFile.seek((numWritten - 1) * Longs.BYTES); - currentValueOffset = Long.reverseBytes(headerFile.readLong()); + currentValueOffset = headerOutLong.getLong(numWritten - 1); } if (currentValueOffset - lastValueOffset <= fileSizeLimit) { @@ -296,132 +419,17 @@ private boolean actuallyFits(int powerTwo, RandomAccessFile headerFile) throws I return true; } - public long getSerializedSize() - { - // for version 2 getSerializedSize() returns number of bytes in meta file. - if (!requireMultipleFiles) { - return 2 + // version and sorted flag - Ints.BYTES + // numBytesWritten - Ints.BYTES + // numElements - headerOut.getCount() + // header length - valuesOut.getCount(); // value length - } else { - return 2 + // version and sorted flag - Ints.BYTES + // numElements as log base 2. - Ints.BYTES + // number of files - Ints.BYTES + // column name Size - fileNameByteArray.length; - } - } - - @Deprecated - public InputSupplier combineStreams() - { - // ByteSource.concat is only available in guava 15 and higher - // This is guava 14 compatible - if (requireMultipleFiles) { - throw new ISE("Can not combine streams for version 2."); //fallback to old behaviour. - } - - return ByteStreams.join( - Iterables.transform( - Arrays.asList("meta", "header", "values"), - new Function>() - { - @Override - public InputSupplier apply(final String input) - { - return new InputSupplier() - { - @Override - public InputStream getInput() throws IOException - { - return ioPeon.makeInputStream(makeFilename(input)); - } - }; - } - } - ) - ); - } - - private void writeToChannelVersionOne(WritableByteChannel channel) throws IOException - { - try (ReadableByteChannel from = Channels.newChannel(combineStreams().getInput())) { - ByteStreams.copy(from, channel); - } - - } - - private void writeToChannelVersionTwo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException - { - if (smoosher == null) { - throw new IAE("version 2 GenericIndexedWriter requires FileSmoosher."); - } - - int bagSizePower = bagSizePower(); - OutputStream metaOut = Channels.newOutputStream(channel); - metaOut.write(GenericIndexed.VERSION_TWO); - metaOut.write(objectsSorted ? GenericIndexed.REVERSE_LOOKUP_ALLOWED : GenericIndexed.REVERSE_LOOKUP_DISALLOWED); - metaOut.write(Ints.toByteArray(bagSizePower)); - metaOut.write(Ints.toByteArray(Ints.checkedCast(numWritten))); - metaOut.write(Ints.toByteArray(fileNameByteArray.length)); - metaOut.write(fileNameByteArray); - - try (RandomAccessFile headerFile = new RandomAccessFile(ioPeon.getFile(makeFilename("headerLong")), "r")) { - Preconditions.checkNotNull(headerFile, "header file missing."); - long previousValuePosition = 0; - int bagSize = 1 << bagSizePower; - - int numberOfFilesRequired = GenericIndexed.getNumberOfFilesRequired(bagSize, numWritten); - byte[] buffer = new byte[1 << 16]; - - try (InputStream is = new FileInputStream(ioPeon.getFile(makeFilename("values")))) { - int counter = -1; - - for (int i = 0; i < numberOfFilesRequired; i++) { - if (i != numberOfFilesRequired - 1) { - headerFile.seek((bagSize + counter) * Longs.BYTES); // 8 for long bytes. - counter = counter + bagSize; - } else { - headerFile.seek((numWritten - 1) * Longs.BYTES); // for remaining items. - } - - long valuePosition = Long.reverseBytes(headerFile.readLong()); - long numBytesToPutInFile = valuePosition - previousValuePosition; - - try (SmooshedWriter smooshChannel = smoosher - .addWithSmooshedWriter(generateValueFileName(filenameBase, i), numBytesToPutInFile)) { - writeBytesIntoSmooshedChannel(numBytesToPutInFile, buffer, smooshChannel, is); - previousValuePosition = valuePosition; - } - } - } - writeHeaderLong(smoosher, headerFile, bagSizePower, buffer); - } - } - - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException - { - if (!requireMultipleFiles) { - writeToChannelVersionOne(channel); - } else { - writeToChannelVersionTwo(channel, smoosher); - } - } - - private void writeHeaderLong(FileSmoosher smoosher, RandomAccessFile headerFile, int bagSizePower, byte[] buffer) + private void writeHeaderLong(FileSmoosher smoosher, int bagSizePower) throws IOException { ByteBuffer helperBuffer = ByteBuffer.allocate(Ints.BYTES).order(ByteOrder.nativeOrder()); - try (CountingOutputStream finalHeaderOut = new CountingOutputStream( - ioPeon.makeOutputStream(makeFilename("header_final")))) { - int numberOfElementsPerValueFile = 1 << bagSizePower; - long currentNumBytes = 0; - long relativeRefBytes = 0; - long relativeNumBytes; - headerFile.seek(0); + int numberOfElementsPerValueFile = 1 << bagSizePower; + long currentNumBytes = 0; + long relativeRefBytes = 0; + long relativeNumBytes; + try (SmooshedWriter smooshChannel = smoosher + .addWithSmooshedWriter(generateHeaderFileName(filenameBase), numWritten * Integer.BYTES)) { // following block converts long header indexes into int header indexes. for (int pos = 0; pos < numWritten; pos++) { @@ -430,38 +438,22 @@ private void writeHeaderLong(FileSmoosher smoosher, RandomAccessFile headerFile, if ((pos & (numberOfElementsPerValueFile - 1)) == 0) { relativeRefBytes = currentNumBytes; } - currentNumBytes = Long.reverseBytes(headerFile.readLong()); + currentNumBytes = headerOutLong.getLong(pos); relativeNumBytes = currentNumBytes - relativeRefBytes; - SerializerUtils.writeNativeOrderedIntToOutputStream( - finalHeaderOut, - Ints.checkedCast(relativeNumBytes), - helperBuffer - ); - } - - long numBytesToPutInFile = finalHeaderOut.getCount(); - finalHeaderOut.close(); - try (InputStream is = new FileInputStream(ioPeon.getFile(makeFilename("header_final")))) { - try (SmooshedWriter smooshChannel = smoosher - .addWithSmooshedWriter(generateHeaderFileName(filenameBase), numBytesToPutInFile)) { - writeBytesIntoSmooshedChannel(numBytesToPutInFile, buffer, smooshChannel, is); - } + helperBuffer.putInt(0, Ints.checkedCast(relativeNumBytes)); + helperBuffer.clear(); + smooshChannel.write(helperBuffer); } - } } private void initializeHeaderOutLong() throws IOException { - headerOut.close(); - headerOutLong = new CountingOutputStream(ioPeon.makeOutputStream(makeFilename("headerLong"))); - - try (RandomAccessFile headerFile = new RandomAccessFile(ioPeon.getFile(makeFilename("header")), "r")) { - ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES).order(ByteOrder.nativeOrder()); - for (int i = 0; i < numWritten; i++) { - int count = headerFile.readInt(); - SerializerUtils.writeNativeOrderedLongToOutputStream(headerOutLong, count, buf); - } + headerOutLong = new LongArrayList(); + DataInput headerOutAsIntInput = new DataInputStream(headerOut.asInputStream()); + for (int i = 0; i < numWritten; i++) { + int count = headerOutAsIntInput.readInt(); + headerOutLong.add(count); } } diff --git a/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java index 8623b0a59c68..14f38f4ef49e 100644 --- a/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java @@ -22,6 +22,7 @@ import com.google.common.collect.Ordering; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.spatial.ImmutableRTree; +import it.unimi.dsi.fastutil.bytes.ByteArrays; import java.nio.ByteBuffer; @@ -71,7 +72,7 @@ public ImmutableRTree fromByteBuffer(ByteBuffer buffer, int numBytes) public byte[] toBytes(ImmutableRTree val) { if (val == null || val.size() == 0) { - return new byte[]{}; + return ByteArrays.EMPTY_ARRAY; } return val.toBytes(); } diff --git a/processing/src/main/java/io/druid/segment/data/IndexedIntsWriter.java b/processing/src/main/java/io/druid/segment/data/IndexedIntsWriter.java index f18e897b4749..b34c40e431d8 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedIntsWriter.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedIntsWriter.java @@ -19,19 +19,13 @@ package io.druid.segment.data; -import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.Serializer; -import java.io.Closeable; import java.io.IOException; -import java.nio.channels.WritableByteChannel; -public interface IndexedIntsWriter extends Closeable +public interface IndexedIntsWriter extends Serializer { void open() throws IOException; void add(Object obj) throws IOException; - - long getSerializedSize(); - - void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/data/IntermediateLongSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/IntermediateLongSupplierSerializer.java index abc5aff145a3..8c2be60c1509 100644 --- a/processing/src/main/java/io/druid/segment/data/IntermediateLongSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/IntermediateLongSupplierSerializer.java @@ -19,21 +19,15 @@ package io.druid.segment.data; -import com.google.common.io.ByteSink; -import com.google.common.io.CountingOutputStream; import com.google.common.math.LongMath; -import com.google.common.primitives.Longs; -import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.SegmentWriteOutMedium; import it.unimi.dsi.fastutil.longs.Long2IntMap; import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.longs.LongList; -import java.io.BufferedInputStream; -import java.io.DataInputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -42,14 +36,11 @@ */ public class IntermediateLongSupplierSerializer implements LongSupplierSerializer { - - private final IOPeon ioPeon; + private final SegmentWriteOutMedium segmentWriteOutMedium; private final String filenameBase; - private final String tempFile; private final ByteOrder order; - private final CompressedObjectStrategy.CompressionStrategy compression; - private CountingOutputStream tempOut = null; - private final ByteBuffer helperBuffer = ByteBuffer.allocate(Longs.BYTES); + private final CompressionStrategy compression; + private LongList tempOut = null; private int numInserted = 0; @@ -61,15 +52,14 @@ public class IntermediateLongSupplierSerializer implements LongSupplierSerialize private LongSupplierSerializer delegate; - public IntermediateLongSupplierSerializer( - IOPeon ioPeon, + IntermediateLongSupplierSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder order, - CompressedObjectStrategy.CompressionStrategy compression + CompressionStrategy compression ) { - this.ioPeon = ioPeon; - this.tempFile = filenameBase + ".temp"; + this.segmentWriteOutMedium = segmentWriteOutMedium; this.filenameBase = filenameBase; this.order = order; this.compression = compression; @@ -78,7 +68,7 @@ public IntermediateLongSupplierSerializer( @Override public void open() throws IOException { - tempOut = new CountingOutputStream(ioPeon.makeOutputStream(tempFile)); + tempOut = new LongArrayList(); } @Override @@ -90,7 +80,11 @@ public int size() @Override public void add(long value) throws IOException { - SerializerUtils.writeBigEndianLongToOutputStream(tempOut, value, helperBuffer); + //noinspection VariableNotUsedInsideIf + if (delegate != null) { + throw new IllegalStateException("written out already"); + } + tempOut.add(value); ++numInserted; if (uniqueValues.size() <= CompressionFactory.MAX_TABLE_SIZE && !uniqueValues.containsKey(value)) { uniqueValues.put(value, uniqueValues.size()); @@ -106,6 +100,10 @@ public void add(long value) throws IOException private void makeDelegate() throws IOException { + //noinspection VariableNotUsedInsideIf + if (delegate != null) { + return; + } CompressionFactory.LongEncodingWriter writer; long delta; try { @@ -122,51 +120,29 @@ private void makeDelegate() throws IOException writer = new LongsLongEncodingWriter(order); } - if (compression == CompressedObjectStrategy.CompressionStrategy.NONE) { - delegate = new EntireLayoutLongSupplierSerializer( - ioPeon, filenameBase, writer - ); + if (compression == CompressionStrategy.NONE) { + delegate = new EntireLayoutLongSupplierSerializer(segmentWriteOutMedium, writer); } else { - delegate = new BlockLayoutLongSupplierSerializer( - ioPeon, filenameBase, order, writer, compression - ); + delegate = new BlockLayoutLongSupplierSerializer(segmentWriteOutMedium, filenameBase, order, writer, compression); } - try (DataInputStream tempIn = new DataInputStream(new BufferedInputStream(ioPeon.makeInputStream(tempFile)))) { - delegate.open(); - int available = numInserted; - while (available > 0) { - delegate.add(tempIn.readLong()); - available--; - } + delegate.open(); + for (int i = 0; i < tempOut.size(); i++) { + delegate.add(tempOut.getLong(i)); } } @Override - public void closeAndConsolidate(ByteSink consolidatedOut) throws IOException + public long getSerializedSize() throws IOException { - tempOut.close(); makeDelegate(); - delegate.closeAndConsolidate(consolidatedOut); - } - - @Override - public void close() throws IOException - { - tempOut.close(); - makeDelegate(); - delegate.close(); - } - - @Override - public long getSerializedSize() - { return delegate.getSerializedSize(); } @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - delegate.writeToChannel(channel, smoosher); + makeDelegate(); + delegate.writeTo(channel, smoosher); } } diff --git a/processing/src/main/java/io/druid/segment/data/LongSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/LongSupplierSerializer.java index b40554fddfe4..d03fb94760cf 100644 --- a/processing/src/main/java/io/druid/segment/data/LongSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/LongSupplierSerializer.java @@ -19,21 +19,15 @@ package io.druid.segment.data; -import com.google.common.io.ByteSink; -import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.serde.Serializer; -import java.io.Closeable; import java.io.IOException; -import java.nio.channels.WritableByteChannel; /** */ -public interface LongSupplierSerializer extends Closeable +public interface LongSupplierSerializer extends Serializer { void open() throws IOException; int size(); void add(long value) throws IOException; - void closeAndConsolidate(ByteSink consolidatedOut) throws IOException; - long getSerializedSize(); - void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/data/LongsLongEncodingReader.java b/processing/src/main/java/io/druid/segment/data/LongsLongEncodingReader.java index 9cbb4f96a70b..078226e1cfc8 100644 --- a/processing/src/main/java/io/druid/segment/data/LongsLongEncodingReader.java +++ b/processing/src/main/java/io/druid/segment/data/LongsLongEncodingReader.java @@ -19,8 +19,6 @@ package io.druid.segment.data; -import com.google.common.primitives.Longs; - import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.LongBuffer; @@ -51,12 +49,6 @@ public long read(int index) return buffer.get(buffer.position() + index); } - @Override - public int getNumBytes(int values) - { - return values * Longs.BYTES; - } - @Override public CompressionFactory.LongEncodingReader duplicate() { diff --git a/processing/src/main/java/io/druid/segment/data/LongsLongEncodingWriter.java b/processing/src/main/java/io/druid/segment/data/LongsLongEncodingWriter.java index 8a32d01e999f..baa01879ae93 100644 --- a/processing/src/main/java/io/druid/segment/data/LongsLongEncodingWriter.java +++ b/processing/src/main/java/io/druid/segment/data/LongsLongEncodingWriter.java @@ -20,6 +20,7 @@ package io.druid.segment.data; import com.google.common.primitives.Longs; +import io.druid.segment.writeout.WriteOutBytes; import java.io.IOException; import java.io.OutputStream; @@ -52,7 +53,7 @@ public void setBuffer(ByteBuffer buffer) } @Override - public void setOutputStream(OutputStream output) + public void setOutputStream(WriteOutBytes output) { outBuffer = null; outStream = output; @@ -77,9 +78,15 @@ public void flush() throws IOException } @Override - public void putMeta(OutputStream metaOut, CompressedObjectStrategy.CompressionStrategy strategy) throws IOException + public void putMeta(ByteBuffer metaOut, CompressionStrategy strategy) throws IOException { - metaOut.write(strategy.getId()); + metaOut.put(strategy.getId()); + } + + @Override + public int metaSize() + { + return 1; } @Override diff --git a/processing/src/main/java/io/druid/segment/data/MultiValueIndexedIntsWriter.java b/processing/src/main/java/io/druid/segment/data/MultiValueIndexedIntsWriter.java index e1d096b15ce0..05d6a578045c 100644 --- a/processing/src/main/java/io/druid/segment/data/MultiValueIndexedIntsWriter.java +++ b/processing/src/main/java/io/druid/segment/data/MultiValueIndexedIntsWriter.java @@ -19,11 +19,11 @@ package io.druid.segment.data; -import com.google.common.primitives.Ints; import io.druid.java.util.common.IAE; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; import java.io.IOException; -import java.util.List; public abstract class MultiValueIndexedIntsWriter implements IndexedIntsWriter { @@ -33,13 +33,13 @@ public void add(Object obj) throws IOException if (obj == null) { addValues(null); } else if (obj instanceof int[]) { - addValues(Ints.asList((int[]) obj)); - } else if (obj instanceof List) { - addValues((List) obj); + addValues(IntArrayList.wrap((int[]) obj)); + } else if (obj instanceof IntList) { + addValues((IntList) obj); } else { throw new IAE("unsupported multi-value type: " + obj.getClass()); } } - protected abstract void addValues(List vals) throws IOException; + protected abstract void addValues(IntList vals) throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java index b6063a1c140e..c1442367e125 100644 --- a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java @@ -20,8 +20,10 @@ package io.druid.segment.data; import io.druid.guice.annotations.ExtensionPoint; +import io.druid.segment.writeout.WriteOutBytes; import javax.annotation.Nullable; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Comparator; @@ -32,15 +34,16 @@ public interface ObjectStrategy extends Comparator /** * Convert values from their underlying byte representation. - * + *

* Implementations of this method may change the given buffer's mark, or limit, and position. - * + *

* Implementations of this method may not store the given buffer in a field of the "deserialized" object, * need to use {@link ByteBuffer#slice()}, {@link ByteBuffer#asReadOnlyBuffer()} or {@link ByteBuffer#duplicate()} in * this case. * - * @param buffer buffer to read value from + * @param buffer buffer to read value from * @param numBytes number of bytes used to store the value, starting at buffer.position() + * * @return an object created from the given byte buffer representation */ @Nullable @@ -48,4 +51,12 @@ public interface ObjectStrategy extends Comparator @Nullable byte[] toBytes(@Nullable T val); + + default void writeTo(T val, WriteOutBytes out) throws IOException + { + byte[] bytes = toBytes(val); + if (bytes != null) { + out.write(toBytes(val)); + } + } } diff --git a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java index fcdc63dfd768..4125927f8937 100644 --- a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java @@ -68,8 +68,7 @@ public BitmapFactory getBitmapFactory() return bitmapFactory; } - private static class ImmutableRoaringBitmapObjectStrategy - implements ObjectStrategy + private static class ImmutableRoaringBitmapObjectStrategy implements ObjectStrategy { @Override public Class getClazz() diff --git a/processing/src/main/java/io/druid/segment/data/TableLongEncodingReader.java b/processing/src/main/java/io/druid/segment/data/TableLongEncodingReader.java index db79c23229b3..bd1c10e029fc 100644 --- a/processing/src/main/java/io/druid/segment/data/TableLongEncodingReader.java +++ b/processing/src/main/java/io/druid/segment/data/TableLongEncodingReader.java @@ -71,12 +71,6 @@ public long read(int index) return table[(int) deserializer.get(index)]; } - @Override - public int getNumBytes(int values) - { - return VSizeLongSerde.getSerializedSize(bitsPerValue, values); - } - @Override public CompressionFactory.LongEncodingReader duplicate() { diff --git a/processing/src/main/java/io/druid/segment/data/TableLongEncodingWriter.java b/processing/src/main/java/io/druid/segment/data/TableLongEncodingWriter.java index 69fde7e03fbf..36382a2b13b5 100644 --- a/processing/src/main/java/io/druid/segment/data/TableLongEncodingWriter.java +++ b/processing/src/main/java/io/druid/segment/data/TableLongEncodingWriter.java @@ -21,13 +21,12 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; -import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.IAE; +import io.druid.segment.writeout.WriteOutBytes; import it.unimi.dsi.fastutil.longs.Long2IntMap; import it.unimi.dsi.fastutil.longs.LongList; import java.io.IOException; -import java.io.OutputStream; import java.nio.ByteBuffer; public class TableLongEncodingWriter implements CompressionFactory.LongEncodingWriter @@ -55,7 +54,7 @@ public void setBuffer(ByteBuffer buffer) } @Override - public void setOutputStream(OutputStream output) + public void setOutputStream(WriteOutBytes output) { serializer = VSizeLongSerde.getSerializer(bitsPerValue, output); } @@ -75,18 +74,23 @@ public void flush() throws IOException } @Override - public void putMeta(OutputStream metaOut, CompressedObjectStrategy.CompressionStrategy strategy) throws IOException + public void putMeta(ByteBuffer metaOut, CompressionStrategy strategy) throws IOException { - metaOut.write(CompressionFactory.setEncodingFlag(strategy.getId())); - metaOut.write(CompressionFactory.LongEncodingFormat.TABLE.getId()); - metaOut.write(CompressionFactory.TABLE_ENCODING_VERSION); - metaOut.write(Ints.toByteArray(table.size())); - ByteBuffer helperBuffer = ByteBuffer.allocate(Longs.BYTES); + metaOut.put(CompressionFactory.setEncodingFlag(strategy.getId())); + metaOut.put(CompressionFactory.LongEncodingFormat.TABLE.getId()); + metaOut.put(CompressionFactory.TABLE_ENCODING_VERSION); + metaOut.putInt(table.size()); for (int i = 0; i < valueAddedInOrder.size(); i++) { - SerializerUtils.writeBigEndianLongToOutputStream(metaOut, valueAddedInOrder.getLong(i), helperBuffer); + metaOut.putLong(valueAddedInOrder.getLong(i)); } } + @Override + public int metaSize() + { + return 1 + 1 + 1 + Ints.BYTES + (table.size() * Longs.BYTES); + } + @Override public int getBlockSize(int bytesPerBlock) { diff --git a/processing/src/main/java/io/druid/segment/data/TmpFileIOPeon.java b/processing/src/main/java/io/druid/segment/data/TmpFileIOPeon.java deleted file mode 100644 index ff22058f6937..000000000000 --- a/processing/src/main/java/io/druid/segment/data/TmpFileIOPeon.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - -import com.google.common.collect.Maps; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Map; - -/** - */ -public class TmpFileIOPeon implements IOPeon -{ - private final File dir; - private final boolean allowOverwrite; - private final Map createdFiles = Maps.newLinkedHashMap(); - - public TmpFileIOPeon() - { - this(true); - } - - public TmpFileIOPeon(boolean allowOverwrite) - { - this(null, allowOverwrite); - } - - public TmpFileIOPeon(File dir, boolean allowOverwrite) - { - this.dir = dir; - this.allowOverwrite = allowOverwrite; - } - - @Override - public OutputStream makeOutputStream(String filename) throws IOException - { - File retFile = createdFiles.get(filename); - if (retFile == null) { - retFile = File.createTempFile("filePeon", filename, dir); - createdFiles.put(filename, retFile); - return new BufferedOutputStream(new FileOutputStream(retFile)); - } else if (allowOverwrite) { - return new BufferedOutputStream(new FileOutputStream(retFile)); - } else { - throw new IOException("tmp file conflicts, file[" + filename + "] already exist!"); - } - } - - @Override - public InputStream makeInputStream(String filename) throws IOException - { - final File retFile = createdFiles.get(filename); - - return retFile == null ? null : new FileInputStream(retFile); - } - - @Override - public void close() throws IOException - { - for (File file : createdFiles.values()) { - file.delete(); - } - createdFiles.clear(); - } - - @Override - public File getFile(String filename) - { - return createdFiles.get(filename); - } - -} diff --git a/processing/src/main/java/io/druid/segment/data/VSizeCompressedObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/VSizeCompressedObjectStrategy.java deleted file mode 100644 index 1595e2b9af3a..000000000000 --- a/processing/src/main/java/io/druid/segment/data/VSizeCompressedObjectStrategy.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - -import io.druid.java.util.common.guava.Comparators; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -public class VSizeCompressedObjectStrategy extends CompressedObjectStrategy -{ - - private final int expectedBytes; - - public static VSizeCompressedObjectStrategy getBufferForOrder( - final ByteOrder order, - final CompressionStrategy compression, - final int expectedBytes - ) - { - return new VSizeCompressedObjectStrategy(order, compression, expectedBytes); - } - - protected VSizeCompressedObjectStrategy( - ByteOrder order, - CompressionStrategy compression, - int expectedBytes - ) - { - super(order, new BufferConverter() - { - @Override - public ByteBuffer convert(ByteBuffer buf) - { - return buf; - } - - @Override - public int compare(ByteBuffer lhs, ByteBuffer rhs) - { - return Comparators.naturalNullsFirst().compare(lhs, rhs); - } - - @Override - public int sizeOf(int count) - { - return count; // 1 byte per element - } - - @Override - public ByteBuffer combine(ByteBuffer into, ByteBuffer from) - { - return into.put(from); - } - }, compression); - - this.expectedBytes = expectedBytes; - } - - @Override - protected ByteBuffer bufferFor(ByteBuffer val) - { - return ByteBuffer.allocate(expectedBytes).order(order); - } - - @Override - protected void decompress(ByteBuffer buffer, int numBytes, ByteBuffer buf) - { - decompressor.decompress(buffer, numBytes, buf, expectedBytes); - } -} diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java index 16731a93bffc..84db1835c267 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java @@ -20,11 +20,14 @@ package io.druid.segment.data; import com.google.common.primitives.Ints; -import io.druid.common.utils.SerializerUtils; -import io.druid.io.ZeroCopyByteArrayOutputStream; +import io.druid.common.utils.ByteUtils; +import io.druid.io.Channels; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.HeapByteBufferWriteOutBytes; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.serde.MetaSerdeHelper; import java.io.IOException; import java.nio.ByteBuffer; @@ -33,17 +36,22 @@ /** */ -public class VSizeIndexed implements IndexedMultivalue +public class VSizeIndexed implements IndexedMultivalue, WritableSupplier> { - private static final byte version = 0x1; + private static final byte VERSION = 0x1; + + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((VSizeIndexed x) -> VERSION) + .writeByte(x -> ByteUtils.checkedCast(x.numBytes)) + .writeInt(x -> Ints.checkedCast(x.theBuffer.remaining() + (long) Integer.BYTES)) + .writeInt(x -> x.size); public static VSizeIndexed fromIterable(Iterable objectsIterable) { Iterator objects = objectsIterable.iterator(); if (!objects.hasNext()) { - final ByteBuffer buffer = ByteBuffer.allocate(4).putInt(0); - buffer.flip(); - return new VSizeIndexed(buffer, 4); + final ByteBuffer buffer = ByteBuffer.allocate(Ints.BYTES).putInt(0, 0); + return new VSizeIndexed(buffer, Ints.BYTES); } int numBytes = -1; @@ -57,29 +65,27 @@ public static VSizeIndexed fromIterable(Iterable objectsIterab ++count; } - ZeroCopyByteArrayOutputStream headerBytes = new ZeroCopyByteArrayOutputStream(4 + (count * 4)); - ZeroCopyByteArrayOutputStream valueBytes = new ZeroCopyByteArrayOutputStream(); - ByteBuffer helperBuffer = ByteBuffer.allocate(Ints.BYTES); + HeapByteBufferWriteOutBytes headerBytes = new HeapByteBufferWriteOutBytes(); + HeapByteBufferWriteOutBytes valueBytes = new HeapByteBufferWriteOutBytes(); int offset = 0; try { - SerializerUtils.writeBigEndianIntToOutputStream(headerBytes, count, helperBuffer); + headerBytes.writeInt(count); for (VSizeIndexedInts object : objectsIterable) { if (object.getNumBytes() != numBytes) { throw new ISE("val.numBytes[%s] != numBytesInValue[%s]", object.getNumBytes(), numBytes); } - byte[] bytes = object.getBytesNoPadding(); - offset += bytes.length; - SerializerUtils.writeBigEndianIntToOutputStream(headerBytes, offset, helperBuffer); - valueBytes.write(bytes); + offset += object.getNumBytesNoPadding(); + headerBytes.writeInt(offset); + object.writeBytesNoPaddingTo(valueBytes); } - valueBytes.write(new byte[4 - numBytes]); + valueBytes.write(new byte[Ints.BYTES - numBytes]); } catch (IOException e) { throw new RuntimeException(e); } - ByteBuffer theBuffer = ByteBuffer.allocate(headerBytes.size() + valueBytes.size()); + ByteBuffer theBuffer = ByteBuffer.allocate(Ints.checkedCast(headerBytes.size() + valueBytes.size())); headerBytes.writeTo(theBuffer); valueBytes.writeTo(theBuffer); theBuffer.flip(); @@ -149,24 +155,30 @@ public int indexOf(IndexedInts value) throw new UnsupportedOperationException("Reverse lookup not allowed."); } - public int getSerializedSize() + @Override + public long getSerializedSize() throws IOException { - return theBuffer.remaining() + 4 + 4 + 2; + return metaSerdeHelper.size(this) + (long) theBuffer.remaining(); } - public void writeToChannel(WritableByteChannel channel) throws IOException + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{version, (byte) numBytes})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(theBuffer.remaining() + 4))); - channel.write(ByteBuffer.wrap(Ints.toByteArray(size))); - channel.write(theBuffer.asReadOnlyBuffer()); + metaSerdeHelper.writeTo(channel, this); + Channels.writeFully(channel, theBuffer.asReadOnlyBuffer()); + } + + @Override + public IndexedMultivalue get() + { + return this; } public static VSizeIndexed readFromByteBuffer(ByteBuffer buffer) { byte versionFromBuffer = buffer.get(); - if (version == versionFromBuffer) { + if (VERSION == versionFromBuffer) { int numBytes = buffer.get(); int size = buffer.getInt(); ByteBuffer bufferToUse = buffer.asReadOnlyBuffer(); @@ -196,37 +208,4 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("theBuffer", theBuffer); } - - public WritableSupplier> asWritableSupplier() - { - return new VSizeIndexedSupplier(this); - } - - public static class VSizeIndexedSupplier implements WritableSupplier> - { - final VSizeIndexed delegate; - - public VSizeIndexedSupplier(VSizeIndexed delegate) - { - this.delegate = delegate; - } - - @Override - public long getSerializedSize() - { - return delegate.getSerializedSize(); - } - - @Override - public void writeToChannel(WritableByteChannel channel) throws IOException - { - delegate.writeToChannel(channel); - } - - @Override - public IndexedMultivalue get() - { - return delegate; - } - } } diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java index c9bee91cf823..1c499040bb00 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java @@ -20,20 +20,31 @@ package io.druid.segment.data; import com.google.common.primitives.Ints; +import io.druid.common.utils.ByteUtils; +import io.druid.io.Channels; import io.druid.java.util.common.IAE; +import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.serde.MetaSerdeHelper; +import io.druid.segment.writeout.HeapByteBufferWriteOutBytes; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; -import java.util.List; /** */ -public class VSizeIndexedInts implements IndexedInts, Comparable +public class VSizeIndexedInts implements IndexedInts, Comparable, WritableSupplier { public static final byte VERSION = 0x0; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((VSizeIndexedInts x) -> VERSION) + .writeByte(x -> ByteUtils.checkedCast(x.numBytes)) + .writeInt(x -> x.buffer.remaining()); + public static VSizeIndexedInts fromArray(int[] array) { return fromArray(array, Ints.max(array)); @@ -41,23 +52,10 @@ public static VSizeIndexedInts fromArray(int[] array) public static VSizeIndexedInts fromArray(int[] array, int maxValue) { - return fromList(Ints.asList(array), maxValue); - } - - /** - * provide for performance reason. - */ - public static byte[] getBytesNoPaddingFromList(List list, int maxValue) - { - int numBytes = getNumBytesForMax(maxValue); - - final ByteBuffer buffer = ByteBuffer.allocate((list.size() * numBytes)); - writeToBuffer(buffer, list, numBytes, maxValue); - - return buffer.array(); + return fromList(IntArrayList.wrap(array), maxValue); } - public static VSizeIndexedInts fromList(List list, int maxValue) + public static VSizeIndexedInts fromList(IntList list, int maxValue) { int numBytes = getNumBytesForMax(maxValue); @@ -67,11 +65,11 @@ public static VSizeIndexedInts fromList(List list, int maxValue) return new VSizeIndexedInts(buffer.asReadOnlyBuffer(), numBytes); } - private static void writeToBuffer(ByteBuffer buffer, List list, int numBytes, int maxValue) + private static void writeToBuffer(ByteBuffer buffer, IntList list, int numBytes, int maxValue) { - int i = 0; ByteBuffer helperBuffer = ByteBuffer.allocate(Ints.BYTES); - for (Integer val : list) { + for (int i = 0; i < list.size(); i++) { + int val = list.getInt(i); if (val < 0) { throw new IAE("integer values must be positive, got[%d], i[%d]", val, i); } @@ -81,7 +79,6 @@ private static void writeToBuffer(ByteBuffer buffer, List list, int num helperBuffer.putInt(0, val); buffer.put(helperBuffer.array(), Ints.BYTES - numBytes, numBytes); - ++i; } buffer.position(0); } @@ -92,15 +89,14 @@ public static byte getNumBytesForMax(int maxValue) throw new IAE("maxValue[%s] must be positive", maxValue); } - byte numBytes = 4; if (maxValue <= 0xFF) { - numBytes = 1; + return 1; } else if (maxValue <= 0xFFFF) { - numBytes = 2; + return 2; } else if (maxValue <= 0xFFFFFF) { - numBytes = 3; + return 3; } - return numBytes; + return 4; } private final ByteBuffer buffer; @@ -132,12 +128,16 @@ public int get(int index) return buffer.getInt(buffer.position() + (index * numBytes)) >>> bitsToShift; } - public byte[] getBytesNoPadding() + public int getNumBytesNoPadding() + { + return buffer.remaining() - (Ints.BYTES - numBytes); + } + + public void writeBytesNoPaddingTo(HeapByteBufferWriteOutBytes out) { - int bytesToTake = buffer.remaining() - (4 - numBytes); - byte[] bytes = new byte[bytesToTake]; - buffer.asReadOnlyBuffer().get(bytes); - return bytes; + ByteBuffer toWrite = buffer.slice(); + toWrite.limit(toWrite.limit() - (Ints.BYTES - numBytes)); + out.write(toWrite); } @Override @@ -157,17 +157,23 @@ public int getNumBytes() return numBytes; } - public long getSerializedSize() + @Override + public long getSerializedSize() throws IOException + { + return metaSerdeHelper.size(this) + buffer.remaining(); + } + + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - // version, numBytes, size, remaining - return 1 + 1 + 4 + buffer.remaining(); + metaSerdeHelper.writeTo(channel, this); + Channels.writeFully(channel, buffer.asReadOnlyBuffer()); } - public void writeToChannel(WritableByteChannel channel) throws IOException + @Override + public IndexedInts get() { - channel.write(ByteBuffer.wrap(new byte[]{VERSION, (byte) numBytes})); - channel.write(ByteBuffer.wrap(Ints.toByteArray(buffer.remaining()))); - channel.write(buffer.asReadOnlyBuffer()); + return this; } public static VSizeIndexedInts readFromByteBuffer(ByteBuffer buffer) @@ -200,37 +206,4 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("buffer", buffer); } - - public WritableSupplier asWritableSupplier() - { - return new VSizeIndexedIntsSupplier(this); - } - - public static class VSizeIndexedIntsSupplier implements WritableSupplier - { - final VSizeIndexedInts delegate; - - public VSizeIndexedIntsSupplier(VSizeIndexedInts delegate) - { - this.delegate = delegate; - } - - @Override - public long getSerializedSize() - { - return delegate.getSerializedSize(); - } - - @Override - public void writeToChannel(WritableByteChannel channel) throws IOException - { - delegate.writeToChannel(channel); - } - - @Override - public IndexedInts get() - { - return delegate; - } - } } diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexedIntsWriter.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexedIntsWriter.java index 624aa3d4e73e..edbaf2f4f6ca 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeIndexedIntsWriter.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexedIntsWriter.java @@ -19,16 +19,15 @@ package io.druid.segment.data; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; import com.google.common.primitives.Ints; -import io.druid.java.util.common.StringUtils; +import io.druid.common.utils.ByteUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.WriteOutBytes; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.serde.MetaSerdeHelper; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; /** @@ -38,61 +37,61 @@ public class VSizeIndexedIntsWriter extends SingleValueIndexedIntsWriter { private static final byte VERSION = VSizeIndexedInts.VERSION; - private final IOPeon ioPeon; - private final String valueFileName; + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((VSizeIndexedIntsWriter x) -> VERSION) + .writeByte(x -> ByteUtils.checkedCast(x.numBytes)) + .writeInt(x -> Ints.checkedCast(x.valuesOut.size())); + + private final SegmentWriteOutMedium segmentWriteOutMedium; private final int numBytes; - private CountingOutputStream valuesOut = null; private final ByteBuffer helperBuffer = ByteBuffer.allocate(Ints.BYTES); + private WriteOutBytes valuesOut = null; + private boolean bufPaddingWritten = false; - public VSizeIndexedIntsWriter( - final IOPeon ioPeon, - final String filenameBase, - final int maxValue - ) + public VSizeIndexedIntsWriter(final SegmentWriteOutMedium segmentWriteOutMedium, final int maxValue) { - this.ioPeon = ioPeon; - this.valueFileName = StringUtils.format("%s.values", filenameBase); + this.segmentWriteOutMedium = segmentWriteOutMedium; this.numBytes = VSizeIndexedInts.getNumBytesForMax(maxValue); } @Override public void open() throws IOException { - valuesOut = new CountingOutputStream(ioPeon.makeOutputStream(valueFileName)); + valuesOut = segmentWriteOutMedium.makeWriteOutBytes(); } @Override protected void addValue(int val) throws IOException { + if (bufPaddingWritten) { + throw new IllegalStateException("written out already"); + } helperBuffer.putInt(0, val); valuesOut.write(helperBuffer.array(), Ints.BYTES - numBytes, numBytes); } @Override - public void close() throws IOException + public long getSerializedSize() throws IOException { - byte[] bufPadding = new byte[4 - numBytes]; - valuesOut.write(bufPadding); - valuesOut.close(); + writeBufPadding(); + return metaSerdeHelper.size(this) + valuesOut.size(); } @Override - public long getSerializedSize() + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - return 2 + // version and numBytes - 4 + // dataLen - valuesOut.getCount(); + writeBufPadding(); + metaSerdeHelper.writeTo(channel, this); + valuesOut.writeTo(channel); } - @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + private void writeBufPadding() throws IOException { - long numBytesWritten = valuesOut.getCount(); - channel.write(ByteBuffer.wrap(new byte[]{VERSION, (byte) numBytes})); - channel.write(ByteBuffer.wrap(Ints.toByteArray((int) numBytesWritten))); - try (final ReadableByteChannel from = Channels.newChannel(ioPeon.makeInputStream(valueFileName))) { - ByteStreams.copy(from, channel); + if (!bufPaddingWritten) { + byte[] bufPadding = new byte[Ints.BYTES - numBytes]; + valuesOut.write(bufPadding); + bufPaddingWritten = true; } } } diff --git a/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java index 732b3722ad50..7f4f1d4e1f2e 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java @@ -19,152 +19,145 @@ package io.druid.segment.data; -import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; -import com.google.common.io.ByteStreams; -import com.google.common.io.Closeables; -import com.google.common.io.CountingOutputStream; -import com.google.common.io.InputSupplier; import com.google.common.primitives.Ints; -import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.WriteOutBytes; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.serde.MetaSerdeHelper; +import it.unimi.dsi.fastutil.ints.IntList; -import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; -import java.util.Arrays; -import java.util.List; /** * Streams arrays of objects out in the binary format described by VSizeIndexed */ -public class VSizeIndexedWriter extends MultiValueIndexedIntsWriter implements Closeable +public class VSizeIndexedWriter extends MultiValueIndexedIntsWriter { private static final byte VERSION = 0x1; - private static final byte[] EMPTY_ARRAY = new byte[]{}; + + private static final MetaSerdeHelper metaSerdeHelper = MetaSerdeHelper + .firstWriteByte((VSizeIndexedWriter x) -> VERSION) + .writeByte(x -> VSizeIndexedInts.getNumBytesForMax(x.maxId)) + .writeInt(x -> Ints.checkedCast(x.headerOut.size() + x.valuesOut.size() + Integer.BYTES)) + .writeInt(x -> x.numWritten); + + private enum WriteInt + { + ONE_BYTE { + @Override + void write(WriteOutBytes out, int v) throws IOException + { + out.write(v); + } + }, + TWO_BYTES { + @Override + void write(WriteOutBytes out, int v) throws IOException + { + out.write(v >> 8); + out.write(v); + } + }, + THREE_BYTES { + @Override + void write(WriteOutBytes out, int v) throws IOException + { + out.write(v >> 16); + out.write(v >> 8); + out.write(v); + } + }, + FOUR_BYTES { + @Override + void write(WriteOutBytes out, int v) throws IOException + { + out.writeInt(v); + } + }; + + abstract void write(WriteOutBytes out, int v) throws IOException; + } private final int maxId; + private final WriteInt writeInt; - private CountingOutputStream headerOut = null; - private CountingOutputStream valuesOut = null; - int numWritten = 0; - private final IOPeon ioPeon; - private final String metaFileName; - private final String headerFileName; - private final String valuesFileName; - - public VSizeIndexedWriter( - IOPeon ioPeon, - String filenameBase, - int maxId - ) + private final SegmentWriteOutMedium segmentWriteOutMedium; + private WriteOutBytes headerOut = null; + private WriteOutBytes valuesOut = null; + private int numWritten = 0; + private boolean numBytesForMaxWritten = false; + + public VSizeIndexedWriter(SegmentWriteOutMedium segmentWriteOutMedium, int maxId) { - this.ioPeon = ioPeon; - this.metaFileName = StringUtils.format("%s.meta", filenameBase); - this.headerFileName = StringUtils.format("%s.header", filenameBase); - this.valuesFileName = StringUtils.format("%s.values", filenameBase); + this.segmentWriteOutMedium = segmentWriteOutMedium; this.maxId = maxId; + this.writeInt = WriteInt.values()[VSizeIndexedInts.getNumBytesForMax(maxId) - 1]; } @Override public void open() throws IOException { - headerOut = new CountingOutputStream(ioPeon.makeOutputStream(headerFileName)); - valuesOut = new CountingOutputStream(ioPeon.makeOutputStream(valuesFileName)); + headerOut = segmentWriteOutMedium.makeWriteOutBytes(); + valuesOut = segmentWriteOutMedium.makeWriteOutBytes(); } @Override - protected void addValues(List val) throws IOException + protected void addValues(IntList ints) throws IOException { - write(val); + if (numBytesForMaxWritten) { + throw new IllegalStateException("written out already"); + } + if (ints != null) { + for (int i = 0; i < ints.size(); i++) { + int value = ints.getInt(i); + Preconditions.checkState(value >= 0 && value <= maxId); + writeInt.write(valuesOut, value); + } + } + headerOut.writeInt(Ints.checkedCast(valuesOut.size())); + ++numWritten; } - public void write(List ints) throws IOException + @Override + public long getSerializedSize() throws IOException { - byte[] bytesToWrite = ints == null ? EMPTY_ARRAY : VSizeIndexedInts.getBytesNoPaddingFromList(ints, maxId); - - valuesOut.write(bytesToWrite); - - headerOut.write(Ints.toByteArray((int) valuesOut.getCount())); - - ++numWritten; + writeNumBytesForMax(); + return metaSerdeHelper.size(this) + headerOut.size() + valuesOut.size(); } @Override - public void close() throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - final byte numBytesForMax = VSizeIndexedInts.getNumBytesForMax(maxId); + writeNumBytesForMax(); - valuesOut.write(new byte[4 - numBytesForMax]); - - Closeables.close(headerOut, false); - Closeables.close(valuesOut, false); - - final long numBytesWritten = headerOut.getCount() + valuesOut.getCount(); + final long numBytesWritten = headerOut.size() + valuesOut.size(); Preconditions.checkState( - headerOut.getCount() == (numWritten * 4), + headerOut.size() == (numWritten * 4), "numWritten[%s] number of rows should have [%s] bytes written to headerOut, had[%s]", numWritten, numWritten * 4, - headerOut.getCount() + headerOut.size() ); Preconditions.checkState( - numBytesWritten < Integer.MAX_VALUE, "Wrote[%s] bytes, which is too many.", numBytesWritten - ); - - try (OutputStream metaOut = ioPeon.makeOutputStream(metaFileName)) { - metaOut.write(new byte[]{VERSION, numBytesForMax}); - metaOut.write(Ints.toByteArray((int) numBytesWritten + 4)); - metaOut.write(Ints.toByteArray(numWritten)); - } - } - - public InputSupplier combineStreams() - { - return ByteStreams.join( - Iterables.transform( - Arrays.asList(metaFileName, headerFileName, valuesFileName), - new Function>() - { - @Override - public InputSupplier apply(final String input) - { - return new InputSupplier() - { - @Override - public InputStream getInput() throws IOException - { - return ioPeon.makeInputStream(input); - } - }; - } - } - ) + numBytesWritten < Integer.MAX_VALUE - Integer.BYTES, + "Wrote[%s] bytes, which is too many.", + numBytesWritten ); - } - @Override - public long getSerializedSize() - { - return 1 + // version - 1 + // numBytes - 4 + // numBytesWritten - 4 + // numElements - headerOut.getCount() + - valuesOut.getCount(); + metaSerdeHelper.writeTo(channel, this); + headerOut.writeTo(channel); + valuesOut.writeTo(channel); } - @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + private void writeNumBytesForMax() throws IOException { - try (final ReadableByteChannel from = Channels.newChannel(combineStreams().getInput())) { - ByteStreams.copy(from, channel); + if (!numBytesForMaxWritten) { + final byte numBytesForMax = VSizeIndexedInts.getNumBytesForMax(maxId); + valuesOut.write(new byte[4 - numBytesForMax]); + numBytesForMaxWritten = true; } } } diff --git a/processing/src/main/java/io/druid/segment/data/VSizeLongSerde.java b/processing/src/main/java/io/druid/segment/data/VSizeLongSerde.java index 585f949724c2..4ca906b398ad 100644 --- a/processing/src/main/java/io/druid/segment/data/VSizeLongSerde.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeLongSerde.java @@ -191,6 +191,7 @@ private static final class Size1Ser implements LongSerializer ByteBuffer buffer; byte curByte = 0; int count = 0; + private boolean closed = false; public Size1Ser(OutputStream output) { @@ -222,6 +223,9 @@ public void write(long value) throws IOException @Override public void close() throws IOException { + if (closed) { + return; + } buffer.put((byte) (curByte << (8 - count))); if (output != null) { output.write(buffer.array()); @@ -230,6 +234,7 @@ public void close() throws IOException } else { buffer.putInt(0); } + closed = true; } } @@ -239,6 +244,7 @@ private static final class Size2Ser implements LongSerializer ByteBuffer buffer; byte curByte = 0; int count = 0; + private boolean closed = false; public Size2Ser(OutputStream output) { @@ -270,6 +276,9 @@ public void write(long value) throws IOException @Override public void close() throws IOException { + if (closed) { + return; + } buffer.put((byte) (curByte << (8 - count))); if (output != null) { output.write(buffer.array()); @@ -278,6 +287,7 @@ public void close() throws IOException } else { buffer.putInt(0); } + closed = true; } } @@ -289,6 +299,7 @@ private static final class Mult4Ser implements LongSerializer int numBytes; byte curByte = 0; boolean first = true; + private boolean closed = false; public Mult4Ser(OutputStream output, int numBytes) { @@ -329,6 +340,9 @@ public void write(long value) throws IOException @Override public void close() throws IOException { + if (closed) { + return; + } if (!first) { buffer.put((byte) (curByte << 4)); } @@ -339,6 +353,7 @@ public void close() throws IOException } else { buffer.putInt(0); } + closed = true; } } @@ -347,6 +362,7 @@ private static final class Mult8Ser implements LongSerializer OutputStream output; ByteBuffer buffer; int numBytes; + private boolean closed = false; public Mult8Ser(OutputStream output, int numBytes) { @@ -377,12 +393,16 @@ public void write(long value) throws IOException @Override public void close() throws IOException { + if (closed) { + return; + } if (output != null) { output.write(EMPTY); output.flush(); } else { buffer.putInt(0); } + closed = true; } } diff --git a/processing/src/main/java/io/druid/segment/data/WritableSupplier.java b/processing/src/main/java/io/druid/segment/data/WritableSupplier.java index 5e704aa37ed3..49bffd57924f 100644 --- a/processing/src/main/java/io/druid/segment/data/WritableSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/WritableSupplier.java @@ -20,12 +20,8 @@ package io.druid.segment.data; import com.google.common.base.Supplier; +import io.druid.segment.serde.Serializer; -import java.io.IOException; -import java.nio.channels.WritableByteChannel; - -public interface WritableSupplier extends Supplier +public interface WritableSupplier extends Supplier, Serializer { - long getSerializedSize(); - void writeToChannel(WritableByteChannel channel) throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 47873be3eb85..2b73d8f30e22 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -447,7 +447,8 @@ protected abstract Integer addToFacts( AtomicInteger numEntries, TimeAndDims key, ThreadLocal rowContainer, - Supplier rowSupplier + Supplier rowSupplier, + boolean skipMaxRowsInMemoryCheck ) throws IndexSizeExceededException; public abstract int getLastRowIndex(); @@ -502,6 +503,11 @@ public Map getColumnCapabilities() * @return the number of rows in the data set after adding the InputRow */ public int add(InputRow row) throws IndexSizeExceededException + { + return add(row, false); + } + + public int add(InputRow row, boolean skipMaxRowsInMemoryCheck) throws IndexSizeExceededException { TimeAndDims key = toTimeAndDims(row); final int rv = addToFacts( @@ -512,14 +518,15 @@ public int add(InputRow row) throws IndexSizeExceededException numEntries, key, in, - rowSupplier + rowSupplier, + skipMaxRowsInMemoryCheck ); updateMaxIngestedTime(row.getTimestamp()); return rv; } @VisibleForTesting - TimeAndDims toTimeAndDims(InputRow row) throws IndexSizeExceededException + TimeAndDims toTimeAndDims(InputRow row) { row = formatRow(row); if (row.getTimestampFromEpoch() < minTimestamp) { diff --git a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java index 50ed19a9fbb2..d377634e0bc8 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java @@ -146,7 +146,8 @@ protected Integer addToFacts( AtomicInteger numEntries, TimeAndDims key, ThreadLocal rowContainer, - Supplier rowSupplier + Supplier rowSupplier, + boolean skipMaxRowsInMemoryCheck // ignored, we always want to check this for offheap ) throws IndexSizeExceededException { ByteBuffer aggBuffer; diff --git a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java index 5467877fc8aa..1a377ea826be 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java @@ -109,7 +109,8 @@ protected Integer addToFacts( AtomicInteger numEntries, TimeAndDims key, ThreadLocal rowContainer, - Supplier rowSupplier + Supplier rowSupplier, + boolean skipMaxRowsInMemoryCheck ) throws IndexSizeExceededException { final int priorIndex = facts.getPriorIndex(key); @@ -128,7 +129,9 @@ protected Integer addToFacts( concurrentSet(rowIndex, aggs); // Last ditch sanity checks - if (numEntries.get() >= maxRowCount && facts.getPriorIndex(key) == TimeAndDims.EMPTY_ROW_INDEX) { + if (numEntries.get() >= maxRowCount + && facts.getPriorIndex(key) == TimeAndDims.EMPTY_ROW_INDEX + && !skipMaxRowsInMemoryCheck) { throw new IndexSizeExceededException("Maximum number of rows [%d] reached", maxRowCount); } final int prev = facts.putIfAbsent(key, rowIndex); diff --git a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java index dc6ac14c007a..6662e60269bc 100644 --- a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java @@ -21,13 +21,10 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; -import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; /** */ @@ -48,13 +45,6 @@ public interface ColumnPartSerde Deserializer getDeserializer(); - interface Serializer - { - long numBytes(); - - void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOException; - } - interface Deserializer { void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig); diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java index 2b38f5ae70d5..432e1bf7a390 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java @@ -21,14 +21,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; -import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; /** */ @@ -103,23 +100,7 @@ public SerializerBuilder withDelegate(final GenericColumnSerializer delegate) public ComplexColumnPartSerde build() { - return new ComplexColumnPartSerde( - typeName, - new Serializer() - { - @Override - public long numBytes() - { - return delegate.getSerializedSize(); - } - - @Override - public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOException - { - delegate.writeToChannel(channel, smoosher); - } - } - ); + return new ComplexColumnPartSerde(typeName, delegate); } } } diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java index a76878491009..97798eaeddcf 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnSerializer.java @@ -22,9 +22,9 @@ import io.druid.guice.annotations.PublicApi; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.data.GenericIndexedWriter; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; import java.io.IOException; @@ -32,38 +32,29 @@ public class ComplexColumnSerializer implements GenericColumnSerializer { - private final IOPeon ioPeon; + @PublicApi + public static ComplexColumnSerializer create(SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy) + { + return new ComplexColumnSerializer(segmentWriteOutMedium, filenameBase, strategy); + } + + private final SegmentWriteOutMedium segmentWriteOutMedium; private final String filenameBase; private final ObjectStrategy strategy; private GenericIndexedWriter writer; - public ComplexColumnSerializer( - IOPeon ioPeon, - String filenameBase, - ObjectStrategy strategy - ) + + private ComplexColumnSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy) { - this.ioPeon = ioPeon; + this.segmentWriteOutMedium = segmentWriteOutMedium; this.filenameBase = filenameBase; this.strategy = strategy; } - @PublicApi - public static ComplexColumnSerializer create( - IOPeon ioPeon, - String filenameBase, - ObjectStrategy strategy - ) - { - return new ComplexColumnSerializer(ioPeon, filenameBase, strategy); - } - @SuppressWarnings(value = "unchecked") @Override public void open() throws IOException { - writer = new GenericIndexedWriter( - ioPeon, StringUtils.format("%s.complex_column", filenameBase), strategy - ); + writer = new GenericIndexedWriter(segmentWriteOutMedium, StringUtils.format("%s.complex_column", filenameBase), strategy); writer.open(); } @@ -75,28 +66,20 @@ public void serialize(Object obj) throws IOException } @Override - public void close() throws IOException - { - writer.close(); - } - - @Override - public long getSerializedSize() + public long getSerializedSize() throws IOException { return writer.getSerializedSize(); } @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - writeToChannelVersionOne(channel); + writeToVersionOne(channel); } - private void writeToChannelVersionOne(WritableByteChannel channel) throws IOException + private void writeToVersionOne(WritableByteChannel channel) throws IOException { - writer.writeToChannel( - channel, - null - ); //null for the FileSmoosher means that we default to "version 1" of GenericIndexed. + //null for the FileSmoosher means that we default to "version 1" of GenericIndexed. + writer.writeTo(channel, null); } } diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index e0b2aaa662d8..dc3c6647f3a1 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -21,9 +21,9 @@ import com.google.common.base.Function; import io.druid.guice.annotations.ExtensionPoint; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; import java.nio.ByteBuffer; @@ -108,12 +108,10 @@ public Object fromBytes(byte[] data, int start, int numBytes) * For large column (i.e columns greater than Integer.MAX) use * (@link LargeColumnSupportedComplexColumnSerializer) * - * @param peon IOPeon - * @param column name of the column * @return an instance of GenericColumnSerializer used for serialization. */ - public GenericColumnSerializer getSerializer(IOPeon peon, String column) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { - return ComplexColumnSerializer.create(peon, column, this.getObjectStrategy()); + return ComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } } diff --git a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 319fd550b2f4..c77078efb41f 100644 --- a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -25,9 +25,9 @@ import com.google.common.primitives.Ints; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.spatial.ImmutableRTree; +import io.druid.io.Channels; import io.druid.java.util.common.IAE; import io.druid.java.util.common.io.smoosh.FileSmoosher; -import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.segment.CompressedVSizeIndexedSupplier; import io.druid.segment.CompressedVSizeIndexedV3Supplier; import io.druid.segment.column.ColumnBuilder; @@ -211,7 +211,7 @@ public DictionaryEncodedColumnPartSerde build() new Serializer() { @Override - public long numBytes() + public long getSerializedSize() throws IOException { long size = 1 + // version (version.compareTo(VERSION.COMPRESSED) >= 0 @@ -233,23 +233,23 @@ public long numBytes() } @Override - public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - channel.write(ByteBuffer.wrap(new byte[]{version.asByte()})); + Channels.writeFully(channel, ByteBuffer.wrap(new byte[]{version.asByte()})); if (version.compareTo(VERSION.COMPRESSED) >= 0) { channel.write(ByteBuffer.wrap(Ints.toByteArray(flags))); } if (dictionaryWriter != null) { - dictionaryWriter.writeToChannel(channel, smoosher); + dictionaryWriter.writeTo(channel, smoosher); } if (valueWriter != null) { - valueWriter.writeToChannel(channel, smoosher); + valueWriter.writeTo(channel, smoosher); } if (bitmapIndexWriter != null) { - bitmapIndexWriter.writeToChannel(channel, smoosher); + bitmapIndexWriter.writeTo(channel, smoosher); } if (spatialIndexWriter != null) { - spatialIndexWriter.writeToChannel(channel); + spatialIndexWriter.writeTo(channel, smoosher); } } } @@ -295,10 +295,10 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo final WritableSupplier> rMultiValuedColumn; if (hasMultipleValues) { - rMultiValuedColumn = readMultiValuedColumn(rVersion, buffer, rFlags, builder.getFileMapper()); + rMultiValuedColumn = readMultiValuedColumn(rVersion, buffer, rFlags); rSingleValuedColumn = null; } else { - rSingleValuedColumn = readSingleValuedColumn(rVersion, buffer, builder.getFileMapper()); + rSingleValuedColumn = readSingleValuedColumn(rVersion, buffer); rMultiValuedColumn = null; } @@ -324,42 +324,39 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo ImmutableRTree rSpatialIndex = null; if (buffer.hasRemaining()) { rSpatialIndex = ByteBufferSerializer.read( - buffer, new ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()) + buffer, + new ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()) ); builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(rSpatialIndex)); } } - private WritableSupplier readSingleValuedColumn( - VERSION version, - ByteBuffer buffer, - SmooshedFileMapper fileMapper - ) + private WritableSupplier readSingleValuedColumn(VERSION version, ByteBuffer buffer) { switch (version) { case UNCOMPRESSED_SINGLE_VALUE: - return VSizeIndexedInts.readFromByteBuffer(buffer).asWritableSupplier(); + return VSizeIndexedInts.readFromByteBuffer(buffer); case COMPRESSED: - return CompressedVSizeIntsIndexedSupplier.fromByteBuffer(buffer, byteOrder, fileMapper); + return CompressedVSizeIntsIndexedSupplier.fromByteBuffer(buffer, byteOrder); default: throw new IAE("Unsupported single-value version[%s]", version); } } private WritableSupplier> readMultiValuedColumn( - VERSION version, ByteBuffer buffer, int flags, SmooshedFileMapper fileMapper + VERSION version, ByteBuffer buffer, int flags ) { switch (version) { case UNCOMPRESSED_MULTI_VALUE: { - return VSizeIndexed.readFromByteBuffer(buffer).asWritableSupplier(); + return VSizeIndexed.readFromByteBuffer(buffer); } case COMPRESSED: { if (Feature.MULTI_VALUE.isSet(flags)) { - return CompressedVSizeIndexedSupplier.fromByteBuffer(buffer, byteOrder, fileMapper); + return CompressedVSizeIndexedSupplier.fromByteBuffer(buffer, byteOrder); } else if (Feature.MULTI_VALUE_V3.isSet(flags)) { - return CompressedVSizeIndexedV3Supplier.fromByteBuffer(buffer, byteOrder, fileMapper); + return CompressedVSizeIndexedV3Supplier.fromByteBuffer(buffer, byteOrder); } else { throw new IAE("Unrecognized multi-value flag[%d]", flags); } diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java index 4127b01f3bb9..cf2c908986e0 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java @@ -19,26 +19,22 @@ package io.druid.segment.serde; - import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Supplier; -import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.DoubleColumnSerializer; import io.druid.segment.IndexIO; +import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.column.ColumnConfig; import io.druid.segment.column.ValueType; import io.druid.segment.data.CompressedDoublesIndexedSupplier; import io.druid.segment.data.IndexedDoubles; -import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.WritableByteChannel; public class DoubleGenericColumnPartSerde implements ColumnPartSerde { - private final ByteOrder byteOrder; - private Serializer serialize; - @JsonCreator public static DoubleGenericColumnPartSerde getDoubleGenericColumnPartSerde( @JsonProperty("byteOrder") ByteOrder byteOrder @@ -47,43 +43,19 @@ public static DoubleGenericColumnPartSerde getDoubleGenericColumnPartSerde( return new DoubleGenericColumnPartSerde(byteOrder, null); } - @JsonProperty - public ByteOrder getByteOrder() - { - return byteOrder; - } - + private final ByteOrder byteOrder; + private final Serializer serializer; - public DoubleGenericColumnPartSerde(ByteOrder byteOrder, Serializer serialize) + private DoubleGenericColumnPartSerde(ByteOrder byteOrder, Serializer serializer) { this.byteOrder = byteOrder; - this.serialize = serialize; - } - - @Override - public Serializer getSerializer() - { - return serialize; + this.serializer = serializer; } - @Override - public Deserializer getDeserializer() + @JsonProperty + public ByteOrder getByteOrder() { - return (buffer, builder, columnConfig) -> { - final Supplier column = CompressedDoublesIndexedSupplier.fromByteBuffer( - buffer, - byteOrder, - builder.getFileMapper() - ); - builder.setType(ValueType.DOUBLE) - .setHasMultipleValues(false) - .setGenericColumn( - new DoubleGenericColumnSupplier( - column, - IndexIO.LEGACY_FACTORY.getBitmapFactory() - .makeEmptyImmutableBitmap() - )); - }; + return byteOrder; } public static SerializerBuilder serializerBuilder() @@ -110,23 +82,34 @@ public SerializerBuilder withDelegate(final DoubleColumnSerializer delegate) public DoubleGenericColumnPartSerde build() { - return new DoubleGenericColumnPartSerde( - byteOrder, - new Serializer() - { - @Override - public long numBytes() - { - return delegate.getSerializedSize(); - } - - @Override - public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException - { - delegate.writeToChannel(channel, fileSmoosher); - } - } - ); + return new DoubleGenericColumnPartSerde(byteOrder, delegate); } } + + @Override + public Serializer getSerializer() + { + return serializer; + } + + @Override + public Deserializer getDeserializer() + { + return new Deserializer() + { + @Override + public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) + { + final Supplier column = CompressedDoublesIndexedSupplier.fromByteBuffer( + buffer, + byteOrder + ); + builder.setType(ValueType.DOUBLE) + .setHasMultipleValues(false) + .setGenericColumn(new DoubleGenericColumnSupplier(column, IndexIO.LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap())); + + } + }; + } } diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java index 60c559924522..d39c0d347b8c 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -94,8 +94,7 @@ public Deserializer getDeserializer() int initialPos = buffer.position(); final Supplier column = CompressedDoublesIndexedSupplier.fromByteBuffer( buffer, - byteOrder, - builder.getFileMapper() + byteOrder ); buffer.position(initialPos + offset); @@ -148,15 +147,15 @@ public DoubleGenericColumnPartSerdeV2 build() new Serializer() { @Override - public long numBytes() + public long getSerializedSize() throws IOException { return delegate.getSerializedSize(); } @Override - public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException { - delegate.writeToChannel(channel, fileSmoosher); + delegate.writeTo(channel, fileSmoosher); } } ); diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java index cbbde5743965..b6e0cde8de00 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.FloatColumnSerializer; import io.druid.segment.IndexIO; import io.druid.segment.column.ColumnBuilder; @@ -29,10 +28,8 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.CompressedFloatsIndexedSupplier; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.WritableByteChannel; /** */ @@ -47,7 +44,7 @@ public static FloatGenericColumnPartSerde createDeserializer( } private final ByteOrder byteOrder; - private Serializer serializer; + private final Serializer serializer; private FloatGenericColumnPartSerde(ByteOrder byteOrder, Serializer serializer) { @@ -85,23 +82,7 @@ public SerializerBuilder withDelegate(final FloatColumnSerializer delegate) public FloatGenericColumnPartSerde build() { - return new FloatGenericColumnPartSerde( - byteOrder, - new Serializer() - { - @Override - public long numBytes() - { - return delegate.getSerializedSize(); - } - - @Override - public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException - { - delegate.writeToChannel(channel, fileSmoosher); - } - } - ); + return new FloatGenericColumnPartSerde(byteOrder, delegate); } } @@ -121,8 +102,7 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo { final CompressedFloatsIndexedSupplier column = CompressedFloatsIndexedSupplier.fromByteBuffer( buffer, - byteOrder, - builder.getFileMapper() + byteOrder ); builder.setType(ValueType.FLOAT) .setHasMultipleValues(false) diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java index 2eb0c77f43ef..089efd183804 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -115,15 +115,15 @@ public FloatGenericColumnPartSerdeV2 build() new Serializer() { @Override - public long numBytes() + public long getSerializedSize() throws IOException { return delegate.getSerializedSize(); } @Override - public void write(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException { - delegate.writeToChannel(channel, fileSmoosher); + delegate.writeTo(channel, fileSmoosher); } } ); @@ -145,8 +145,7 @@ public Deserializer getDeserializer() int initialPos = buffer.position(); final CompressedFloatsIndexedSupplier column = CompressedFloatsIndexedSupplier.fromByteBuffer( buffer, - byteOrder, - builder.getFileMapper() + byteOrder ); buffer.position(initialPos + offset); final ImmutableBitmap bitmap; diff --git a/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java b/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java index f3c12d8a919f..6486f7eeaa06 100644 --- a/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializer.java @@ -22,9 +22,9 @@ import io.druid.guice.annotations.PublicApi; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.data.GenericIndexedWriter; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; import java.io.IOException; @@ -32,51 +32,52 @@ public class LargeColumnSupportedComplexColumnSerializer implements GenericColumnSerializer { - - private final IOPeon ioPeon; - private final String filenameBase; - private final ObjectStrategy strategy; - private final int columnSize; - private GenericIndexedWriter writer; - public LargeColumnSupportedComplexColumnSerializer( - IOPeon ioPeon, + @PublicApi + public static LargeColumnSupportedComplexColumnSerializer create( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy ) { - this(ioPeon, filenameBase, strategy, Integer.MAX_VALUE); + return new LargeColumnSupportedComplexColumnSerializer(segmentWriteOutMedium, filenameBase, strategy); } - public LargeColumnSupportedComplexColumnSerializer( - IOPeon ioPeon, + + public static LargeColumnSupportedComplexColumnSerializer createWithColumnSize( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy, int columnSize ) { - this.ioPeon = ioPeon; - this.filenameBase = filenameBase; - this.strategy = strategy; - this.columnSize = columnSize; + return new LargeColumnSupportedComplexColumnSerializer(segmentWriteOutMedium, filenameBase, strategy, columnSize); } - @PublicApi - public static LargeColumnSupportedComplexColumnSerializer create( - IOPeon ioPeon, + private final SegmentWriteOutMedium segmentWriteOutMedium; + private final String filenameBase; + private final ObjectStrategy strategy; + private final int columnSize; + private GenericIndexedWriter writer; + + private LargeColumnSupportedComplexColumnSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy ) { - return new LargeColumnSupportedComplexColumnSerializer(ioPeon, filenameBase, strategy); + this(segmentWriteOutMedium, filenameBase, strategy, Integer.MAX_VALUE); } - public static LargeColumnSupportedComplexColumnSerializer createWithColumnSize( - IOPeon ioPeon, + private LargeColumnSupportedComplexColumnSerializer( + SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ObjectStrategy strategy, int columnSize ) { - return new LargeColumnSupportedComplexColumnSerializer(ioPeon, filenameBase, strategy, columnSize); + this.segmentWriteOutMedium = segmentWriteOutMedium; + this.filenameBase = filenameBase; + this.strategy = strategy; + this.columnSize = columnSize; } @SuppressWarnings(value = "unchecked") @@ -84,7 +85,11 @@ public static LargeColumnSupportedComplexColumnSerializer createWithColumnSize( public void open() throws IOException { writer = new GenericIndexedWriter( - ioPeon, StringUtils.format("%s.complex_column", filenameBase), strategy, columnSize); + segmentWriteOutMedium, + StringUtils.format("%s.complex_column", filenameBase), + strategy, + columnSize + ); writer.open(); } @@ -96,21 +101,15 @@ public void serialize(Object obj) throws IOException } @Override - public void close() throws IOException - { - writer.close(); - } - - @Override - public long getSerializedSize() + public long getSerializedSize() throws IOException { return writer.getSerializedSize(); } @Override - public void writeToChannel(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - writer.writeToChannel(channel, smoosher); + writer.writeTo(channel, smoosher); } } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java index 72f98a1ab4bd..3e34caa27e3b 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.segment.IndexIO; import io.druid.segment.LongColumnSerializer; import io.druid.segment.column.ColumnBuilder; @@ -29,10 +28,8 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.CompressedLongsIndexedSupplier; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.WritableByteChannel; /** */ @@ -47,7 +44,7 @@ public static LongGenericColumnPartSerde createDeserializer( } private final ByteOrder byteOrder; - private Serializer serializer; + private final Serializer serializer; private LongGenericColumnPartSerde(ByteOrder byteOrder, Serializer serializer) { @@ -85,23 +82,7 @@ public SerializerBuilder withDelegate(final LongColumnSerializer delegate) public LongGenericColumnPartSerde build() { - return new LongGenericColumnPartSerde( - byteOrder, - new Serializer() - { - @Override - public long numBytes() - { - return delegate.getSerializedSize(); - } - - @Override - public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOException - { - delegate.writeToChannel(channel, smoosher); - } - } - ); + return new LongGenericColumnPartSerde(byteOrder, delegate); } } @@ -121,8 +102,7 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo { final CompressedLongsIndexedSupplier column = CompressedLongsIndexedSupplier.fromByteBuffer( buffer, - byteOrder, - builder.getFileMapper() + byteOrder ); builder.setType(ValueType.LONG) .setHasMultipleValues(false) diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java index c9b6485d0c1b..1846102c8e49 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -114,15 +114,15 @@ public LongGenericColumnPartSerdeV2 build() new Serializer() { @Override - public long numBytes() + public long getSerializedSize() throws IOException { return delegate.getSerializedSize(); } @Override - public void write(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { - delegate.writeToChannel(channel, smoosher); + delegate.writeTo(channel, smoosher); } } ); @@ -143,8 +143,7 @@ public Deserializer getDeserializer() int initialPos = buffer.position(); final CompressedLongsIndexedSupplier column = CompressedLongsIndexedSupplier.fromByteBuffer( buffer, - byteOrder, - builder.getFileMapper() + byteOrder ); buffer.position(initialPos + offset); final ImmutableBitmap bitmap; diff --git a/processing/src/main/java/io/druid/segment/serde/MetaSerdeHelper.java b/processing/src/main/java/io/druid/segment/serde/MetaSerdeHelper.java new file mode 100644 index 000000000000..0e8c6987d389 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/serde/MetaSerdeHelper.java @@ -0,0 +1,158 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import io.druid.io.Channels; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; + +public final class MetaSerdeHelper +{ + public static MetaSerdeHelper firstWriteByte(ByteFieldWriter fieldWriter) + { + return new MetaSerdeHelper().writeByte(fieldWriter); + } + + private final List> fieldWriters = new ArrayList<>(); + + private MetaSerdeHelper() {} + + public MetaSerdeHelper writeInt(IntFieldWriter fieldWriter) + { + return writeSomething(fieldWriter); + } + + public MetaSerdeHelper writeByte(ByteFieldWriter fieldWriter) + { + return writeSomething(fieldWriter); + } + + public MetaSerdeHelper maybeWriteByte(Predicate condition, ByteFieldWriter fieldWriter) + { + return writeSomething( + new FieldWriter() + { + @Override + public void writeTo(ByteBuffer buffer, T x) throws IOException + { + if (condition.test(x)) { + buffer.put(fieldWriter.getField(x)); + } + } + + @Override + public int size(T x) + { + return condition.test(x) ? Byte.BYTES : 0; + } + } + ); + } + + public MetaSerdeHelper writeByteArray(Function getByteArray) + { + return writeSomething( + new FieldWriter() + { + @Override + public void writeTo(ByteBuffer buffer, T x) throws IOException + { + buffer.put(getByteArray.apply(x)); + } + + @Override + public int size(T x) + { + return getByteArray.apply(x).length; + } + } + ); + } + + public MetaSerdeHelper writeSomething(FieldWriter fieldWriter) + { + fieldWriters.add(fieldWriter); + return this; + } + + public void writeTo(WritableByteChannel channel, T x) throws IOException + { + ByteBuffer meta = ByteBuffer.allocate(size(x)); + for (FieldWriter w : fieldWriters) { + w.writeTo(meta, x); + } + meta.flip(); + Channels.writeFully(channel, meta); + } + + public int size(T x) + { + return fieldWriters.stream().mapToInt(w -> w.size(x)).sum(); + } + + public interface FieldWriter + { + void writeTo(ByteBuffer buffer, T x) throws IOException; + + int size(T x); + } + + @FunctionalInterface + public interface IntFieldWriter extends FieldWriter + { + int getField(T x) throws IOException; + + @Override + default void writeTo(ByteBuffer buffer, T x) throws IOException + { + buffer.putInt(getField(x)); + } + + @Override + default int size(T x) + { + return Integer.BYTES; + } + } + + @FunctionalInterface + public interface ByteFieldWriter extends FieldWriter + { + byte getField(T x) throws IOException; + + @Override + default void writeTo(ByteBuffer buffer, T x) throws IOException + { + buffer.put(getField(x)); + } + + @Override + default int size(T x) + { + return Byte.BYTES; + } + } +} diff --git a/processing/src/main/java/io/druid/segment/serde/Serializer.java b/processing/src/main/java/io/druid/segment/serde/Serializer.java new file mode 100644 index 000000000000..5c20c11b5025 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/serde/Serializer.java @@ -0,0 +1,43 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import io.druid.java.util.common.io.smoosh.FileSmoosher; + +import java.io.IOException; +import java.nio.channels.WritableByteChannel; + +/** + * Main interface for "serializeable something" in Druid segment serialization. + */ +public interface Serializer +{ + /** + * Returns the number of bytes, that this Serializer will write to the output _channel_ (not smoosher) on a {@link + * #writeTo} call. + */ + long getSerializedSize() throws IOException; + + /** + * Writes serialized form of this object to the given channel. If parallel data streams are needed, they could be + * created with the provided smoosher. + */ + void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException; +} diff --git a/processing/src/main/java/io/druid/segment/transform/TransformingInputRowParser.java b/processing/src/main/java/io/druid/segment/transform/TransformingInputRowParser.java index d9801b3f2afa..d90e4ec65233 100644 --- a/processing/src/main/java/io/druid/segment/transform/TransformingInputRowParser.java +++ b/processing/src/main/java/io/druid/segment/transform/TransformingInputRowParser.java @@ -23,6 +23,9 @@ import io.druid.data.input.impl.InputRowParser; import io.druid.data.input.impl.ParseSpec; +import java.util.List; +import java.util.stream.Collectors; + public class TransformingInputRowParser implements InputRowParser { private final InputRowParser parser; @@ -42,9 +45,9 @@ public InputRowParser getParser() } @Override - public InputRow parse(final T row) + public List parseBatch(final T row) { - return transformer.transform(parser.parse(row)); + return parser.parseBatch(row).stream().map(transformer::transform).collect(Collectors.toList()); } @Override diff --git a/processing/src/main/java/io/druid/segment/transform/TransformingStringInputRowParser.java b/processing/src/main/java/io/druid/segment/transform/TransformingStringInputRowParser.java index 20e0430aab5a..9e6ba11a890d 100644 --- a/processing/src/main/java/io/druid/segment/transform/TransformingStringInputRowParser.java +++ b/processing/src/main/java/io/druid/segment/transform/TransformingStringInputRowParser.java @@ -25,6 +25,8 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; +import java.util.List; +import java.util.stream.Collectors; public class TransformingStringInputRowParser extends StringInputRowParser { @@ -43,9 +45,9 @@ public TransformingStringInputRowParser( } @Override - public InputRow parse(final ByteBuffer input) + public List parseBatch(final ByteBuffer input) { - return transformer.transform(super.parse(input)); + return super.parseBatch(input).stream().map(transformer::transform).collect(Collectors.toList()); } @Nullable diff --git a/processing/src/main/java/io/druid/segment/writeout/ByteBufferWriteOutBytes.java b/processing/src/main/java/io/druid/segment/writeout/ByteBufferWriteOutBytes.java new file mode 100644 index 000000000000..263c74679daa --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/ByteBufferWriteOutBytes.java @@ -0,0 +1,265 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import com.google.common.base.Preconditions; +import com.google.common.io.ByteSource; +import com.google.common.primitives.Ints; +import io.druid.io.ByteBufferInputStream; +import io.druid.io.Channels; +import io.druid.java.util.common.IAE; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; +import java.util.function.Function; +import java.util.stream.Collectors; + +public abstract class ByteBufferWriteOutBytes extends WriteOutBytes +{ + /** + * There is no reason why 64K is chosen. Other power of 2 between 4K and 64K could be more reasonable. + */ + static final int BUFFER_SIZE = 64 * 1024; + + final ArrayList buffers = new ArrayList<>(); + int headBufferIndex; + ByteBuffer headBuffer; + long size; + long capacity; + + ByteBufferWriteOutBytes() + { + size = 0; + headBufferIndex = 0; + headBuffer = allocateBuffer(); + buffers.add(headBuffer); + capacity = BUFFER_SIZE; + } + + @Override + public long size() + { + return size; + } + + protected abstract ByteBuffer allocateBuffer(); + + private void ensureCapacity(int len) + { + long remaining = capacity - size; + for (long toAllocate = len - remaining; toAllocate >= 0; toAllocate -= BUFFER_SIZE) { + buffers.add(allocateBuffer()); + capacity += BUFFER_SIZE; + } + if (headBuffer.remaining() == 0) { + nextHead(); + } + } + + private void nextHead() + { + headBufferIndex++; + headBuffer = buffers.get(headBufferIndex); + } + + @Override + public void write(int b) + { + checkOpen(); + if (headBuffer.remaining() == 0) { + ensureCapacity(1); + } + headBuffer.put((byte) b); + size += 1; + } + + @Override + public void writeInt(int v) + { + checkOpen(); + if (headBuffer.remaining() >= Ints.BYTES) { + headBuffer.putInt(v); + size += Ints.BYTES; + } else { + ensureCapacity(Ints.BYTES); + if (headBuffer.remaining() >= Ints.BYTES) { + headBuffer.putInt(v); + size += Ints.BYTES; + } else { + write(v >> 24); + write(v >> 16); + write(v >> 8); + write(v); + } + } + } + + @Override + public void write(byte[] b) throws IOException + { + write0(b, 0, b.length); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException + { + Preconditions.checkPositionIndexes(off, off + len, b.length); + write0(b, off, len); + } + + private void write0(byte[] b, int off, int len) + { + checkOpen(); + if (headBuffer.remaining() < len) { + ensureCapacity(len); + } + int headRemaining = headBuffer.remaining(); + if (len <= headRemaining) { + headBuffer.put(b, off, len); + } else { + headBuffer.put(b, off, headRemaining); + int bytesLeft = len - headRemaining; + off += headRemaining; + for (; bytesLeft > 0; bytesLeft -= BUFFER_SIZE, off += BUFFER_SIZE) { + nextHead(); + headBuffer.put(b, off, Math.min(BUFFER_SIZE, bytesLeft)); + } + } + size += len; + } + + @Override + public int write(ByteBuffer src) + { + checkOpen(); + int len = src.remaining(); + if (headBuffer.remaining() < len) { + ensureCapacity(len); + } + int headRemaining = headBuffer.remaining(); + src.limit(src.position() + Math.min(headRemaining, len)); + headBuffer.put(src); + for (int bytesLeft = len - headRemaining; bytesLeft > 0; bytesLeft -= BUFFER_SIZE) { + nextHead(); + src.limit(src.position() + Math.min(BUFFER_SIZE, bytesLeft)); + headBuffer.put(src); + } + size += len; + return len; + } + + @Override + public void writeTo(WritableByteChannel channel) throws IOException + { + checkOpen(); + for (int i = 0; i <= headBufferIndex; i++) { + ByteBuffer buffer = buffers.get(i); + buffer.flip(); + Channels.writeFully(channel, buffer); + // switch back to the initial state + buffer.limit(buffer.capacity()); + } + } + + /** + * Takes all bytes that are written to this WriteOutBytes so far and writes them into the given ByteBuffer. This method + * changes the position of the out buffer by the {@link #size()} of this WriteOutBytes. + * + * @throws java.nio.BufferOverflowException if the {@link ByteBuffer#remaining()} capacity of the given buffer is + * smaller than the size of this WriteOutBytes + */ + public void writeTo(ByteBuffer out) + { + checkOpen(); + for (int i = 0; i <= headBufferIndex; i++) { + ByteBuffer buffer = buffers.get(i); + buffer.flip(); + out.put(buffer); + // switch back to the initial state + buffer.limit(buffer.capacity()); + } + } + + @Override + public void readFully(long pos, ByteBuffer buffer) + { + checkOpen(); + if (pos < 0 || pos > size) { + throw new IAE("pos %d out of range [%d, %d]", pos, 0, size); + } + int ourBufferIndex = Ints.checkedCast(pos / BUFFER_SIZE); + int ourBufferOffset = Ints.checkedCast(pos % BUFFER_SIZE); + for (int bytesLeft = buffer.remaining(); bytesLeft > 0;) { + int bytesToWrite = Math.min(BUFFER_SIZE - ourBufferOffset, bytesLeft); + ByteBuffer ourBuffer = buffers.get(ourBufferIndex); + int ourBufferPosition = ourBuffer.position(); + if (bytesToWrite > ourBufferPosition - ourBufferOffset) { + throw new BufferUnderflowException(); + } + try { + ourBuffer.position(ourBufferOffset); + ourBuffer.limit(ourBufferOffset + bytesToWrite); + buffer.put(ourBuffer); + } + finally { + // switch back to the initial state + ourBuffer.limit(ourBuffer.capacity()); + ourBuffer.position(ourBufferPosition); + } + ourBufferIndex++; + ourBufferOffset = 0; + bytesLeft -= bytesToWrite; + } + } + + @Override + public InputStream asInputStream() throws IOException + { + checkOpen(); + Function byteBufferToByteSource = buf -> new ByteSource() + { + @Override + public InputStream openStream() + { + ByteBuffer inputBuf = buf.duplicate(); + inputBuf.flip(); + return new ByteBufferInputStream(inputBuf); + } + }; + return ByteSource.concat(buffers.stream().map(byteBufferToByteSource).collect(Collectors.toList())).openStream(); + } + + @Override + public boolean isOpen() + { + return true; + } + + private void checkOpen() + { + if (!isOpen()) { + throw new IllegalStateException(); + } + } +} diff --git a/processing/src/main/java/io/druid/segment/writeout/DirectByteBufferWriteOutBytes.java b/processing/src/main/java/io/druid/segment/writeout/DirectByteBufferWriteOutBytes.java new file mode 100644 index 000000000000..172c99afb78f --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/DirectByteBufferWriteOutBytes.java @@ -0,0 +1,52 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import io.druid.java.util.common.ByteBufferUtils; + +import java.nio.ByteBuffer; + +final class DirectByteBufferWriteOutBytes extends ByteBufferWriteOutBytes +{ + private boolean open = true; + + @Override + protected ByteBuffer allocateBuffer() + { + return ByteBuffer.allocateDirect(BUFFER_SIZE); + } + + @Override + public boolean isOpen() + { + return open; + } + + void free() + { + open = false; + buffers.forEach(ByteBufferUtils::free); + buffers.clear(); + headBufferIndex = -1; + headBuffer = null; + size = 0; + capacity = 0; + } +} diff --git a/processing/src/main/java/io/druid/segment/writeout/FileWriteOutBytes.java b/processing/src/main/java/io/druid/segment/writeout/FileWriteOutBytes.java new file mode 100644 index 000000000000..9844afd94970 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/FileWriteOutBytes.java @@ -0,0 +1,150 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import com.google.common.io.ByteStreams; +import io.druid.io.Channels; +import io.druid.java.util.common.IAE; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; + +final class FileWriteOutBytes extends WriteOutBytes +{ + private final File file; + private final FileChannel ch; + + /** Purposely big-endian, for {@link #writeInt(int)} implementation */ + private final ByteBuffer buffer = ByteBuffer.allocate(4096); // 4K page sized buffer + + FileWriteOutBytes(File file, FileChannel ch) + { + this.file = file; + this.ch = ch; + } + + private void flushIfNeeded(int bytesNeeded) throws IOException + { + if (buffer.remaining() < bytesNeeded) { + flush(); + } + } + + @Override + public void flush() throws IOException + { + buffer.flip(); + Channels.writeFully(ch, buffer); + buffer.clear(); + } + + @Override + public void write(int b) throws IOException + { + flushIfNeeded(1); + buffer.put((byte) b); + } + + @Override + public void writeInt(int v) throws IOException + { + flushIfNeeded(Integer.SIZE); + buffer.putInt(v); + } + + @Override + public int write(ByteBuffer src) throws IOException + { + int len = src.remaining(); + flushIfNeeded(len); + while (src.remaining() > buffer.capacity()) { + int srcLimit = src.limit(); + try { + src.limit(src.position() + buffer.capacity()); + buffer.put(src); + flush(); + } + finally { + // IOException may occur in flush(), reset src limit to the original + src.limit(srcLimit); + } + } + buffer.put(src); + return len; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException + { + write(ByteBuffer.wrap(b, off, len)); + } + + @Override + public long size() throws IOException + { + flush(); + return ch.size(); + } + + @Override + public void writeTo(WritableByteChannel channel) throws IOException + { + flush(); + ch.position(0); + try { + ByteStreams.copy(ch, channel); + } + finally { + ch.position(ch.size()); + } + } + + @Override + public void readFully(long pos, ByteBuffer buffer) throws IOException + { + flush(); + if (pos < 0 || pos > ch.size()) { + throw new IAE("pos %d out of range [%d, %d]", pos, 0, ch.size()); + } + ch.read(buffer, pos); + if (buffer.remaining() > 0) { + throw new BufferUnderflowException(); + } + } + + @Override + public InputStream asInputStream() throws IOException + { + flush(); + return new FileInputStream(file); + } + + @Override + public boolean isOpen() + { + return ch.isOpen(); + } +} diff --git a/processing/src/main/java/io/druid/segment/writeout/HeapByteBufferWriteOutBytes.java b/processing/src/main/java/io/druid/segment/writeout/HeapByteBufferWriteOutBytes.java new file mode 100644 index 000000000000..4823d49375d1 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/HeapByteBufferWriteOutBytes.java @@ -0,0 +1,31 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import java.nio.ByteBuffer; + +public final class HeapByteBufferWriteOutBytes extends ByteBufferWriteOutBytes +{ + @Override + protected ByteBuffer allocateBuffer() + { + return ByteBuffer.allocate(BUFFER_SIZE); + } +} diff --git a/processing/src/main/java/io/druid/segment/writeout/OffHeapMemorySegmentWriteOutMedium.java b/processing/src/main/java/io/druid/segment/writeout/OffHeapMemorySegmentWriteOutMedium.java new file mode 100644 index 000000000000..4a8974d129e5 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/OffHeapMemorySegmentWriteOutMedium.java @@ -0,0 +1,49 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import io.druid.java.util.common.io.Closer; + +import java.io.IOException; + +public final class OffHeapMemorySegmentWriteOutMedium implements SegmentWriteOutMedium +{ + private final Closer closer = Closer.create(); + + @Override + public WriteOutBytes makeWriteOutBytes() + { + DirectByteBufferWriteOutBytes writeOutBytes = new DirectByteBufferWriteOutBytes(); + closer.register(writeOutBytes::free); + return writeOutBytes; + } + + @Override + public Closer getCloser() + { + return closer; + } + + @Override + public void close() throws IOException + { + closer.close(); + } +} diff --git a/processing/src/main/java/io/druid/segment/writeout/OffHeapMemorySegmentWriteOutMediumFactory.java b/processing/src/main/java/io/druid/segment/writeout/OffHeapMemorySegmentWriteOutMediumFactory.java new file mode 100644 index 000000000000..d45c5b7908d6 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/OffHeapMemorySegmentWriteOutMediumFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import java.io.File; +import java.io.IOException; + +public class OffHeapMemorySegmentWriteOutMediumFactory implements SegmentWriteOutMediumFactory +{ + private static final OffHeapMemorySegmentWriteOutMediumFactory INSTANCE = + new OffHeapMemorySegmentWriteOutMediumFactory(); + + @JsonCreator + public static OffHeapMemorySegmentWriteOutMediumFactory instance() + { + return INSTANCE; + } + + private OffHeapMemorySegmentWriteOutMediumFactory() {} + + @Override + public SegmentWriteOutMedium makeSegmentWriteOutMedium(File outDir) throws IOException + { + return new OffHeapMemorySegmentWriteOutMedium(); + } +} diff --git a/processing/src/main/java/io/druid/segment/writeout/OnHeapMemorySegmentWriteOutMedium.java b/processing/src/main/java/io/druid/segment/writeout/OnHeapMemorySegmentWriteOutMedium.java new file mode 100644 index 000000000000..ab83db62a6ff --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/OnHeapMemorySegmentWriteOutMedium.java @@ -0,0 +1,49 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import com.google.common.annotations.VisibleForTesting; +import io.druid.java.util.common.io.Closer; + +import java.io.IOException; + +@VisibleForTesting +public final class OnHeapMemorySegmentWriteOutMedium implements SegmentWriteOutMedium +{ + private final Closer closer = Closer.create(); + + @Override + public WriteOutBytes makeWriteOutBytes() throws IOException + { + return new HeapByteBufferWriteOutBytes(); + } + + @Override + public Closer getCloser() + { + return closer; + } + + @Override + public void close() throws IOException + { + closer.close(); + } +} diff --git a/processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMedium.java b/processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMedium.java new file mode 100644 index 000000000000..4d16b7695bbc --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMedium.java @@ -0,0 +1,47 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import io.druid.java.util.common.io.Closer; + +import java.io.Closeable; +import java.io.IOException; + +/** + * SegmentWriteOutMedium is an umbrella "resource disposer" for temporary buffers (in the form of {@link WriteOutBytes}, + * obtained by calling {@link #makeWriteOutBytes()} on the SegmentWriteOutMedium instance), that are used during new Druid + * segment creation, and other resources (see {@link #getCloser()}). + * + * When SegmentWriteOutMedium is closed, all child WriteOutBytes couldn't be used anymore. + */ +public interface SegmentWriteOutMedium extends Closeable +{ + /** + * Creates a new empty {@link WriteOutBytes}, attached to this SegmentWriteOutMedium. When this SegmentWriteOutMedium is + * closed, the returned WriteOutBytes couldn't be used anymore. + */ + WriteOutBytes makeWriteOutBytes() throws IOException; + + /** + * Returns a closer of this SegmentWriteOutMedium, which is closed in this SegmentWriteOutMedium's close() method. + * Could be used to "attach" some random resources to this SegmentWriteOutMedium, to be closed at the same time. + */ + Closer getCloser(); +} diff --git a/processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMediumFactory.java b/processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMediumFactory.java new file mode 100644 index 000000000000..29c606ba23a0 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMediumFactory.java @@ -0,0 +1,51 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.collect.ImmutableSet; + +import java.io.File; +import java.io.IOException; +import java.util.Set; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, defaultImpl = TmpFileSegmentWriteOutMediumFactory.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "tmpFile", value = TmpFileSegmentWriteOutMediumFactory.class), + @JsonSubTypes.Type(name = "offHeapMemory", value = OffHeapMemorySegmentWriteOutMediumFactory.class), +}) +public interface SegmentWriteOutMediumFactory +{ + static Set builtInFactories() + { + return ImmutableSet.of( + TmpFileSegmentWriteOutMediumFactory.instance(), + OffHeapMemorySegmentWriteOutMediumFactory.instance() + ); + } + + /** + * Creates a new SegmentWriteOutMedium. If this type of SegmentWriteOutMedium needs to create some temprorary files, + * it creates a *subdirectory* in the given outDir, stores the files there, and removes the files and the subdirectory + * when closed. + */ + SegmentWriteOutMedium makeSegmentWriteOutMedium(File outDir) throws IOException; +} diff --git a/processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMediumModule.java b/processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMediumModule.java new file mode 100644 index 000000000000..3f7a6fed2043 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/SegmentWriteOutMediumModule.java @@ -0,0 +1,37 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.guice.JsonConfigProvider; + +public class SegmentWriteOutMediumModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind( + binder, + "druid.peon.defaultSegmentWriteOutMediumFactory", + SegmentWriteOutMediumFactory.class + ); + } +} diff --git a/processing/src/main/java/io/druid/segment/writeout/TmpFileSegmentWriteOutMedium.java b/processing/src/main/java/io/druid/segment/writeout/TmpFileSegmentWriteOutMedium.java new file mode 100644 index 000000000000..b3ee365e7324 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/TmpFileSegmentWriteOutMedium.java @@ -0,0 +1,68 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import io.druid.java.util.common.io.Closer; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; + +public final class TmpFileSegmentWriteOutMedium implements SegmentWriteOutMedium +{ + private final File dir; + private final Closer closer = Closer.create(); + + TmpFileSegmentWriteOutMedium(File outDir) throws IOException + { + File tmpOutputFilesDir = new File(outDir, "tmpOutputFiles"); + FileUtils.forceMkdir(tmpOutputFilesDir); + closer.register(() -> FileUtils.deleteDirectory(tmpOutputFilesDir)); + this.dir = tmpOutputFilesDir; + } + + @Override + public WriteOutBytes makeWriteOutBytes() throws IOException + { + File file = File.createTempFile("filePeon", null, dir); + FileChannel ch = FileChannel.open( + file.toPath(), + StandardOpenOption.READ, + StandardOpenOption.WRITE + ); + closer.register(file::delete); + closer.register(ch); + return new FileWriteOutBytes(file, ch); + } + + @Override + public Closer getCloser() + { + return closer; + } + + @Override + public void close() throws IOException + { + closer.close(); + } +} diff --git a/processing/src/main/java/io/druid/segment/data/FixedSizeCompressedObjectStrategy.java b/processing/src/main/java/io/druid/segment/writeout/TmpFileSegmentWriteOutMediumFactory.java similarity index 54% rename from processing/src/main/java/io/druid/segment/data/FixedSizeCompressedObjectStrategy.java rename to processing/src/main/java/io/druid/segment/writeout/TmpFileSegmentWriteOutMediumFactory.java index afdd1fc99a38..7fce54f07aeb 100644 --- a/processing/src/main/java/io/druid/segment/data/FixedSizeCompressedObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/writeout/TmpFileSegmentWriteOutMediumFactory.java @@ -17,41 +17,28 @@ * under the License. */ -package io.druid.segment.data; +package io.druid.segment.writeout; -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; +import com.fasterxml.jackson.annotation.JsonCreator; -public abstract class FixedSizeCompressedObjectStrategy extends CompressedObjectStrategy -{ - private final int sizePer; +import java.io.File; +import java.io.IOException; - protected FixedSizeCompressedObjectStrategy( - ByteOrder order, - BufferConverter converter, - CompressionStrategy compression, - int sizePer - ) - { - super(order, converter, compression); - this.sizePer = sizePer; - } +public final class TmpFileSegmentWriteOutMediumFactory implements SegmentWriteOutMediumFactory +{ + private static final TmpFileSegmentWriteOutMediumFactory INSTANCE = new TmpFileSegmentWriteOutMediumFactory(); - public int getSize() + @JsonCreator + public static TmpFileSegmentWriteOutMediumFactory instance() { - return sizePer; + return INSTANCE; } - @Override - protected ByteBuffer bufferFor(T val) - { - return ByteBuffer.allocate(converter.sizeOf(getSize())).order(order); - } + private TmpFileSegmentWriteOutMediumFactory() {} @Override - protected void decompress(ByteBuffer buffer, int numBytes, ByteBuffer buf) + public SegmentWriteOutMedium makeSegmentWriteOutMedium(File outDir) throws IOException { - decompressor.decompress(buffer, numBytes, buf, converter.sizeOf(getSize())); + return new TmpFileSegmentWriteOutMedium(outDir); } } diff --git a/processing/src/main/java/io/druid/segment/writeout/WriteOutBytes.java b/processing/src/main/java/io/druid/segment/writeout/WriteOutBytes.java new file mode 100644 index 000000000000..312341ade448 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/writeout/WriteOutBytes.java @@ -0,0 +1,80 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; + +/** + * Appendable byte sequence for temporary storage. Methods inherited from {@link OutputStream}, {@link + * WritableByteChannel} and {@link #writeInt(int)} append to the sequence. Methods {@link + * #writeTo(WritableByteChannel)} and {@link #asInputStream()} allow to write the sequence somewhere else. {@link + * #readFully} allows to access the sequence randomly. + * + * WriteOutBytes is a resource that is managed by {@link SegmentWriteOutMedium}, so it's own {@link #close()} method + * does nothing. However WriteOutBytes should appear closed, i. e. {@link #isOpen()} returns false, after the parental + * SegmentWriteOutMedium is closed. + */ +public abstract class WriteOutBytes extends OutputStream implements WritableByteChannel +{ + /** + * Writes 4 bytes of the given value in big-endian order, i. e. similar to {@link java.io.DataOutput#writeInt(int)}. + */ + public abstract void writeInt(int v) throws IOException; + + /** + * Returns the number of bytes written to this WriteOutBytes so far. + */ + public abstract long size() throws IOException; + + /** + * Takes all bytes that are written to this WriteOutBytes so far and writes them into the given channel. + */ + public abstract void writeTo(WritableByteChannel channel) throws IOException; + + /** + * Creates a finite {@link InputStream} with the bytes that are written to this WriteOutBytes so far. The returned + * InputStream must be closed properly after it's used up. + */ + public abstract InputStream asInputStream() throws IOException; + + /** + * Reads bytes from the byte sequences, represented by this WriteOutBytes, at the random position, into the given + * buffer. + * + * @throws java.nio.BufferUnderflowException if the byte sequence from the given pos ends before the given buffer + * is filled + * @throws IllegalArgumentException if the given pos is negative + */ + public abstract void readFully(long pos, ByteBuffer buffer) throws IOException; + + /** + * @deprecated does nothing. + */ + @Deprecated + @Override + public final void close() + { + // Does nothing. + } +} diff --git a/processing/src/test/java/io/druid/query/DoubleStorageTest.java b/processing/src/test/java/io/druid/query/DoubleStorageTest.java index d663130ddba2..22281cefe7c1 100644 --- a/processing/src/test/java/io/druid/query/DoubleStorageTest.java +++ b/processing/src/test/java/io/druid/query/DoubleStorageTest.java @@ -32,6 +32,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.guava.Sequences; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.metadata.SegmentMetadataQueryConfig; import io.druid.query.metadata.SegmentMetadataQueryQueryToolChest; @@ -109,8 +110,9 @@ private ScanQuery.ScanQueryBuilder newTestQuery() } - private static final IndexMergerV9 INDEX_MERGER_V9 = TestHelper.getTestIndexMergerV9(); - private static final IndexIO INDEX_IO = TestHelper.getTestIndexIO(); + private static final IndexMergerV9 INDEX_MERGER_V9 = + TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()); + private static final IndexIO INDEX_IO = TestHelper.getTestIndexIO(OffHeapMemorySegmentWriteOutMediumFactory.instance()); private static final Integer MAX_ROWS = 10; private static final String TIME_COLUMN = "__time"; private static final String DIM_NAME = "testDimName"; @@ -330,7 +332,7 @@ private static QueryableIndex buildIndex(String storeDoubleAsFloat) throws IOExc getStreamOfEvents().forEach(o -> { try { - index.add(ROW_PARSER.parse((Map) o)); + index.add(ROW_PARSER.parseBatch((Map) o).get(0)); } catch (IndexSizeExceededException e) { Throwables.propagate(e); @@ -345,7 +347,7 @@ private static QueryableIndex buildIndex(String storeDoubleAsFloat) throws IOExc File someTmpFile = File.createTempFile("billy", "yay"); someTmpFile.delete(); someTmpFile.mkdirs(); - INDEX_MERGER_V9.persist(index, someTmpFile, new IndexSpec()); + INDEX_MERGER_V9.persist(index, someTmpFile, new IndexSpec(), null); someTmpFile.delete(); return INDEX_IO.loadIndex(someTmpFile); } diff --git a/processing/src/test/java/io/druid/query/MultiValuedDimensionTest.java b/processing/src/test/java/io/druid/query/MultiValuedDimensionTest.java index c7603797680f..f3708608ba86 100644 --- a/processing/src/test/java/io/druid/query/MultiValuedDimensionTest.java +++ b/processing/src/test/java/io/druid/query/MultiValuedDimensionTest.java @@ -35,6 +35,9 @@ import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregationTestHelper; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.dimension.DefaultDimensionSpec; @@ -61,8 +64,8 @@ import io.druid.segment.TestHelper; import io.druid.segment.incremental.IncrementalIndex; import org.apache.commons.io.FileUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -81,36 +84,38 @@ @RunWith(Parameterized.class) public class MultiValuedDimensionTest { - private AggregationTestHelper helper; + @Parameterized.Parameters(name = "{0}") + public static Collection constructorFeeder() throws IOException + { + final List constructors = Lists.newArrayList(); + for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) { + constructors.add(new Object[]{config, TmpFileSegmentWriteOutMediumFactory.instance()}); + constructors.add(new Object[]{config, OffHeapMemorySegmentWriteOutMediumFactory.instance()}); + } + return constructors; + } + + private final AggregationTestHelper helper; + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; - private static IncrementalIndex incrementalIndex; - private static QueryableIndex queryableIndex; + private IncrementalIndex incrementalIndex; + private QueryableIndex queryableIndex; - private static File persistedSegmentDir; + private File persistedSegmentDir; - public MultiValuedDimensionTest( - final GroupByQueryConfig config - ) throws Exception + public MultiValuedDimensionTest(final GroupByQueryConfig config, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) + throws Exception { helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( ImmutableList.of(), config, null ); + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } - @Parameterized.Parameters(name = "{0}") - public static Collection constructorFeeder() throws IOException - { - final List constructors = Lists.newArrayList(); - for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) { - constructors.add(new Object[]{config}); - } - return constructors; - } - - @BeforeClass - public static void setupClass() throws Exception + @Before + public void setup() throws Exception { incrementalIndex = new IncrementalIndex.Builder() .setSimpleTestingIndexSchema(new CountAggregatorFactory("count")) @@ -141,10 +146,10 @@ public static void setupClass() throws Exception } persistedSegmentDir = Files.createTempDir(); - TestHelper.getTestIndexMergerV9() - .persist(incrementalIndex, persistedSegmentDir, new IndexSpec()); + TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory) + .persist(incrementalIndex, persistedSegmentDir, new IndexSpec(), null); - queryableIndex = TestHelper.getTestIndexIO().loadIndex(persistedSegmentDir); + queryableIndex = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory).loadIndex(persistedSegmentDir); } @Test @@ -300,8 +305,8 @@ public void testTopNWithDimFilterAndWithFilteredDimSpec() throws Exception ); } - @AfterClass - public static void cleanup() throws Exception + @After + public void cleanup() throws Exception { queryableIndex.close(); incrementalIndex.close(); diff --git a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java index 34ed7d59c569..51a9b189047f 100644 --- a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java +++ b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java @@ -91,12 +91,12 @@ public static List inputRowsWithDimensions(final List dimensio ) ); return ImmutableList.of( - parser.parse(ImmutableMap.of("t", "2000-01-01", "c1", "9", "c2", ImmutableList.of("a"))), - parser.parse(ImmutableMap.of("t", "2000-01-02", "c1", "10.1", "c2", ImmutableList.of())), - parser.parse(ImmutableMap.of("t", "2000-01-03", "c1", "2", "c2", ImmutableList.of(""))), - parser.parse(ImmutableMap.of("t", "2001-01-01", "c1", "1", "c2", ImmutableList.of("a", "c"))), - parser.parse(ImmutableMap.of("t", "2001-01-02", "c1", "4", "c2", ImmutableList.of("abc"))), - parser.parse(ImmutableMap.of("t", "2001-01-03", "c1", "5")) + parser.parseBatch(ImmutableMap.of("t", "2000-01-01", "c1", "9", "c2", ImmutableList.of("a"))).get(0), + parser.parseBatch(ImmutableMap.of("t", "2000-01-02", "c1", "10.1", "c2", ImmutableList.of())).get(0), + parser.parseBatch(ImmutableMap.of("t", "2000-01-03", "c1", "2", "c2", ImmutableList.of(""))).get(0), + parser.parseBatch(ImmutableMap.of("t", "2001-01-01", "c1", "1", "c2", ImmutableList.of("a", "c"))).get(0), + parser.parseBatch(ImmutableMap.of("t", "2001-01-02", "c1", "4", "c2", ImmutableList.of("abc"))).get(0), + parser.parseBatch(ImmutableMap.of("t", "2001-01-03", "c1", "5")).get(0) ); } diff --git a/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java b/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java index 4a3ca1156f43..27e2c5958564 100644 --- a/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java +++ b/processing/src/test/java/io/druid/query/aggregation/AggregationTestHelper.java @@ -35,6 +35,7 @@ import com.google.common.io.Closeables; import com.google.common.util.concurrent.MoreExecutors; import io.druid.collections.StupidPool; +import io.druid.data.input.InputRow; import io.druid.data.input.Row; import io.druid.data.input.impl.InputRowParser; import io.druid.data.input.impl.StringInputRowParser; @@ -45,6 +46,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.guava.Yielder; import io.druid.java.util.common.guava.YieldingAccumulator; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.Query; import io.druid.query.QueryPlus; @@ -141,6 +143,7 @@ public static final AggregationTestHelper createGroupByQueryAggregationTestHelpe IndexIO indexIO = new IndexIO( mapper, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -153,7 +156,7 @@ public int columnCacheSizeBytes() return new AggregationTestHelper( mapper, - new IndexMergerV9(mapper, indexIO), + new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()), indexIO, factory.getToolchest(), factory, @@ -196,6 +199,7 @@ public static final AggregationTestHelper createSelectQueryAggregationTestHelper IndexIO indexIO = new IndexIO( mapper, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -208,7 +212,7 @@ public int columnCacheSizeBytes() return new AggregationTestHelper( mapper, - new IndexMergerV9(mapper, indexIO), + new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()), indexIO, toolchest, factory, @@ -236,6 +240,7 @@ public static final AggregationTestHelper createTimeseriesQueryAggregationTestHe IndexIO indexIO = new IndexIO( mapper, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -248,7 +253,7 @@ public int columnCacheSizeBytes() return new AggregationTestHelper( mapper, - new IndexMergerV9(mapper, indexIO), + new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()), indexIO, toolchest, factory, @@ -287,6 +292,7 @@ public ByteBuffer get() IndexIO indexIO = new IndexIO( mapper, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -299,7 +305,7 @@ public int columnCacheSizeBytes() return new AggregationTestHelper( mapper, - new IndexMergerV9(mapper, indexIO), + new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()), indexIO, toolchest, factory, @@ -428,7 +434,7 @@ public void createIndex( if (!index.canAppendRow()) { File tmp = tempFolder.newFolder(); toMerge.add(tmp); - indexMerger.persist(index, tmp, new IndexSpec()); + indexMerger.persist(index, tmp, new IndexSpec(), null); index.close(); index = new IncrementalIndex.Builder() .setIndexSchema( @@ -447,26 +453,26 @@ public void createIndex( //InputRowsParser index.add(((StringInputRowParser) parser).parse((String) row)); } else { - index.add(parser.parse(row)); + index.add(((List) parser.parseBatch(row)).get(0)); } } if (toMerge.size() > 0) { File tmp = tempFolder.newFolder(); toMerge.add(tmp); - indexMerger.persist(index, tmp, new IndexSpec()); + indexMerger.persist(index, tmp, new IndexSpec(), null); List indexes = new ArrayList<>(toMerge.size()); for (File file : toMerge) { indexes.add(indexIO.loadIndex(file)); } - indexMerger.mergeQueryableIndex(indexes, true, metrics, outDir, new IndexSpec()); + indexMerger.mergeQueryableIndex(indexes, true, metrics, outDir, new IndexSpec(), null); for (QueryableIndex qi : indexes) { qi.close(); } } else { - indexMerger.persist(index, outDir, new IndexSpec()); + indexMerger.persist(index, outDir, new IndexSpec(), null); } } finally { diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index ef719a776a36..164764131231 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -48,8 +48,8 @@ import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ValueType; -import io.druid.segment.data.ArrayBasedIndexedInts; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.SingleIndexedInt; import org.junit.Assert; import org.junit.Test; @@ -102,9 +102,9 @@ public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) public IndexedInts getRow() { if (selector.getIndex() % 3 == 2) { - return ArrayBasedIndexedInts.of(new int[]{1}); + return SingleIndexedInt.of(1); } else { - return ArrayBasedIndexedInts.of(new int[]{0}); + return SingleIndexedInt.of(0); } } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java index 55eda96ee560..edcbd974d738 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java @@ -48,6 +48,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.BySegmentQueryRunner; import io.druid.query.DruidProcessingConfig; import io.druid.query.FinalizeResultsQueryRunner; @@ -117,6 +118,7 @@ public class GroupByLimitPushDownInsufficientBufferTest ); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -126,7 +128,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } @@ -210,7 +212,8 @@ public void setup() throws Exception final File fileA = INDEX_MERGER_V9.persist( indexA, new File(tmpDir, "A"), - new IndexSpec() + new IndexSpec(), + OffHeapMemorySegmentWriteOutMediumFactory.instance() ); QueryableIndex qindexA = INDEX_IO.loadIndex(fileA); @@ -251,7 +254,8 @@ public void setup() throws Exception final File fileB = INDEX_MERGER_V9.persist( indexB, new File(tmpDir, "B"), - new IndexSpec() + new IndexSpec(), + OffHeapMemorySegmentWriteOutMediumFactory.instance() ); QueryableIndex qindexB = INDEX_IO.loadIndex(fileB); diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java index cc117df0abc0..d11f8e62d3f0 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java @@ -49,6 +49,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.BySegmentQueryRunner; import io.druid.query.DruidProcessingConfig; import io.druid.query.FinalizeResultsQueryRunner; @@ -124,6 +125,7 @@ public class GroupByLimitPushDownMultiNodeMergeTest ); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -133,7 +135,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } private IncrementalIndex makeIncIndex(boolean withRollup) @@ -197,7 +199,8 @@ public void setup() throws Exception final File fileA = INDEX_MERGER_V9.persist( indexA, new File(tmpDir, "A"), - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qindexA = INDEX_IO.loadIndex(fileA); @@ -232,7 +235,8 @@ public void setup() throws Exception final File fileB = INDEX_MERGER_V9.persist( indexB, new File(tmpDir, "B"), - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qindexB = INDEX_IO.loadIndex(fileB); diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByMultiSegmentTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByMultiSegmentTest.java index 740e87ceda0f..8bcbcc80a01b 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByMultiSegmentTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByMultiSegmentTest.java @@ -46,6 +46,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.BySegmentQueryRunner; import io.druid.query.DruidProcessingConfig; import io.druid.query.FinalizeResultsQueryRunner; @@ -113,6 +114,7 @@ public class GroupByMultiSegmentTest ); INDEX_IO = new IndexIO( JSON_MAPPER, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -122,7 +124,7 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); } @@ -172,7 +174,8 @@ public void setup() throws Exception final File fileA = INDEX_MERGER_V9.persist( indexA, new File(tmpDir, "A"), - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qindexA = INDEX_IO.loadIndex(fileA); @@ -193,7 +196,8 @@ public void setup() throws Exception final File fileB = INDEX_MERGER_V9.persist( indexB, new File(tmpDir, "B"), - new IndexSpec() + new IndexSpec(), + null ); QueryableIndex qindexB = INDEX_IO.loadIndex(fileB); diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 660a5cda0ccd..14ca7fb122d2 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -5617,4 +5617,36 @@ public void testFullOnTopNBoundFilterAndLongSumMetric() ); assertExpectedResults(expectedResults, query); } + + /** + * Regression test for https://github.com/druid-io/druid/issues/5132 + */ + @Test + public void testTopNWithNonBitmapFilter() + { + TopNQuery query = new TopNQueryBuilder() + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.allGran) + .filters(new BoundDimFilter( + Column.TIME_COLUMN_NAME, + "0", + String.valueOf(Long.MAX_VALUE), + true, + true, + false, + null, + StringComparators.NUMERIC + )) + .dimension(QueryRunnerTestHelper.marketDimension) + .metric("count") + .threshold(4) + .intervals(QueryRunnerTestHelper.firstToThird) + .aggregators( + Collections.singletonList(new DoubleSumAggregatorFactory("count", "qualityDouble")) + ) + .build(); + + // Don't check results, just the fact that the query could complete + Assert.assertNotNull(Sequences.toList(runWithMerge(query), new ArrayList<>())); + } } diff --git a/processing/src/test/java/io/druid/segment/AppendTest.java b/processing/src/test/java/io/druid/segment/AppendTest.java index 4eb92b58e9ef..39f659c99e32 100644 --- a/processing/src/test/java/io/druid/segment/AppendTest.java +++ b/processing/src/test/java/io/druid/segment/AppendTest.java @@ -27,6 +27,7 @@ import io.druid.java.util.common.Pair; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.granularity.Granularity; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.QueryPlus; import io.druid.query.QueryRunner; @@ -113,10 +114,11 @@ public class AppendTest @Before public void setUp() throws Exception { + SchemalessIndexTest schemalessIndexTest = new SchemalessIndexTest(OffHeapMemorySegmentWriteOutMediumFactory.instance()); // (1, 2) cover overlapping segments of the form // |------| // |--------| - QueryableIndex appendedIndex = SchemalessIndexTest.getAppendedIncrementalIndex( + QueryableIndex appendedIndex = schemalessIndexTest.getAppendedIncrementalIndex( Arrays.asList( new Pair("append.json.1", METRIC_AGGS_NO_UNIQ), new Pair("append.json.2", METRIC_AGGS) @@ -131,7 +133,7 @@ public void setUp() throws Exception // (3, 4) cover overlapping segments of the form // |------------| // |-----| - QueryableIndex append2 = SchemalessIndexTest.getAppendedIncrementalIndex( + QueryableIndex append2 = schemalessIndexTest.getAppendedIncrementalIndex( Arrays.asList( new Pair("append.json.3", METRIC_AGGS_NO_UNIQ), new Pair("append.json.4", METRIC_AGGS) @@ -147,7 +149,7 @@ public void setUp() throws Exception // |-------------| // |---| // |---| - QueryableIndex append3 = SchemalessIndexTest.getAppendedIncrementalIndex( + QueryableIndex append3 = schemalessIndexTest.getAppendedIncrementalIndex( Arrays.asList( new Pair("append.json.5", METRIC_AGGS), new Pair("append.json.6", METRIC_AGGS), diff --git a/processing/src/test/java/io/druid/segment/CompressedVSizeIndexedV3SupplierTest.java b/processing/src/test/java/io/druid/segment/CompressedVSizeIndexedV3SupplierTest.java index 55e6cb829a21..e3538fd70e20 100644 --- a/processing/src/test/java/io/druid/segment/CompressedVSizeIndexedV3SupplierTest.java +++ b/processing/src/test/java/io/druid/segment/CompressedVSizeIndexedV3SupplierTest.java @@ -21,8 +21,9 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; -import io.druid.segment.data.CompressedObjectStrategy; +import io.druid.java.util.common.io.Closer; import io.druid.segment.data.CompressedVSizeIndexedSupplierTest; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.IndexedMultivalue; import io.druid.segment.data.VSizeIndexedInts; @@ -30,12 +31,16 @@ import org.junit.After; import org.junit.Before; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; public class CompressedVSizeIndexedV3SupplierTest extends CompressedVSizeIndexedSupplierTest { + + private Closer closer; + @Override @Before public void setUpSimple() @@ -46,7 +51,7 @@ public void setUpSimple() new int[]{6, 7, 8, 9, 10}, new int[]{11, 12, 13, 14, 15, 16, 17, 18, 19, 20} ); - + closer = Closer.create(); indexedSupplier = CompressedVSizeIndexedV3Supplier.fromIterable( Iterables.transform( vals, @@ -59,15 +64,17 @@ public IndexedInts apply(int[] input) } } ), 2, 20, ByteOrder.nativeOrder(), - CompressedObjectStrategy.CompressionStrategy.LZ4 + CompressionStrategy.LZ4, + closer ); } @Override @After - public void teardown() + public void teardown() throws IOException { indexedSupplier = null; + closer.close(); vals = null; } @@ -76,8 +83,7 @@ protected WritableSupplier> fromByteBuffer(ByteBu { return CompressedVSizeIndexedV3Supplier.fromByteBuffer( buffer, - ByteOrder.nativeOrder(), - null + ByteOrder.nativeOrder() ); } } diff --git a/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerV9Test.java b/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerV9Test.java index 595a1e80e046..785f03c52728 100644 --- a/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerV9Test.java +++ b/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerV9Test.java @@ -19,8 +19,9 @@ package io.druid.segment; -import io.druid.segment.data.CompressedObjectStrategy.CompressionStrategy; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -31,10 +32,17 @@ public class ConciseBitmapIndexMergerV9Test extends IndexMergerTestBase public ConciseBitmapIndexMergerV9Test( CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, - LongEncodingStrategy longEncodingStrategy + LongEncodingStrategy longEncodingStrategy, + SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { - super(new ConciseBitmapSerdeFactory(), compressionStrategy, dimCompressionStrategy, longEncodingStrategy); - indexMerger = TestHelper.getTestIndexMergerV9(); + super( + new ConciseBitmapSerdeFactory(), + compressionStrategy, + dimCompressionStrategy, + longEncodingStrategy, + segmentWriteOutMediumFactory + ); + indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); } } diff --git a/processing/src/test/java/io/druid/segment/EmptyIndexTest.java b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java index 99654c6ff188..f7091e2bce4a 100644 --- a/processing/src/test/java/io/druid/segment/EmptyIndexTest.java +++ b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java @@ -19,10 +19,14 @@ package io.druid.segment; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.java.util.common.Intervals; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.column.Column; import io.druid.segment.incremental.IncrementalIndex; @@ -30,11 +34,33 @@ import org.apache.commons.io.FileUtils; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; +import java.io.IOException; +import java.util.Collection; +@RunWith(Parameterized.class) public class EmptyIndexTest { + + @Parameterized.Parameters + public static Collection constructorFeeder() throws IOException + { + return ImmutableList.of( + new Object[] {TmpFileSegmentWriteOutMediumFactory.instance()}, + new Object[] {OffHeapMemorySegmentWriteOutMediumFactory.instance()} + ); + } + + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; + + public EmptyIndexTest(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) + { + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; + } + @Test public void testEmptyIndex() throws Exception { @@ -57,7 +83,7 @@ public void testEmptyIndex() throws Exception emptyIndex, new ConciseBitmapFactory() ); - TestHelper.getTestIndexMergerV9().merge( + TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory).merge( Lists.newArrayList(emptyIndexAdapter), true, new AggregatorFactory[0], @@ -65,7 +91,7 @@ public void testEmptyIndex() throws Exception new IndexSpec() ); - QueryableIndex emptyQueryableIndex = TestHelper.getTestIndexIO().loadIndex(tmpDir); + QueryableIndex emptyQueryableIndex = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory).loadIndex(tmpDir); Assert.assertEquals("getDimensionNames", 0, Iterables.size(emptyQueryableIndex.getAvailableDimensions())); Assert.assertEquals("getMetricNames", 0, Iterables.size(emptyQueryableIndex.getColumnNames())); diff --git a/processing/src/test/java/io/druid/segment/IndexBuilder.java b/processing/src/test/java/io/druid/segment/IndexBuilder.java index 990b4b21a408..a2213ac8d4de 100644 --- a/processing/src/test/java/io/druid/segment/IndexBuilder.java +++ b/processing/src/test/java/io/druid/segment/IndexBuilder.java @@ -26,6 +26,8 @@ import com.google.common.collect.Lists; import io.druid.data.input.InputRow; import io.druid.java.util.common.StringUtils; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.segment.incremental.IncrementalIndex; @@ -50,7 +52,8 @@ public class IndexBuilder private IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() .withMetrics(new CountAggregatorFactory("count")) .build(); - private IndexMerger indexMerger = TestHelper.getTestIndexMergerV9(); + private SegmentWriteOutMediumFactory segmentWriteOutMediumFactory = OffHeapMemorySegmentWriteOutMediumFactory.instance(); + private IndexMerger indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); private File tmpDir; private IndexSpec indexSpec = new IndexSpec(); private int maxRows = DEFAULT_MAX_ROWS; @@ -73,9 +76,10 @@ public IndexBuilder schema(IncrementalIndexSchema schema) return this; } - public IndexBuilder indexMerger(IndexMerger indexMerger) + public IndexBuilder segmentWriteOutMediumFactory(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) { - this.indexMerger = indexMerger; + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; + this.indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); return this; } @@ -108,11 +112,12 @@ public QueryableIndex buildMMappedIndex() Preconditions.checkNotNull(indexMerger, "indexMerger"); Preconditions.checkNotNull(tmpDir, "tmpDir"); try (final IncrementalIndex incrementalIndex = buildIncrementalIndex()) { - return TestHelper.getTestIndexIO().loadIndex( + return TestHelper.getTestIndexIO(segmentWriteOutMediumFactory).loadIndex( indexMerger.persist( incrementalIndex, new File(tmpDir, StringUtils.format("testIndex-%s", new Random().nextInt(Integer.MAX_VALUE))), - indexSpec + indexSpec, + null ) ); } @@ -123,14 +128,14 @@ public QueryableIndex buildMMappedIndex() public QueryableIndex buildMMappedMergedIndex() { - Preconditions.checkNotNull(indexMerger, "indexMerger"); + IndexMerger indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); Preconditions.checkNotNull(tmpDir, "tmpDir"); final List persisted = Lists.newArrayList(); try { for (int i = 0; i < rows.size(); i += ROWS_PER_INDEX_FOR_MERGING) { persisted.add( - TestHelper.getTestIndexIO().loadIndex( + TestHelper.getTestIndexIO(segmentWriteOutMediumFactory).loadIndex( indexMerger.persist( buildIncrementalIndexWithRows( schema, @@ -138,12 +143,13 @@ public QueryableIndex buildMMappedMergedIndex() rows.subList(i, Math.min(rows.size(), i + ROWS_PER_INDEX_FOR_MERGING)) ), new File(tmpDir, StringUtils.format("testIndex-%s", UUID.randomUUID().toString())), - indexSpec + indexSpec, + null ) ) ); } - final QueryableIndex merged = TestHelper.getTestIndexIO().loadIndex( + final QueryableIndex merged = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory).loadIndex( indexMerger.merge( Lists.transform( persisted, diff --git a/processing/src/test/java/io/druid/segment/IndexIOTest.java b/processing/src/test/java/io/druid/segment/IndexIOTest.java index e0ea8c1e6cf7..a50704120589 100644 --- a/processing/src/test/java/io/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/io/druid/segment/IndexIOTest.java @@ -31,10 +31,11 @@ import io.druid.data.input.impl.DimensionsSpec; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.UOE; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.CountAggregatorFactory; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexAdapter; @@ -66,8 +67,8 @@ public class IndexIOTest private static Interval DEFAULT_INTERVAL = Intervals.of("1970-01-01/2000-01-01"); private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( new ConciseBitmapSerdeFactory(), - CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.LZ4, + CompressionStrategy.LZ4, + CompressionStrategy.LZ4, CompressionFactory.LongEncodingStrategy.LONGS ); @@ -328,7 +329,7 @@ public void testRowValidatorEquals() throws Exception { Exception ex = null; try { - TestHelper.getTestIndexIO().validateTwoSegments(adapter1, adapter2); + TestHelper.getTestIndexIO(OffHeapMemorySegmentWriteOutMediumFactory.instance()).validateTwoSegments(adapter1, adapter2); } catch (Exception e) { ex = e; diff --git a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java index c38f31f9e033..f40ace07d97e 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java @@ -33,6 +33,7 @@ import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.data.IncrementalIndexTest; import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -63,8 +64,8 @@ public class IndexMergerNullHandlingTest @Before public void setUp() { - indexMerger = TestHelper.getTestIndexMergerV9(); - indexIO = TestHelper.getTestIndexIO(); + indexMerger = TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()); + indexIO = TestHelper.getTestIndexIO(OffHeapMemorySegmentWriteOutMediumFactory.instance()); indexSpec = new IndexSpec(); } @@ -114,7 +115,7 @@ public void testStringColumnNullHandling() throws Exception } final File tempDir = temporaryFolder.newFolder(); - try (QueryableIndex index = indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec))) { + try (QueryableIndex index = indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))) { final Column column = index.getColumn("d"); if (subsetList.stream().allMatch(nullFlavors::contains)) { diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index 5efca99c34c9..6a68941a3847 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -41,6 +41,7 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; @@ -50,8 +51,8 @@ import io.druid.segment.column.SimpleDictionaryEncodedColumn; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.BitmapValues; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.IncrementalIndexTest; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexAdapter; @@ -81,22 +82,22 @@ public class IndexMergerTestBase { - private final static IndexIO INDEX_IO = TestHelper.getTestIndexIO(); @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); protected IndexMerger indexMerger; - @Parameterized.Parameters(name = "{index}: metric compression={0}, dimension compression={1}, long encoding={2}") + @Parameterized.Parameters(name = "{index}: metric compression={0}, dimension compression={1}, long encoding={2}, segment write-out medium={3}") public static Collection data() { return Collections2.transform( Sets.cartesianProduct( ImmutableList.of( - EnumSet.allOf(CompressedObjectStrategy.CompressionStrategy.class), - ImmutableSet.copyOf(CompressedObjectStrategy.CompressionStrategy.noNoneValues()), - EnumSet.allOf(CompressionFactory.LongEncodingStrategy.class) + EnumSet.allOf(CompressionStrategy.class), + ImmutableSet.copyOf(CompressionStrategy.noNoneValues()), + EnumSet.allOf(CompressionFactory.LongEncodingStrategy.class), + SegmentWriteOutMediumFactory.builtInFactories() ) ), new Function, Object[]>() { @@ -112,8 +113,8 @@ public Object[] apply(List input) static IndexSpec makeIndexSpec( BitmapSerdeFactory bitmapSerdeFactory, - CompressedObjectStrategy.CompressionStrategy compressionStrategy, - CompressedObjectStrategy.CompressionStrategy dimCompressionStrategy, + CompressionStrategy compressionStrategy, + CompressionStrategy dimCompressionStrategy, CompressionFactory.LongEncodingStrategy longEncodingStrategy ) { @@ -130,17 +131,26 @@ static IndexSpec makeIndexSpec( } private final IndexSpec indexSpec; + private final IndexIO indexIO; + @Rule public final CloserRule closer = new CloserRule(false); protected IndexMergerTestBase( BitmapSerdeFactory bitmapSerdeFactory, - CompressedObjectStrategy.CompressionStrategy compressionStrategy, - CompressedObjectStrategy.CompressionStrategy dimCompressionStrategy, - CompressionFactory.LongEncodingStrategy longEncodingStrategy + CompressionStrategy compressionStrategy, + CompressionStrategy dimCompressionStrategy, + CompressionFactory.LongEncodingStrategy longEncodingStrategy, + SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { - this.indexSpec = makeIndexSpec(bitmapSerdeFactory, compressionStrategy, dimCompressionStrategy, longEncodingStrategy); + this.indexSpec = makeIndexSpec( + bitmapSerdeFactory, + compressionStrategy, + dimCompressionStrategy, + longEncodingStrategy + ); + this.indexIO = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory); } @Test @@ -153,11 +163,12 @@ public void testPersist() throws Exception final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist, tempDir, - indexSpec + indexSpec, + null ) ) ); @@ -200,11 +211,12 @@ public void testPersistWithDifferentDims() throws Exception final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist, tempDir, - indexSpec + indexSpec, + null ) ) ); @@ -242,11 +254,12 @@ public void testPersistWithSegmentMetadata() throws Exception final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist, tempDir, - indexSpec + indexSpec, + null ) ) ); @@ -302,11 +315,12 @@ public void testPersistMerge() throws Exception final File mergedDir = temporaryFolder.newFolder(); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist1, tempDir1, - indexSpec + indexSpec, + null ) ) ); @@ -316,11 +330,12 @@ public void testPersistMerge() throws Exception Assert.assertEquals(3, index1.getColumnNames().size()); QueryableIndex index2 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist2, tempDir2, - indexSpec + indexSpec, + null ) ) ); @@ -333,13 +348,14 @@ public void testPersistMerge() throws Exception new CountAggregatorFactory("count") }; QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2), true, mergedAggregators, mergedDir, - indexSpec + indexSpec, + null ) ) ); @@ -391,31 +407,34 @@ public void testPersistEmptyColumn() throws Exception ); final QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist1, tmpDir1, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex index2 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist2, tmpDir2, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2), true, new AggregatorFactory[]{}, tmpDir3, - indexSpec + indexSpec, + null ) ) ); @@ -451,11 +470,12 @@ public void testMergeRetainsValues() throws Exception ); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist1, tempDir1, - indexSpec + indexSpec, + null ) ) ); @@ -463,7 +483,7 @@ public void testMergeRetainsValues() throws Exception final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1); - INDEX_IO.validateTwoSegments(incrementalAdapter, queryableAdapter); + indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); @@ -471,13 +491,14 @@ public void testMergeRetainsValues() throws Exception QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( ImmutableList.of(index1), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, mergedDir, - indexSpec + indexSpec, + null ) ) ); @@ -486,7 +507,7 @@ public void testMergeRetainsValues() throws Exception Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); - INDEX_IO.validateTwoSegments(tempDir1, mergedDir); + indexIO.validateTwoSegments(tempDir1, mergedDir); assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(merged, indexSpec.getDimensionCompression()); @@ -509,15 +530,11 @@ public void testAppendRetainsValues() throws Exception ); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex( - indexMerger.append( - ImmutableList.of(incrementalAdapter), null, tempDir1, indexSpec - ) - ) + indexIO.loadIndex(indexMerger.append(ImmutableList.of(incrementalAdapter), null, tempDir1, indexSpec, null)) ); final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1); - INDEX_IO.validateTwoSegments(incrementalAdapter, queryableAdapter); + indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); @@ -530,13 +547,14 @@ public void testAppendRetainsValues() throws Exception AggregatorFactory[] mergedAggregators = new AggregatorFactory[]{new CountAggregatorFactory("count")}; QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( ImmutableList.of(index1), true, mergedAggregators, mergedDir, - indexSpec + indexSpec, + null ) ) ); @@ -545,7 +563,7 @@ public void testAppendRetainsValues() throws Exception Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); - INDEX_IO.validateTwoSegments(tempDir1, mergedDir); + indexIO.validateTwoSegments(tempDir1, mergedDir); assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(merged, indexSpec.getDimensionCompression()); @@ -573,11 +591,12 @@ public void testMergeSpecChange() throws Exception ); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist1, tempDir1, - indexSpec + indexSpec, + null ) ) ); @@ -585,7 +604,7 @@ public void testMergeSpecChange() throws Exception final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1); - INDEX_IO.validateTwoSegments(incrementalAdapter, queryableAdapter); + indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); @@ -593,12 +612,12 @@ public void testMergeSpecChange() throws Exception IndexSpec newSpec = new IndexSpec( indexSpec.getBitmapSerdeFactory(), - CompressedObjectStrategy.CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? - CompressedObjectStrategy.CompressionStrategy.LZF : - CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? - CompressedObjectStrategy.CompressionStrategy.LZF : - CompressedObjectStrategy.CompressionStrategy.LZ4, + CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? + CompressionStrategy.LZF : + CompressionStrategy.LZ4, + CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? + CompressionStrategy.LZF : + CompressionStrategy.LZ4, CompressionFactory.LongEncodingStrategy.LONGS.equals(indexSpec.getLongEncoding()) ? CompressionFactory.LongEncodingStrategy.AUTO : CompressionFactory.LongEncodingStrategy.LONGS @@ -606,13 +625,14 @@ public void testMergeSpecChange() throws Exception AggregatorFactory[] mergedAggregators = new AggregatorFactory[]{new CountAggregatorFactory("count")}; QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( ImmutableList.of(index1), true, mergedAggregators, mergedDir, - newSpec + newSpec, + null ) ) ); @@ -621,7 +641,7 @@ public void testMergeSpecChange() throws Exception Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions())); Assert.assertEquals(3, merged.getColumnNames().size()); - INDEX_IO.validateTwoSegments(tempDir1, mergedDir); + indexIO.validateTwoSegments(tempDir1, mergedDir); assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(merged, newSpec.getDimensionCompression()); @@ -653,12 +673,12 @@ public void testConvertSame() throws Exception ); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex(indexMerger.persist(toPersist1, tempDir1, indexSpec)) + indexIO.loadIndex(indexMerger.persist(toPersist1, tempDir1, indexSpec, null)) ); final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1); - INDEX_IO.validateTwoSegments(incrementalAdapter, queryableAdapter); + indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); @@ -666,7 +686,7 @@ public void testConvertSame() throws Exception QueryableIndex converted = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.convert( tempDir1, convertDir, @@ -679,7 +699,7 @@ public void testConvertSame() throws Exception Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(converted.getAvailableDimensions())); Assert.assertEquals(4, converted.getColumnNames().size()); - INDEX_IO.validateTwoSegments(tempDir1, convertDir); + indexIO.validateTwoSegments(tempDir1, convertDir); assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(converted, indexSpec.getDimensionCompression()); @@ -716,11 +736,12 @@ public void testConvertDifferent() throws Exception ); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist1, tempDir1, - indexSpec + indexSpec, + null ) ) ); @@ -728,7 +749,7 @@ public void testConvertDifferent() throws Exception final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1); - INDEX_IO.validateTwoSegments(incrementalAdapter, queryableAdapter); + indexIO.validateTwoSegments(incrementalAdapter, queryableAdapter); Assert.assertEquals(2, index1.getColumn(Column.TIME_COLUMN_NAME).getLength()); Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions())); @@ -737,19 +758,19 @@ public void testConvertDifferent() throws Exception IndexSpec newSpec = new IndexSpec( indexSpec.getBitmapSerdeFactory(), - CompressedObjectStrategy.CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? - CompressedObjectStrategy.CompressionStrategy.LZF : - CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? - CompressedObjectStrategy.CompressionStrategy.LZF : - CompressedObjectStrategy.CompressionStrategy.LZ4, + CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? + CompressionStrategy.LZF : + CompressionStrategy.LZ4, + CompressionStrategy.LZ4.equals(indexSpec.getDimensionCompression()) ? + CompressionStrategy.LZF : + CompressionStrategy.LZ4, CompressionFactory.LongEncodingStrategy.LONGS.equals(indexSpec.getLongEncoding()) ? CompressionFactory.LongEncodingStrategy.AUTO : CompressionFactory.LongEncodingStrategy.LONGS ); QueryableIndex converted = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.convert( tempDir1, convertDir, @@ -762,7 +783,7 @@ public void testConvertDifferent() throws Exception Assert.assertEquals(Arrays.asList("dim1", "dim2"), Lists.newArrayList(converted.getAvailableDimensions())); Assert.assertEquals(4, converted.getColumnNames().size()); - INDEX_IO.validateTwoSegments(tempDir1, convertDir); + indexIO.validateTwoSegments(tempDir1, convertDir); assertDimCompression(index1, indexSpec.getDimensionCompression()); assertDimCompression(converted, newSpec.getDimensionCompression()); @@ -773,11 +794,11 @@ public void testConvertDifferent() throws Exception ); } - private void assertDimCompression(QueryableIndex index, CompressedObjectStrategy.CompressionStrategy expectedStrategy) + private void assertDimCompression(QueryableIndex index, CompressionStrategy expectedStrategy) throws Exception { // Java voodoo - if (expectedStrategy == null || expectedStrategy == CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED) { + if (expectedStrategy == null || expectedStrategy == CompressionStrategy.UNCOMPRESSED) { return; } @@ -823,44 +844,48 @@ public void testNonLexicographicDimOrderMerge() throws Exception final File tmpDirMerged = temporaryFolder.newFolder(); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist1, tmpDir, - indexSpec + indexSpec, + null ) ) ); QueryableIndex index2 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist2, tmpDir2, - indexSpec + indexSpec, + null ) ) ); QueryableIndex index3 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist3, tmpDir3, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2, index3), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged, - indexSpec + indexSpec, + null ) ) ); @@ -931,43 +956,47 @@ public void testMergeWithDimensionsList() throws Exception final File tmpDirMerged = temporaryFolder.newFolder(); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist1, tmpDir, - indexSpec + indexSpec, + null ) ) ); QueryableIndex index2 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist2, tmpDir2, - indexSpec + indexSpec, + null ) ) ); QueryableIndex index3 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist3, tmpDir3, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2, index3), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged, - indexSpec + indexSpec, + null ) ) ); @@ -1012,55 +1041,60 @@ public void testDisjointDimMerge() throws Exception final File tmpDirMerged = temporaryFolder.newFolder(); QueryableIndex indexA = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistA, tmpDirA, - indexSpec + indexSpec, + null ) ) ); QueryableIndex indexB = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistB, tmpDirB, - indexSpec + indexSpec, + null ) ) ); QueryableIndex indexB2 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistB2, tmpDirB2, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged2 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB2), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged, - indexSpec + indexSpec, + null ) ) ); @@ -1180,33 +1214,36 @@ public void testJointDimMerge() throws Exception final File tmpDirMerged = temporaryFolder.newFolder(); QueryableIndex indexA = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistA, tmpDirA, - indexSpec + indexSpec, + null ) ) ); QueryableIndex indexB = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistB, tmpDirB, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged, - indexSpec + indexSpec, + null ) ) ); @@ -1342,33 +1379,36 @@ public void testNoRollupMergeWithoutDuplicateRow() throws Exception final File tmpDirMerged = temporaryFolder.newFolder(); QueryableIndex indexA = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistA, tmpDirA, - indexSpec + indexSpec, + null ) ) ); QueryableIndex indexB = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistB, tmpDirB, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged, - indexSpec + indexSpec, + null ) ) ); @@ -1507,33 +1547,36 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception final File tmpDirMerged = temporaryFolder.newFolder(); QueryableIndex indexA = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistA, tmpDirA, - indexSpec + indexSpec, + null ) ) ); QueryableIndex indexB = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistB, tmpDirB, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB), false, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged, - indexSpec + indexSpec, + null ) ) ); @@ -1633,75 +1676,82 @@ public void testMergeWithSupersetOrdering() throws Exception final File tmpDirMerged2 = temporaryFolder.newFolder(); QueryableIndex indexA = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistA, tmpDirA, - indexSpec + indexSpec, + null ) ) ); QueryableIndex indexB = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistB, tmpDirB, - indexSpec + indexSpec, + null ) ) ); QueryableIndex indexBA = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistBA, tmpDirBA, - indexSpec + indexSpec, + null ) ) ); QueryableIndex indexBA2 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistBA2, tmpDirBA2, - indexSpec + indexSpec, + null ) ) ); QueryableIndex indexC = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersistC, tmpDirC, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB, indexBA, indexBA2), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged2 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB, indexBA, indexC), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged2, - indexSpec + indexSpec, + null ) ) ); @@ -1878,7 +1928,7 @@ public void testAddMetrics() throws IOException tmpDirMerged, indexSpec ); - final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(INDEX_IO.loadIndex( + final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex( merged))); Assert.assertEquals(ImmutableSet.of("A", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics())); @@ -1949,7 +1999,7 @@ public void testAddMetricsBothSidesNull() throws IOException tmpDirMerged, indexSpec ); - final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(INDEX_IO.loadIndex( + final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex( merged))); Assert.assertEquals(ImmutableSet.of("A", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics())); @@ -2014,7 +2064,7 @@ public void testMismatchedMetrics() throws IOException ); // Since D was not present in any of the indices, it is not present in the output - final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(INDEX_IO.loadIndex( + final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex( merged))); Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics())); @@ -2056,7 +2106,7 @@ public void testMismatchedMetricsVarying() throws IOException tmpDirMerged, indexSpec ); - final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(INDEX_IO.loadIndex( + final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex( merged))); Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics())); } @@ -2072,33 +2122,36 @@ public void testMergeNumericDims() throws Exception final File tmpDirMerged = temporaryFolder.newFolder(); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist1, tmpDir, - indexSpec + indexSpec, + null ) ) ); QueryableIndex index2 = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( toPersist2, tmpDir2, - indexSpec + indexSpec, + null ) ) ); final QueryableIndex merged = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, tmpDirMerged, - indexSpec + indexSpec, + null ) ) ); @@ -2203,11 +2256,12 @@ public void testPersistNullColumnSkipping() throws Exception final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( indexMerger.persist( index1, tempDir, - indexSpec + indexSpec, + null ) ) ); @@ -2364,7 +2418,8 @@ public void testCloser() throws Exception indexMerger.persist( toPersist, tempDir, - indexSpec + indexSpec, + null ); } finally { @@ -2483,6 +2538,6 @@ private QueryableIndex persistAndLoad(List schema, InputRow... } final File tempDir = temporaryFolder.newFolder(); - return closer.closeLater(INDEX_IO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec))); + return closer.closeLater(indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))); } } diff --git a/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java b/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java index 060bca51e449..aef1e55e2681 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java @@ -27,10 +27,13 @@ import io.druid.data.input.MapBasedInputRow; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.JodaUtils; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; @@ -40,6 +43,8 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; import java.io.IOException; @@ -52,10 +57,18 @@ import java.util.List; import java.util.Map; +@RunWith(Parameterized.class) public class IndexMergerV9CompatibilityTest { - @Rule - public final CloserRule closer = new CloserRule(false); + @Parameterized.Parameters + public static Collection constructorFeeder() throws IOException + { + return ImmutableList.of( + new Object[] {TmpFileSegmentWriteOutMediumFactory.instance()}, + new Object[] {OffHeapMemorySegmentWriteOutMediumFactory.instance()} + ); + } + private static final long TIMESTAMP = DateTimes.of("2014-01-01").getMillis(); private static final AggregatorFactory[] DEFAULT_AGG_FACTORIES = new AggregatorFactory[]{ new CountAggregatorFactory( @@ -63,21 +76,24 @@ public class IndexMergerV9CompatibilityTest ) }; - private static final IndexMergerV9 INDEX_MERGER_V9 = TestHelper.getTestIndexMergerV9(); - private static final IndexIO INDEX_IO = TestHelper.getTestIndexIO(); - private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( new ConciseBitmapSerdeFactory(), - CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.LZ4, + CompressionStrategy.LZ4, + CompressionStrategy.LZ4, CompressionFactory.LongEncodingStrategy.LONGS ); private static final List DIMS = ImmutableList.of("dim0", "dim1"); private final Collection events; + @Rule + public final CloserRule closer = new CloserRule(false); + private final IndexMerger indexMerger; + private final IndexIO indexIO; - public IndexMergerV9CompatibilityTest() + public IndexMergerV9CompatibilityTest(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) { + indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); + indexIO = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory); events = new ArrayList<>(); final Map map1 = ImmutableMap.of( @@ -161,7 +177,7 @@ public void testPersistWithSegmentMetadata() throws IOException QueryableIndex index = null; try { outDir = Files.createTempDir(); - index = INDEX_IO.loadIndex(INDEX_MERGER_V9.persist(toPersist, outDir, INDEX_SPEC)); + index = indexIO.loadIndex(indexMerger.persist(toPersist, outDir, INDEX_SPEC, null)); Assert.assertEquals("value", index.getMetadata().get("key")); } @@ -181,7 +197,7 @@ public void testSimpleReprocess() throws IOException { final IndexableAdapter adapter = new QueryableIndexIndexableAdapter( closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( persistTmpDir ) ) @@ -192,12 +208,12 @@ public void testSimpleReprocess() throws IOException private File reprocessAndValidate(File inDir, File tmpDir) throws IOException { - final File outDir = INDEX_MERGER_V9.convert( + final File outDir = indexMerger.convert( inDir, tmpDir, INDEX_SPEC ); - INDEX_IO.validateTwoSegments(persistTmpDir, outDir); + indexIO.validateTwoSegments(persistTmpDir, outDir); return outDir; } @@ -206,7 +222,7 @@ public void testIdempotentReprocess() throws IOException { final IndexableAdapter adapter = new QueryableIndexIndexableAdapter( closer.closeLater( - INDEX_IO.loadIndex( + indexIO.loadIndex( persistTmpDir ) ) @@ -216,12 +232,12 @@ public void testIdempotentReprocess() throws IOException reprocessAndValidate(persistTmpDir, tmpDir1); final File tmpDir2 = new File(tmpDir, "reprocessed2"); - final IndexableAdapter adapter2 = new QueryableIndexIndexableAdapter(closer.closeLater(INDEX_IO.loadIndex(tmpDir1))); + final IndexableAdapter adapter2 = new QueryableIndexIndexableAdapter(closer.closeLater(indexIO.loadIndex(tmpDir1))); Assert.assertEquals(events.size(), adapter2.getNumRows()); reprocessAndValidate(tmpDir1, tmpDir2); final File tmpDir3 = new File(tmpDir, "reprocessed3"); - final IndexableAdapter adapter3 = new QueryableIndexIndexableAdapter(closer.closeLater(INDEX_IO.loadIndex(tmpDir2))); + final IndexableAdapter adapter3 = new QueryableIndexIndexableAdapter(closer.closeLater(indexIO.loadIndex(tmpDir2))); Assert.assertEquals(events.size(), adapter3.getNumRows()); reprocessAndValidate(tmpDir2, tmpDir3); } diff --git a/processing/src/test/java/io/druid/segment/IndexMergerV9WithSpatialIndexTest.java b/processing/src/test/java/io/druid/segment/IndexMergerV9WithSpatialIndexTest.java index 9c17dc9078dc..7dca06cb57f6 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerV9WithSpatialIndexTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerV9WithSpatialIndexTest.java @@ -31,6 +31,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.QueryPlus; @@ -56,6 +57,7 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -66,8 +68,6 @@ @RunWith(Parameterized.class) public class IndexMergerV9WithSpatialIndexTest { - private static IndexMergerV9 INDEX_MERGER_V9 = TestHelper.getTestIndexMergerV9(); - private static IndexIO INDEX_IO = TestHelper.getTestIndexIO(); public static final int NUM_POINTS = 5000; private static Interval DATA_INTERVAL = Intervals.of("2013-01-01/2013-01-07"); @@ -82,23 +82,20 @@ public class IndexMergerV9WithSpatialIndexTest @Parameterized.Parameters public static Collection constructorFeeder() throws IOException { - final IndexSpec indexSpec = new IndexSpec(); - final IncrementalIndex rtIndex = makeIncrementalIndex(); - final QueryableIndex mMappedTestIndex = makeQueryableIndex(indexSpec); - final QueryableIndex mergedRealtimeIndex = makeMergedQueryableIndex(indexSpec); - return Arrays.asList( - new Object[][]{ - { - new IncrementalIndexSegment(rtIndex, null) - }, - { - new QueryableIndexSegment(null, mMappedTestIndex) - }, - { - new QueryableIndexSegment(null, mergedRealtimeIndex) - } - } - ); + List argumentArrays = new ArrayList<>(); + for (SegmentWriteOutMediumFactory segmentWriteOutMediumFactory : SegmentWriteOutMediumFactory.builtInFactories()) { + IndexMergerV9 indexMergerV9 = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); + IndexIO indexIO = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory); + + final IndexSpec indexSpec = new IndexSpec(); + final IncrementalIndex rtIndex = makeIncrementalIndex(); + final QueryableIndex mMappedTestIndex = makeQueryableIndex(indexSpec, indexMergerV9, indexIO); + final QueryableIndex mergedRealtimeIndex = makeMergedQueryableIndex(indexSpec, indexMergerV9, indexIO); + argumentArrays.add(new Object[] {new IncrementalIndexSegment(rtIndex, null)}); + argumentArrays.add(new Object[] {new QueryableIndexSegment(null, mMappedTestIndex)}); + argumentArrays.add(new Object[] {new QueryableIndexSegment(null, mergedRealtimeIndex)}); + } + return argumentArrays; } private static IncrementalIndex makeIncrementalIndex() throws IOException @@ -255,7 +252,8 @@ private static IncrementalIndex makeIncrementalIndex() throws IOException return theIndex; } - private static QueryableIndex makeQueryableIndex(IndexSpec indexSpec) throws IOException + private static QueryableIndex makeQueryableIndex(IndexSpec indexSpec, IndexMergerV9 indexMergerV9, IndexIO indexIO) + throws IOException { IncrementalIndex theIndex = makeIncrementalIndex(); File tmpFile = File.createTempFile("billy", "yay"); @@ -263,15 +261,19 @@ private static QueryableIndex makeQueryableIndex(IndexSpec indexSpec) throws IOE tmpFile.mkdirs(); try { - INDEX_MERGER_V9.persist(theIndex, tmpFile, indexSpec); - return INDEX_IO.loadIndex(tmpFile); + indexMergerV9.persist(theIndex, tmpFile, indexSpec, null); + return indexIO.loadIndex(tmpFile); } finally { FileUtils.deleteDirectory(tmpFile); } } - private static QueryableIndex makeMergedQueryableIndex(IndexSpec indexSpec) + private static QueryableIndex makeMergedQueryableIndex( + IndexSpec indexSpec, + IndexMergerV9 indexMergerV9, + IndexIO indexIO + ) { try { IncrementalIndex first = new IncrementalIndex.Builder() @@ -493,22 +495,23 @@ private static QueryableIndex makeMergedQueryableIndex(IndexSpec indexSpec) thirdFile.mkdirs(); mergedFile.mkdirs(); - INDEX_MERGER_V9.persist(first, DATA_INTERVAL, firstFile, indexSpec); - INDEX_MERGER_V9.persist(second, DATA_INTERVAL, secondFile, indexSpec); - INDEX_MERGER_V9.persist(third, DATA_INTERVAL, thirdFile, indexSpec); + indexMergerV9.persist(first, DATA_INTERVAL, firstFile, indexSpec, null); + indexMergerV9.persist(second, DATA_INTERVAL, secondFile, indexSpec, null); + indexMergerV9.persist(third, DATA_INTERVAL, thirdFile, indexSpec, null); try { - QueryableIndex mergedRealtime = INDEX_IO.loadIndex( - INDEX_MERGER_V9.mergeQueryableIndex( + QueryableIndex mergedRealtime = indexIO.loadIndex( + indexMergerV9.mergeQueryableIndex( Arrays.asList( - INDEX_IO.loadIndex(firstFile), - INDEX_IO.loadIndex(secondFile), - INDEX_IO.loadIndex(thirdFile) + indexIO.loadIndex(firstFile), + indexIO.loadIndex(secondFile), + indexIO.loadIndex(thirdFile) ), true, METRIC_AGGS, mergedFile, - indexSpec + indexSpec, + null ) ); return mergedRealtime; diff --git a/processing/src/test/java/io/druid/segment/IndexSpecTest.java b/processing/src/test/java/io/druid/segment/IndexSpecTest.java index 68223b5680c2..3fbd76e0c152 100644 --- a/processing/src/test/java/io/druid/segment/IndexSpecTest.java +++ b/processing/src/test/java/io/druid/segment/IndexSpecTest.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.druid.jackson.DefaultObjectMapper; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.RoaringBitmapSerdeFactory; import org.junit.Assert; import org.junit.Test; @@ -38,8 +38,8 @@ public void testSerde() throws Exception final IndexSpec spec = objectMapper.readValue(json, IndexSpec.class); Assert.assertEquals(new RoaringBitmapSerdeFactory(null), spec.getBitmapSerdeFactory()); - Assert.assertEquals(CompressedObjectStrategy.CompressionStrategy.LZ4, spec.getDimensionCompression()); - Assert.assertEquals(CompressedObjectStrategy.CompressionStrategy.LZF, spec.getMetricCompression()); + Assert.assertEquals(CompressionStrategy.LZ4, spec.getDimensionCompression()); + Assert.assertEquals(CompressionStrategy.LZF, spec.getMetricCompression()); Assert.assertEquals(CompressionFactory.LongEncodingStrategy.AUTO, spec.getLongEncoding()); Assert.assertEquals(spec, objectMapper.readValue(objectMapper.writeValueAsBytes(spec), IndexSpec.class)); @@ -53,7 +53,7 @@ public void testSerdeUncompressed() throws Exception final IndexSpec spec = objectMapper.readValue(json, IndexSpec.class); - Assert.assertEquals(CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, spec.getDimensionCompression()); + Assert.assertEquals(CompressionStrategy.UNCOMPRESSED, spec.getDimensionCompression()); Assert.assertEquals(spec, objectMapper.readValue(objectMapper.writeValueAsBytes(spec), IndexSpec.class)); } @@ -61,8 +61,8 @@ public void testSerdeUncompressed() throws Exception public void testDefaults() throws Exception { final IndexSpec spec = new IndexSpec(); - Assert.assertEquals(CompressedObjectStrategy.CompressionStrategy.LZ4, spec.getDimensionCompression()); - Assert.assertEquals(CompressedObjectStrategy.CompressionStrategy.LZ4, spec.getMetricCompression()); + Assert.assertEquals(CompressionStrategy.LZ4, spec.getDimensionCompression()); + Assert.assertEquals(CompressionStrategy.LZ4, spec.getMetricCompression()); Assert.assertEquals(CompressionFactory.LongEncodingStrategy.LONGS, spec.getLongEncoding()); } } diff --git a/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java b/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java index 2f6a1202e036..3fbf954cba84 100644 --- a/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java @@ -19,9 +19,13 @@ package io.druid.segment; +import com.google.common.collect.ImmutableList; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.segment.data.BitmapValues; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.segment.data.IncrementalIndexTest; import io.druid.segment.incremental.IncrementalIndex; @@ -29,25 +33,47 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; +import java.io.IOException; +import java.util.Collection; +@RunWith(Parameterized.class) public class QueryableIndexIndexableAdapterTest { - private final static IndexMerger INDEX_MERGER = TestHelper.getTestIndexMergerV9(); - private final static IndexIO INDEX_IO = TestHelper.getTestIndexIO(); private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( new ConciseBitmapSerdeFactory(), - CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.LZ4, + CompressionStrategy.LZ4, + CompressionStrategy.LZ4, CompressionFactory.LongEncodingStrategy.LONGS ); + + @Parameterized.Parameters + public static Collection constructorFeeder() throws IOException + { + return ImmutableList.of( + new Object[] {TmpFileSegmentWriteOutMediumFactory.instance()}, + new Object[] {OffHeapMemorySegmentWriteOutMediumFactory.instance()} + ); + } + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Rule public final CloserRule closer = new CloserRule(false); + private final IndexMerger indexMerger; + private final IndexIO indexIO; + + public QueryableIndexIndexableAdapterTest(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) + { + indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); + indexIO = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory); + } + @Test public void testGetBitmapIndex() throws Exception { @@ -57,18 +83,19 @@ public void testGetBitmapIndex() throws Exception final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( - INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexIO.loadIndex( + indexMerger.persist( toPersist, tempDir, - INDEX_SPEC + INDEX_SPEC, + null ) ) ); IndexableAdapter adapter = new QueryableIndexIndexableAdapter(index); String dimension = "dim1"; - //null is added to all dimensions with value + @SuppressWarnings("UnusedAssignment") //null is added to all dimensions with value BitmapValues bitmapValues = adapter.getBitmapValues(dimension, 0); for (int i = 0; i < adapter.getDimValueLookup(dimension).size(); i++) { bitmapValues = adapter.getBitmapValues(dimension, i); diff --git a/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerV9Test.java b/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerV9Test.java index c5e59470be9e..403c5f54ebdf 100644 --- a/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerV9Test.java +++ b/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerV9Test.java @@ -19,8 +19,9 @@ package io.druid.segment; -import io.druid.segment.data.CompressedObjectStrategy.CompressionStrategy; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.RoaringBitmapSerdeFactory; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -31,10 +32,17 @@ public class RoaringBitmapIndexMergerV9Test extends IndexMergerTestBase public RoaringBitmapIndexMergerV9Test( CompressionStrategy compressionStrategy, CompressionStrategy dimCompressionStrategy, - LongEncodingStrategy longEncodingStrategy + LongEncodingStrategy longEncodingStrategy, + SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { - super(new RoaringBitmapSerdeFactory(null), compressionStrategy, dimCompressionStrategy, longEncodingStrategy); - indexMerger = TestHelper.getTestIndexMergerV9(); + super( + new RoaringBitmapSerdeFactory(null), + compressionStrategy, + dimCompressionStrategy, + longEncodingStrategy, + segmentWriteOutMediumFactory + ); + indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); } } diff --git a/processing/src/test/java/io/druid/segment/SchemalessIndexTest.java b/processing/src/test/java/io/druid/segment/SchemalessIndexTest.java index ca2caca63f29..f31c4a75f3eb 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessIndexTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessIndexTest.java @@ -35,6 +35,7 @@ import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Comparators; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -89,9 +90,6 @@ public class SchemalessIndexTest private static final Map> mergedIndexes = Maps.newHashMap(); private static final List rowPersistedIndexes = Lists.newArrayList(); - private static final IndexMerger INDEX_MERGER = TestHelper.getTestIndexMergerV9(); - private static final IndexIO INDEX_IO = TestHelper.getTestIndexIO(); - private static IncrementalIndex index = null; private static QueryableIndex mergedIndex = null; @@ -101,6 +99,15 @@ public class SchemalessIndexTest } } + private final IndexMerger indexMerger; + private final IndexIO indexIO; + + public SchemalessIndexTest(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) + { + indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); + indexIO = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory); + } + public static IncrementalIndex getIncrementalIndex() { synchronized (log) { @@ -178,7 +185,7 @@ public static QueryableIndex getIncrementalIndex(int index1, int index2) } } - public static QueryableIndex getMergedIncrementalIndex() + public QueryableIndex getMergedIncrementalIndex() { synchronized (log) { if (mergedIndex != null) { @@ -203,16 +210,17 @@ public static QueryableIndex getMergedIncrementalIndex() mergedFile.mkdirs(); mergedFile.deleteOnExit(); - INDEX_MERGER.persist(top, topFile, indexSpec); - INDEX_MERGER.persist(bottom, bottomFile, indexSpec); + indexMerger.persist(top, topFile, indexSpec, null); + indexMerger.persist(bottom, bottomFile, indexSpec, null); - mergedIndex = INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( - Arrays.asList(INDEX_IO.loadIndex(topFile), INDEX_IO.loadIndex(bottomFile)), + mergedIndex = indexIO.loadIndex( + indexMerger.mergeQueryableIndex( + Arrays.asList(indexIO.loadIndex(topFile), indexIO.loadIndex(bottomFile)), true, METRIC_AGGS, mergedFile, - indexSpec + indexSpec, + null ) ); @@ -225,7 +233,7 @@ public static QueryableIndex getMergedIncrementalIndex() } } - public static QueryableIndex getMergedIncrementalIndex(int index1, int index2) + public QueryableIndex getMergedIncrementalIndex(int index1, int index2) { synchronized (log) { if (rowPersistedIndexes.isEmpty()) { @@ -252,13 +260,14 @@ public static QueryableIndex getMergedIncrementalIndex(int index1, int index2) mergedFile.mkdirs(); mergedFile.deleteOnExit(); - QueryableIndex index = INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + QueryableIndex index = indexIO.loadIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(rowPersistedIndexes.get(index1), rowPersistedIndexes.get(index2)), true, METRIC_AGGS, mergedFile, - indexSpec + indexSpec, + null ) ); @@ -272,7 +281,7 @@ public static QueryableIndex getMergedIncrementalIndex(int index1, int index2) } } - public static QueryableIndex getMergedIncrementalIndex(int[] indexes) + public QueryableIndex getMergedIncrementalIndex(int[] indexes) { synchronized (log) { if (rowPersistedIndexes.isEmpty()) { @@ -293,8 +302,8 @@ public static QueryableIndex getMergedIncrementalIndex(int[] indexes) indexesToMerge.add(rowPersistedIndexes.get(index)); } - return INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex(indexesToMerge, true, METRIC_AGGS, mergedFile, indexSpec) + return indexIO.loadIndex( + indexMerger.mergeQueryableIndex(indexesToMerge, true, METRIC_AGGS, mergedFile, indexSpec, null) ); } catch (IOException e) { @@ -303,7 +312,7 @@ public static QueryableIndex getMergedIncrementalIndex(int[] indexes) } } - public static QueryableIndex getAppendedIncrementalIndex( + public QueryableIndex getAppendedIncrementalIndex( Iterable> files, List intervals ) @@ -311,7 +320,7 @@ public static QueryableIndex getAppendedIncrementalIndex( return makeAppendedMMappedIndex(files, intervals); } - public static QueryableIndex getMergedIncrementalIndexDiffMetrics() + public QueryableIndex getMergedIncrementalIndexDiffMetrics() { return getMergedIncrementalIndex( Arrays.>asList( @@ -321,7 +330,7 @@ public static QueryableIndex getMergedIncrementalIndexDiffMetrics() ); } - public static QueryableIndex getMergedIncrementalIndex(Iterable> files) + public QueryableIndex getMergedIncrementalIndex(Iterable> files) { return makeMergedMMappedIndex(files); } @@ -342,7 +351,7 @@ private static void makeEvents() } } - private static void makeRowPersistedIndexes() + private void makeRowPersistedIndexes() { synchronized (log) { try { @@ -380,8 +389,8 @@ private static void makeRowPersistedIndexes() tmpFile.mkdirs(); tmpFile.deleteOnExit(); - INDEX_MERGER.persist(rowIndex, tmpFile, indexSpec); - rowPersistedIndexes.add(INDEX_IO.loadIndex(tmpFile)); + indexMerger.persist(rowIndex, tmpFile, indexSpec, null); + rowPersistedIndexes.add(indexIO.loadIndex(tmpFile)); } } catch (IOException e) { @@ -437,8 +446,7 @@ public static IncrementalIndex makeIncrementalIndex(final String resourceFilenam return retVal; } - private static List makeFilesToMap(File tmpFile, Iterable> files) - throws IOException + private List makeFilesToMap(File tmpFile, Iterable> files) throws IOException { List filesToMap = Lists.newArrayList(); for (Pair file : files) { @@ -447,13 +455,13 @@ private static List makeFilesToMap(File tmpFile, Iterable> files, final List intervals ) @@ -498,7 +506,7 @@ public IndexableAdapter apply(PartitionChunk chunk) { try { return new RowboatFilteringIndexAdapter( - new QueryableIndexIndexableAdapter(INDEX_IO.loadIndex(chunk.getObject())), + new QueryableIndexIndexableAdapter(indexIO.loadIndex(chunk.getObject())), new Predicate() { @Override @@ -521,14 +529,14 @@ public boolean apply(Rowboat input) ) ); - return INDEX_IO.loadIndex(INDEX_MERGER.append(adapters, null, mergedFile, indexSpec)); + return indexIO.loadIndex(indexMerger.append(adapters, null, mergedFile, indexSpec, null)); } catch (IOException e) { throw Throwables.propagate(e); } } - private static QueryableIndex makeMergedMMappedIndex(Iterable> files) + private QueryableIndex makeMergedMMappedIndex(Iterable> files) { try { File tmpFile = File.createTempFile("yay", "who"); @@ -539,8 +547,8 @@ private static QueryableIndex makeMergedMMappedIndex(Iterable filesToMap = makeFilesToMap(tmpFile, files); - return INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + return indexIO.loadIndex( + indexMerger.mergeQueryableIndex( Lists.newArrayList( Iterables.transform( filesToMap, @@ -550,7 +558,7 @@ private static QueryableIndex makeMergedMMappedIndex(Iterable constructorFeeder() throws IOException + { + return ImmutableList.of( + new Object[] {TmpFileSegmentWriteOutMediumFactory.instance()}, + new Object[] {OffHeapMemorySegmentWriteOutMediumFactory.instance()} + ); + } + final double UNIQUES_2 = 2.000977198748901d; final double UNIQUES_1 = 1.0002442201269182d; + final SchemalessIndexTest schemalessIndexTest; final String dataSource = "testing"; final Granularity allGran = Granularities.ALL; final String marketDimension = "market"; @@ -95,6 +114,11 @@ public class SchemalessTestFullTest Arrays.asList(Intervals.of("1970-01-01T00:00:00.000Z/2020-01-01T00:00:00.000Z")) ); + public SchemalessTestFullTest(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) + { + schemalessIndexTest = new SchemalessIndexTest(segmentWriteOutMediumFactory); + } + @Test public void testCompleteIntersectingSchemas() { @@ -927,7 +951,7 @@ public void testEmptySchemas() runTests( new QueryableIndexSegment( - null, SchemalessIndexTest.getMergedIncrementalIndex(0, 0) + null, schemalessIndexTest.getMergedIncrementalIndex(0, 0) ), expectedTimeseriesResults, expectedFilteredTimeSeriesResults, @@ -1012,7 +1036,7 @@ public void testExactSameSchemas() runTests( new QueryableIndexSegment( - null, SchemalessIndexTest.getMergedIncrementalIndex(1, 1) + null, schemalessIndexTest.getMergedIncrementalIndex(1, 1) ), expectedTimeseriesResults, expectedFilteredTimeSeriesResults, @@ -1143,7 +1167,7 @@ public void testMultiDimensionalValues() ); runTests( - new QueryableIndexSegment(null, SchemalessIndexTest.getMergedIncrementalIndex(new int[]{6, 7, 8})), + new QueryableIndexSegment(null, schemalessIndexTest.getMergedIncrementalIndex(new int[]{6, 7, 8})), expectedTimeseriesResults, expectedFilteredTimeSeriesResults, expectedTopNResults, @@ -1334,7 +1358,7 @@ public void testDifferentMetrics() ); runTests( - new QueryableIndexSegment(null, SchemalessIndexTest.getMergedIncrementalIndexDiffMetrics()), + new QueryableIndexSegment(null, schemalessIndexTest.getMergedIncrementalIndexDiffMetrics()), expectedTimeseriesResults, expectedFilteredTimeSeriesResults, expectedTopNResults, @@ -1359,11 +1383,11 @@ private List> getIndexes(int index1, int index2) StringUtils.format("Failed: II[%,d, %,d]", index2, index1) ), new Pair<>( - SchemalessIndexTest.getMergedIncrementalIndex(index1, index2), + schemalessIndexTest.getMergedIncrementalIndex(index1, index2), StringUtils.format("Failed: MII[%,d, %,d]", index1, index2) ), new Pair<>( - SchemalessIndexTest.getMergedIncrementalIndex(index2, index1), + schemalessIndexTest.getMergedIncrementalIndex(index2, index1), StringUtils.format("Failed: MII[%,d, %,d]", index2, index1) ) ); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java index f634c71efd4f..3f16b1602c58 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java @@ -26,6 +26,7 @@ import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.granularity.Granularity; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.QueryPlus; import io.druid.query.QueryRunner; @@ -60,6 +61,7 @@ import org.junit.runners.Parameterized; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -73,27 +75,17 @@ public class SchemalessTestSimpleTest @Parameterized.Parameters public static Collection constructorFeeder() throws IOException { - final IncrementalIndex incrementalIndex = SchemalessIndexTest.getIncrementalIndex(); - final QueryableIndex persistedIncrementalIndex = TestIndex.persistRealtimeAndLoadMMapped(incrementalIndex); - final QueryableIndex mergedIncrementalIndex = SchemalessIndexTest.getMergedIncrementalIndex(); - - return Arrays.asList( - new Object[][]{ - { - new IncrementalIndexSegment(incrementalIndex, null) - }, - { - new QueryableIndexSegment( - null, persistedIncrementalIndex - ) - }, - { - new QueryableIndexSegment( - null, mergedIncrementalIndex - ) - } - } - ); + List argumentArrays = new ArrayList<>(); + for (SegmentWriteOutMediumFactory segmentWriteOutMediumFactory : SegmentWriteOutMediumFactory.builtInFactories()) { + SchemalessIndexTest schemalessIndexTest = new SchemalessIndexTest(segmentWriteOutMediumFactory); + final IncrementalIndex incrementalIndex = SchemalessIndexTest.getIncrementalIndex(); + final QueryableIndex persistedIncrementalIndex = TestIndex.persistRealtimeAndLoadMMapped(incrementalIndex); + final QueryableIndex mergedIncrementalIndex = schemalessIndexTest.getMergedIncrementalIndex(); + argumentArrays.add(new Object[] {new IncrementalIndexSegment(incrementalIndex, null)}); + argumentArrays.add(new Object[] {new QueryableIndexSegment(null, persistedIncrementalIndex)}); + argumentArrays.add(new Object[] {new QueryableIndexSegment(null, mergedIncrementalIndex)}); + } + return argumentArrays; } final String dataSource = "testing"; diff --git a/processing/src/test/java/io/druid/segment/StringDimensionHandlerTest.java b/processing/src/test/java/io/druid/segment/StringDimensionHandlerTest.java index 6103abfd8678..acc9abd57937 100644 --- a/processing/src/test/java/io/druid/segment/StringDimensionHandlerTest.java +++ b/processing/src/test/java/io/druid/segment/StringDimensionHandlerTest.java @@ -25,8 +25,8 @@ import io.druid.java.util.common.Intervals; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.CountAggregatorFactory; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.segment.data.Indexed; import io.druid.segment.incremental.IncrementalIndex; @@ -49,8 +49,8 @@ public class StringDimensionHandlerTest private static final IndexSpec INDEX_SPEC = new IndexSpec( new ConciseBitmapSerdeFactory(), - CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.LZ4, + CompressionStrategy.LZ4, + CompressionStrategy.LZ4, CompressionFactory.LongEncodingStrategy.LONGS ); diff --git a/processing/src/test/java/io/druid/segment/TestHelper.java b/processing/src/test/java/io/druid/segment/TestHelper.java index 69fd684b4bdf..950ed7961f36 100644 --- a/processing/src/test/java/io/druid/segment/TestHelper.java +++ b/processing/src/test/java/io/druid/segment/TestHelper.java @@ -31,6 +31,7 @@ import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.Result; import io.druid.query.expression.TestExprMacroTable; import io.druid.query.timeseries.TimeseriesResultValue; @@ -48,13 +49,18 @@ */ public class TestHelper { - private static final IndexMergerV9 INDEX_MERGER_V9; - private static final IndexIO INDEX_IO; + private static final ObjectMapper JSON_MAPPER = getJsonMapper(); - static { - final ObjectMapper jsonMapper = getJsonMapper(); - INDEX_IO = new IndexIO( - jsonMapper, + public static IndexMergerV9 getTestIndexMergerV9(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) + { + return new IndexMergerV9(JSON_MAPPER, getTestIndexIO(segmentWriteOutMediumFactory), segmentWriteOutMediumFactory); + } + + public static IndexIO getTestIndexIO(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) + { + return new IndexIO( + JSON_MAPPER, + segmentWriteOutMediumFactory, new ColumnConfig() { @Override @@ -64,17 +70,6 @@ public int columnCacheSizeBytes() } } ); - INDEX_MERGER_V9 = new IndexMergerV9(jsonMapper, INDEX_IO); - } - - public static IndexMergerV9 getTestIndexMergerV9() - { - return INDEX_MERGER_V9; - } - - public static IndexIO getTestIndexIO() - { - return INDEX_IO; } public static ObjectMapper getJsonMapper() diff --git a/processing/src/test/java/io/druid/segment/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java index 4271eaa213b4..a6b03a6dd0d6 100644 --- a/processing/src/test/java/io/druid/segment/TestIndex.java +++ b/processing/src/test/java/io/druid/segment/TestIndex.java @@ -37,6 +37,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.DoubleMaxAggregatorFactory; import io.druid.query.aggregation.DoubleMinAggregatorFactory; @@ -135,8 +136,9 @@ public class TestIndex }; private static final IndexSpec indexSpec = new IndexSpec(); - private static final IndexMerger INDEX_MERGER = TestHelper.getTestIndexMergerV9(); - private static final IndexIO INDEX_IO = TestHelper.getTestIndexIO(); + private static final IndexMerger INDEX_MERGER = + TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()); + private static final IndexIO INDEX_IO = TestHelper.getTestIndexIO(OffHeapMemorySegmentWriteOutMediumFactory.instance()); static { if (ComplexMetrics.getSerdeForType("hyperUnique") == null) { @@ -225,8 +227,8 @@ public static QueryableIndex mergedRealtimeIndex() mergedFile.mkdirs(); mergedFile.deleteOnExit(); - INDEX_MERGER.persist(top, DATA_INTERVAL, topFile, indexSpec); - INDEX_MERGER.persist(bottom, DATA_INTERVAL, bottomFile, indexSpec); + INDEX_MERGER.persist(top, DATA_INTERVAL, topFile, indexSpec, null); + INDEX_MERGER.persist(bottom, DATA_INTERVAL, bottomFile, indexSpec, null); mergedRealtime = INDEX_IO.loadIndex( INDEX_MERGER.mergeQueryableIndex( @@ -234,7 +236,8 @@ public static QueryableIndex mergedRealtimeIndex() true, METRIC_AGGS, mergedFile, - indexSpec + indexSpec, + null ) ); @@ -362,7 +365,7 @@ public static QueryableIndex persistRealtimeAndLoadMMapped(IncrementalIndex inde someTmpFile.mkdirs(); someTmpFile.deleteOnExit(); - INDEX_MERGER.persist(index, someTmpFile, indexSpec); + INDEX_MERGER.persist(index, someTmpFile, indexSpec, null); return INDEX_IO.loadIndex(someTmpFile); } catch (IOException e) { diff --git a/processing/src/test/java/io/druid/segment/data/CompressedFloatsSerdeTest.java b/processing/src/test/java/io/druid/segment/data/CompressedFloatsSerdeTest.java index 16c58c01a248..5eaa9e55a29d 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedFloatsSerdeTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedFloatsSerdeTest.java @@ -20,10 +20,10 @@ package io.druid.segment.data; import com.google.common.base.Supplier; -import com.google.common.io.ByteSink; import com.google.common.primitives.Floats; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; import it.unimi.dsi.fastutil.ints.IntArrays; import org.junit.Assert; import org.junit.Test; @@ -32,7 +32,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.Channels; @@ -50,7 +49,7 @@ public class CompressedFloatsSerdeTest public static Iterable compressionStrategies() { List data = new ArrayList<>(); - for (CompressedObjectStrategy.CompressionStrategy strategy : CompressedObjectStrategy.CompressionStrategy.values()) { + for (CompressionStrategy strategy : CompressionStrategy.values()) { data.add(new Object[]{strategy, ByteOrder.BIG_ENDIAN}); data.add(new Object[]{strategy, ByteOrder.LITTLE_ENDIAN}); } @@ -59,7 +58,7 @@ public static Iterable compressionStrategies() private static final double DELTA = 0.00001; - protected final CompressedObjectStrategy.CompressionStrategy compressionStrategy; + protected final CompressionStrategy compressionStrategy; protected final ByteOrder order; private final float values0[] = {}; @@ -75,7 +74,7 @@ public static Iterable compressionStrategies() }; public CompressedFloatsSerdeTest( - CompressedObjectStrategy.CompressionStrategy compressionStrategy, + CompressionStrategy compressionStrategy, ByteOrder order ) { @@ -108,7 +107,11 @@ public void testChunkSerde() throws Exception public void testWithValues(float[] values) throws Exception { - FloatSupplierSerializer serializer = CompressionFactory.getFloatSerializer(new IOPeonForTesting(), "test", order, compressionStrategy + FloatSupplierSerializer serializer = CompressionFactory.getFloatSerializer( + new OffHeapMemorySegmentWriteOutMedium(), + "test", + order, + compressionStrategy ); serializer.open(); @@ -118,19 +121,10 @@ public void testWithValues(float[] values) throws Exception Assert.assertEquals(values.length, serializer.size()); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - serializer.closeAndConsolidate( - new ByteSink() - { - @Override - public OutputStream openStream() throws IOException - { - return baos; - } - } - ); + serializer.writeTo(Channels.newChannel(baos), null); Assert.assertEquals(baos.size(), serializer.getSerializedSize()); CompressedFloatsIndexedSupplier supplier = CompressedFloatsIndexedSupplier - .fromByteBuffer(ByteBuffer.wrap(baos.toByteArray()), order, null); + .fromByteBuffer(ByteBuffer.wrap(baos.toByteArray()), order); IndexedFloats floats = supplier.get(); assertIndexMatchesVals(floats, values); @@ -180,12 +174,12 @@ private void assertIndexMatchesVals(IndexedFloats indexed, float[] vals) private void testSupplierSerde(CompressedFloatsIndexedSupplier supplier, float[] vals) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - supplier.writeToChannel(Channels.newChannel(baos)); + supplier.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(supplier.getSerializedSize(), bytes.length); CompressedFloatsIndexedSupplier anotherSupplier = CompressedFloatsIndexedSupplier.fromByteBuffer( - ByteBuffer.wrap(bytes), order, null + ByteBuffer.wrap(bytes), order ); IndexedFloats indexed = anotherSupplier.get(); assertIndexMatchesVals(indexed, vals); diff --git a/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedSupplierTest.java b/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedSupplierTest.java index 455b1af0d31f..adc3f8c40c31 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedSupplierTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedSupplierTest.java @@ -23,6 +23,7 @@ import com.google.common.primitives.Longs; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.java.util.common.io.Closer; import io.druid.segment.CompressedPools; import it.unimi.dsi.fastutil.ints.IntArrays; import org.junit.After; @@ -44,11 +45,12 @@ public class CompressedIntsIndexedSupplierTest extends CompressionStrategyTest { - public CompressedIntsIndexedSupplierTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy) + public CompressedIntsIndexedSupplierTest(CompressionStrategy compressionStrategy) { super(compressionStrategy); } + private Closer closer; private IndexedInts indexed; private CompressedIntsIndexedSupplier supplier; private int[] vals; @@ -56,6 +58,7 @@ public CompressedIntsIndexedSupplierTest(CompressedObjectStrategy.CompressionStr @Before public void setUp() throws Exception { + closer = Closer.create(); CloseQuietly.close(indexed); indexed = null; supplier = null; @@ -65,6 +68,7 @@ public void setUp() throws Exception @After public void tearDown() throws Exception { + closer.close(); CloseQuietly.close(indexed); } @@ -78,7 +82,8 @@ private void setupSimple(final int chunkSize) IntBuffer.wrap(vals), chunkSize, ByteOrder.nativeOrder(), - compressionStrategy + compressionStrategy, + closer ); indexed = supplier.get(); @@ -97,14 +102,14 @@ private void makeWithSerde(final int chunkSize) throws IOException ByteArrayOutputStream baos = new ByteArrayOutputStream(); final CompressedIntsIndexedSupplier theSupplier = CompressedIntsIndexedSupplier.fromIntBuffer( - IntBuffer.wrap(vals), chunkSize, ByteOrder.nativeOrder(), compressionStrategy + IntBuffer.wrap(vals), chunkSize, ByteOrder.nativeOrder(), compressionStrategy, closer ); - theSupplier.writeToChannel(Channels.newChannel(baos)); + theSupplier.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(theSupplier.getSerializedSize(), bytes.length); - supplier = CompressedIntsIndexedSupplier.fromByteBuffer(ByteBuffer.wrap(bytes), ByteOrder.nativeOrder(), null); + supplier = CompressedIntsIndexedSupplier.fromByteBuffer(ByteBuffer.wrap(bytes), ByteOrder.nativeOrder()); indexed = supplier.get(); } diff --git a/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedWriterTest.java b/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedWriterTest.java index 303f10c8b9cf..3a2441e4f51e 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedWriterTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedIntsIndexedWriterTest.java @@ -22,7 +22,6 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; -import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; @@ -30,6 +29,10 @@ import io.druid.java.util.common.io.smoosh.Smoosh; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; +import io.druid.segment.writeout.WriteOutBytes; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import it.unimi.dsi.fastutil.ints.IntArrayList; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.After; @@ -42,8 +45,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; import java.nio.file.Files; import java.util.List; import java.util.Random; @@ -56,13 +57,14 @@ public class CompressedIntsIndexedWriterTest { private static final int[] MAX_VALUES = new int[]{0xFF, 0xFFFF, 0xFFFFFF, 0x0FFFFFFF}; private static final int[] CHUNK_FACTORS = new int[]{1, 2, 100, CompressedIntsIndexedSupplier.MAX_INTS_IN_BUFFER}; - private final IOPeon ioPeon = new TmpFileIOPeon(); - private final CompressedObjectStrategy.CompressionStrategy compressionStrategy; + private final SegmentWriteOutMedium segmentWriteOutMedium = new OffHeapMemorySegmentWriteOutMedium(); + private final CompressionStrategy compressionStrategy; private final ByteOrder byteOrder; private final Random rand = new Random(0); private int[] vals; + public CompressedIntsIndexedWriterTest( - CompressedObjectStrategy.CompressionStrategy compressionStrategy, + CompressionStrategy compressionStrategy, ByteOrder byteOrder ) { @@ -74,7 +76,7 @@ public CompressedIntsIndexedWriterTest( public static Iterable compressionStrategiesAndByteOrders() { Set> combinations = Sets.cartesianProduct( - Sets.newHashSet(CompressedObjectStrategy.CompressionStrategy.noNoneValues()), + Sets.newHashSet(CompressionStrategy.noNoneValues()), Sets.newHashSet(ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN) ); @@ -99,7 +101,7 @@ public void setUp() throws Exception @After public void tearDown() throws Exception { - ioPeon.close(); + segmentWriteOutMedium.close(); } private void generateVals(final int totalSize, final int maxValue) throws IOException @@ -115,29 +117,30 @@ private void checkSerializedSizeAndData(int chunkFactor) throws Exception FileSmoosher smoosher = new FileSmoosher(FileUtils.getTempDirectory()); CompressedIntsIndexedWriter writer = new CompressedIntsIndexedWriter( - ioPeon, "test", chunkFactor, byteOrder, compressionStrategy + segmentWriteOutMedium, "test", chunkFactor, byteOrder, compressionStrategy ); CompressedIntsIndexedSupplier supplierFromList = CompressedIntsIndexedSupplier.fromList( - Ints.asList(vals), chunkFactor, byteOrder, compressionStrategy + IntArrayList.wrap(vals), + chunkFactor, + byteOrder, + compressionStrategy, + segmentWriteOutMedium.getCloser() ); writer.open(); for (int val : vals) { writer.add(val); } - writer.close(); long writtenLength = writer.getSerializedSize(); - final WritableByteChannel outputChannel = Channels.newChannel(ioPeon.makeOutputStream("output")); - writer.writeToChannel(outputChannel, smoosher); - outputChannel.close(); + final WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); + writer.writeTo(writeOutBytes, smoosher); smoosher.close(); assertEquals(writtenLength, supplierFromList.getSerializedSize()); // read from ByteBuffer and check values CompressedIntsIndexedSupplier supplierFromByteBuffer = CompressedIntsIndexedSupplier.fromByteBuffer( - ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), - byteOrder, - null + ByteBuffer.wrap(IOUtils.toByteArray(writeOutBytes.asInputStream())), + byteOrder ); IndexedInts indexedInts = supplierFromByteBuffer.get(); assertEquals(vals.length, indexedInts.size()); @@ -187,51 +190,43 @@ private void checkV2SerializedSizeAndData(int chunkFactor) throws Exception )).toFile(); FileSmoosher smoosher = new FileSmoosher(tmpDirectory); - final IOPeon ioPeon = new TmpFileIOPeon(); - try { - - CompressedIntsIndexedWriter writer = new CompressedIntsIndexedWriter( - chunkFactor, - compressionStrategy, - new GenericIndexedWriter<>( - ioPeon, "test", - CompressedIntBufferObjectStrategy.getBufferForOrder(byteOrder, compressionStrategy, - chunkFactor - ), Longs.BYTES * 10000 - ) - ); - - writer.open(); - for (int val : vals) { - writer.add(val); - } - writer.close(); - final SmooshedWriter channel = smoosher.addWithSmooshedWriter( - "test", writer.getSerializedSize() - ); - writer.writeToChannel(channel, smoosher); - channel.close(); - smoosher.close(); - - SmooshedFileMapper mapper = Smoosh.map(tmpDirectory); - - // read from ByteBuffer and check values - CompressedIntsIndexedSupplier supplierFromByteBuffer = CompressedIntsIndexedSupplier.fromByteBuffer( - mapper.mapFile("test"), - byteOrder, - mapper - ); - IndexedInts indexedInts = supplierFromByteBuffer.get(); - assertEquals(vals.length, indexedInts.size()); - for (int i = 0; i < vals.length; ++i) { - assertEquals(vals[i], indexedInts.get(i)); - } - CloseQuietly.close(indexedInts); - mapper.close(); + + CompressedIntsIndexedWriter writer = new CompressedIntsIndexedWriter( + segmentWriteOutMedium, + chunkFactor, + byteOrder, + compressionStrategy, + GenericIndexedWriter.ofCompressedByteBuffers( + segmentWriteOutMedium, + "test", + compressionStrategy, + Longs.BYTES * 10000 + ) + ); + + writer.open(); + for (int val : vals) { + writer.add(val); } - finally { - ioPeon.close(); + final SmooshedWriter channel = smoosher.addWithSmooshedWriter("test", writer.getSerializedSize()); + writer.writeTo(channel, smoosher); + channel.close(); + smoosher.close(); + + SmooshedFileMapper mapper = Smoosh.map(tmpDirectory); + + // read from ByteBuffer and check values + CompressedIntsIndexedSupplier supplierFromByteBuffer = CompressedIntsIndexedSupplier.fromByteBuffer( + mapper.mapFile("test"), + byteOrder + ); + IndexedInts indexedInts = supplierFromByteBuffer.get(); + assertEquals(vals.length, indexedInts.size()); + for (int i = 0; i < vals.length; ++i) { + assertEquals(vals[i], indexedInts.get(i)); } + CloseQuietly.close(indexedInts); + mapper.close(); } @Test diff --git a/processing/src/test/java/io/druid/segment/data/CompressedLongsSerdeTest.java b/processing/src/test/java/io/druid/segment/data/CompressedLongsSerdeTest.java index 3583706973d4..625b0c2f8182 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedLongsSerdeTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedLongsSerdeTest.java @@ -20,10 +20,10 @@ package io.druid.segment.data; import com.google.common.base.Supplier; -import com.google.common.io.ByteSink; import com.google.common.primitives.Longs; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; import it.unimi.dsi.fastutil.ints.IntArrays; import org.junit.Assert; import org.junit.Test; @@ -32,7 +32,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.Channels; @@ -51,7 +50,7 @@ public static Iterable compressionStrategies() { List data = new ArrayList<>(); for (CompressionFactory.LongEncodingStrategy encodingStrategy: CompressionFactory.LongEncodingStrategy.values()) { - for (CompressedObjectStrategy.CompressionStrategy strategy : CompressedObjectStrategy.CompressionStrategy.values()) { + for (CompressionStrategy strategy : CompressionStrategy.values()) { data.add(new Object[]{encodingStrategy, strategy, ByteOrder.BIG_ENDIAN}); data.add(new Object[]{encodingStrategy, strategy, ByteOrder.LITTLE_ENDIAN}); } @@ -60,7 +59,7 @@ public static Iterable compressionStrategies() } protected final CompressionFactory.LongEncodingStrategy encodingStrategy; - protected final CompressedObjectStrategy.CompressionStrategy compressionStrategy; + protected final CompressionStrategy compressionStrategy; protected final ByteOrder order; private final long values0[] = {}; @@ -89,7 +88,7 @@ private static long[] addUniques(long[] val) public CompressedLongsSerdeTest( CompressionFactory.LongEncodingStrategy encodingStrategy, - CompressedObjectStrategy.CompressionStrategy compressionStrategy, + CompressionStrategy compressionStrategy, ByteOrder order ) { @@ -130,8 +129,12 @@ public void testWithValues(long[] values) throws Exception public void testValues(long[] values) throws Exception { - LongSupplierSerializer serializer = CompressionFactory.getLongSerializer(new IOPeonForTesting(), "test", order, - encodingStrategy, compressionStrategy + LongSupplierSerializer serializer = CompressionFactory.getLongSerializer( + new OffHeapMemorySegmentWriteOutMedium(), + "test", + order, + encodingStrategy, + compressionStrategy ); serializer.open(); @@ -141,19 +144,10 @@ public void testValues(long[] values) throws Exception Assert.assertEquals(values.length, serializer.size()); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - serializer.closeAndConsolidate( - new ByteSink() - { - @Override - public OutputStream openStream() throws IOException - { - return baos; - } - } - ); + serializer.writeTo(Channels.newChannel(baos), null); Assert.assertEquals(baos.size(), serializer.getSerializedSize()); CompressedLongsIndexedSupplier supplier = CompressedLongsIndexedSupplier - .fromByteBuffer(ByteBuffer.wrap(baos.toByteArray()), order, null); + .fromByteBuffer(ByteBuffer.wrap(baos.toByteArray()), order); IndexedLongs longs = supplier.get(); assertIndexMatchesVals(longs, values); @@ -203,14 +197,13 @@ private void assertIndexMatchesVals(IndexedLongs indexed, long[] vals) private void testSupplierSerde(CompressedLongsIndexedSupplier supplier, long[] vals) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - supplier.writeToChannel(Channels.newChannel(baos)); + supplier.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(supplier.getSerializedSize(), bytes.length); CompressedLongsIndexedSupplier anotherSupplier = CompressedLongsIndexedSupplier.fromByteBuffer( ByteBuffer.wrap(bytes), - order, - null + order ); IndexedLongs indexed = anotherSupplier.get(); assertIndexMatchesVals(indexed, vals); diff --git a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedSupplierTest.java b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedSupplierTest.java index bd380cb110e8..9a98be6c10e9 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedSupplierTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedSupplierTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; +import io.druid.java.util.common.io.Closer; import io.druid.segment.CompressedVSizeIndexedSupplier; import org.junit.After; import org.junit.Assert; @@ -40,6 +41,7 @@ */ public class CompressedVSizeIndexedSupplierTest { + private Closer closer; protected List vals; protected WritableSupplier> indexedSupplier; @@ -47,6 +49,7 @@ public class CompressedVSizeIndexedSupplierTest @Before public void setUpSimple() { + closer = Closer.create(); vals = Arrays.asList( new int[1], new int[]{1, 2, 3, 4, 5}, @@ -65,16 +68,20 @@ public IndexedInts apply(int[] input) return VSizeIndexedInts.fromArray(input, 20); } } - ), 20, ByteOrder.nativeOrder(), - CompressedObjectStrategy.CompressionStrategy.LZ4 + ), + 20, + ByteOrder.nativeOrder(), + CompressionStrategy.LZ4, + closer ); } @After - public void teardown() + public void teardown() throws IOException { indexedSupplier = null; vals = null; + closer.close(); } @Test @@ -87,7 +94,7 @@ public void testSanity() throws Exception public void testSerde() throws IOException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - indexedSupplier.writeToChannel(Channels.newChannel(baos)); + indexedSupplier.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(indexedSupplier.getSerializedSize(), bytes.length); @@ -139,8 +146,7 @@ protected WritableSupplier> fromByteBuffer(ByteBu { return CompressedVSizeIndexedSupplier.fromByteBuffer( buffer, - ByteOrder.nativeOrder(), - null + ByteOrder.nativeOrder() ); } } diff --git a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedV3WriterTest.java b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedV3WriterTest.java index cf52bab20541..b3f86aef9211 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedV3WriterTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIndexedV3WriterTest.java @@ -32,9 +32,11 @@ import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; import io.druid.segment.CompressedVSizeIndexedV3Supplier; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.writeout.WriteOutBytes; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,8 +47,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; import java.nio.file.Files; import java.util.ArrayList; import java.util.List; @@ -65,14 +65,13 @@ public class CompressedVSizeIndexedV3WriterTest CompressedIntsIndexedSupplier.MAX_INTS_IN_BUFFER }; private static final int[] MAX_VALUES = new int[]{0xFF, 0xFFFF, 0xFFFFFF, 0x0FFFFFFF}; - private final IOPeon ioPeon = new TmpFileIOPeon(); - private final CompressedObjectStrategy.CompressionStrategy compressionStrategy; + private final CompressionStrategy compressionStrategy; private final ByteOrder byteOrder; private final Random rand = new Random(0); private List vals; public CompressedVSizeIndexedV3WriterTest( - CompressedObjectStrategy.CompressionStrategy compressionStrategy, + CompressionStrategy compressionStrategy, ByteOrder byteOrder ) { @@ -84,7 +83,7 @@ public CompressedVSizeIndexedV3WriterTest( public static Iterable compressionStrategiesAndByteOrders() { Set> combinations = Sets.cartesianProduct( - Sets.newHashSet(CompressedObjectStrategy.CompressionStrategy.noNoneValues()), + Sets.newHashSet(CompressionStrategy.noNoneValues()), Sets.newHashSet(ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN) ); @@ -116,59 +115,51 @@ private void generateVals(final int totalSize, final int maxValue) throws IOExce private void checkSerializedSizeAndData(int offsetChunkFactor, int valueChunkFactor) throws Exception { FileSmoosher smoosher = new FileSmoosher(FileUtils.getTempDirectory()); - final IndexedMultivalue indexedMultivalue; - try (IOPeon ioPeon = new TmpFileIOPeon()) { + try (SegmentWriteOutMedium segmentWriteOutMedium = new OffHeapMemorySegmentWriteOutMedium()) { int maxValue = vals.size() > 0 ? getMaxValue(vals) : 0; CompressedIntsIndexedWriter offsetWriter = new CompressedIntsIndexedWriter( - ioPeon, "offset", offsetChunkFactor, byteOrder, compressionStrategy + segmentWriteOutMedium, "offset", offsetChunkFactor, byteOrder, compressionStrategy ); CompressedVSizeIntsIndexedWriter valueWriter = new CompressedVSizeIntsIndexedWriter( - ioPeon, "value", maxValue, valueChunkFactor, byteOrder, compressionStrategy + segmentWriteOutMedium, "value", maxValue, valueChunkFactor, byteOrder, compressionStrategy ); CompressedVSizeIndexedV3Writer writer = new CompressedVSizeIndexedV3Writer(offsetWriter, valueWriter); CompressedVSizeIndexedV3Supplier supplierFromIterable = CompressedVSizeIndexedV3Supplier.fromIterable( - Iterables.transform( - vals, new Function() - { - @Nullable - @Override - public IndexedInts apply(@Nullable final int[] input) - { - return ArrayBasedIndexedInts.of(input); - } - } - ), offsetChunkFactor, maxValue, byteOrder, compressionStrategy + Iterables.transform(vals, ArrayBasedIndexedInts::of), + offsetChunkFactor, + maxValue, + byteOrder, + compressionStrategy, + segmentWriteOutMedium.getCloser() ); writer.open(); for (int[] val : vals) { writer.add(val); } - writer.close(); long writtenLength = writer.getSerializedSize(); - final WritableByteChannel outputChannel = Channels.newChannel(ioPeon.makeOutputStream("output")); - writer.writeToChannel(outputChannel, smoosher); - outputChannel.close(); + final WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); + writer.writeTo(writeOutBytes, smoosher); smoosher.close(); assertEquals(writtenLength, supplierFromIterable.getSerializedSize()); // read from ByteBuffer and check values CompressedVSizeIndexedV3Supplier supplierFromByteBuffer = CompressedVSizeIndexedV3Supplier.fromByteBuffer( - ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), - byteOrder, - null + ByteBuffer.wrap(IOUtils.toByteArray(writeOutBytes.asInputStream())), + byteOrder ); - indexedMultivalue = supplierFromByteBuffer.get(); - assertEquals(indexedMultivalue.size(), vals.size()); - for (int i = 0; i < vals.size(); ++i) { - IndexedInts subVals = indexedMultivalue.get(i); - assertEquals(subVals.size(), vals.get(i).length); - for (int j = 0; j < subVals.size(); ++j) { - assertEquals(subVals.get(j), vals.get(i)[j]); + + try (final IndexedMultivalue indexedMultivalue = supplierFromByteBuffer.get()) { + assertEquals(indexedMultivalue.size(), vals.size()); + for (int i = 0; i < vals.size(); ++i) { + IndexedInts subVals = indexedMultivalue.get(i); + assertEquals(subVals.size(), vals.get(i).length); + for (int j = 0; j < subVals.size(); ++j) { + assertEquals(subVals.get(j), vals.get(i)[j]); + } } } - CloseQuietly.close(indexedMultivalue); } } @@ -195,12 +186,6 @@ public void setUp() throws Exception vals = null; } - @After - public void tearDown() throws Exception - { - ioPeon.close(); - } - @Test public void testSmallData() throws Exception { @@ -244,33 +229,28 @@ private void checkV2SerializedSizeAndData(int offsetChunkFactor, int valueChunkF FileSmoosher smoosher = new FileSmoosher(tmpDirectory); int maxValue = vals.size() > 0 ? getMaxValue(vals) : 0; - try (IOPeon ioPeon = new TmpFileIOPeon()) { + try (SegmentWriteOutMedium segmentWriteOutMedium = new OffHeapMemorySegmentWriteOutMedium()) { CompressedIntsIndexedWriter offsetWriter = new CompressedIntsIndexedWriter( + segmentWriteOutMedium, offsetChunkFactor, + byteOrder, compressionStrategy, - new GenericIndexedWriter<>( - ioPeon, "offset", - CompressedIntBufferObjectStrategy.getBufferForOrder( - byteOrder, - compressionStrategy, - offsetChunkFactor - ), + GenericIndexedWriter.ofCompressedByteBuffers( + segmentWriteOutMedium, + "offset", + compressionStrategy, Longs.BYTES * 250000 ) ); - GenericIndexedWriter genericIndexed = new GenericIndexedWriter<>( - ioPeon, + GenericIndexedWriter genericIndexed = GenericIndexedWriter.ofCompressedByteBuffers( + segmentWriteOutMedium, "value", - CompressedByteBufferObjectStrategy.getBufferForOrder( - byteOrder, - compressionStrategy, - valueChunkFactor * VSizeIndexedInts.getNumBytesForMax(maxValue) - + CompressedVSizeIntsIndexedSupplier.bufferPadding(VSizeIndexedInts.getNumBytesForMax(maxValue)) - ), + compressionStrategy, Longs.BYTES * 250000 ); CompressedVSizeIntsIndexedWriter valueWriter = new CompressedVSizeIntsIndexedWriter( + segmentWriteOutMedium, maxValue, valueChunkFactor, byteOrder, @@ -282,21 +262,16 @@ private void checkV2SerializedSizeAndData(int offsetChunkFactor, int valueChunkF for (int[] val : vals) { writer.add(val); } - writer.close(); - final SmooshedWriter channel = smoosher.addWithSmooshedWriter( - "test", - writer.getSerializedSize() - ); - writer.writeToChannel(channel, smoosher); + final SmooshedWriter channel = smoosher.addWithSmooshedWriter("test", writer.getSerializedSize()); + writer.writeTo(channel, smoosher); channel.close(); smoosher.close(); SmooshedFileMapper mapper = Smoosh.map(tmpDirectory); CompressedVSizeIndexedV3Supplier supplierFromByteBuffer = CompressedVSizeIndexedV3Supplier.fromByteBuffer( mapper.mapFile("test"), - byteOrder, - mapper + byteOrder ); IndexedMultivalue indexedMultivalue = supplierFromByteBuffer.get(); assertEquals(indexedMultivalue.size(), vals.size()); diff --git a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplierTest.java b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplierTest.java index d33d7e48383b..c22a58f9e0ff 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplierTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedSupplierTest.java @@ -26,7 +26,9 @@ import com.google.common.primitives.Longs; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.java.util.common.io.Closer; import io.druid.segment.CompressedPools; +import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntArrays; import org.junit.After; import org.junit.Assert; @@ -54,14 +56,14 @@ public class CompressedVSizeIntsIndexedSupplierTest extends CompressionStrategyT @Parameterized.Parameters(name = "{index}: compression={0}, byteOrder={1}") public static Iterable compressionStrategies() { - final Iterable compressionStrategies = Iterables.transform( + final Iterable compressionStrategies = Iterables.transform( CompressionStrategyTest.compressionStrategies(), - new Function() + new Function() { @Override - public CompressedObjectStrategy.CompressionStrategy apply(Object[] input) + public CompressionStrategy apply(Object[] input) { - return (CompressedObjectStrategy.CompressionStrategy) input[0]; + return (CompressionStrategy) input[0]; } } ); @@ -85,12 +87,13 @@ public Object[] apply(List input) private static final int[] MAX_VALUES = new int[] {0xFF, 0xFFFF, 0xFFFFFF, 0x0FFFFFFF}; - public CompressedVSizeIntsIndexedSupplierTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy, ByteOrder byteOrder) + public CompressedVSizeIntsIndexedSupplierTest(CompressionStrategy compressionStrategy, ByteOrder byteOrder) { super(compressionStrategy); this.byteOrder = byteOrder; } + private Closer closer; private IndexedInts indexed; private CompressedVSizeIntsIndexedSupplier supplier; private int[] vals; @@ -100,6 +103,7 @@ public CompressedVSizeIntsIndexedSupplierTest(CompressedObjectStrategy.Compressi @Before public void setUp() throws Exception { + closer = Closer.create(); CloseQuietly.close(indexed); indexed = null; supplier = null; @@ -110,6 +114,7 @@ public void setUp() throws Exception public void tearDown() throws Exception { CloseQuietly.close(indexed); + closer.close(); } private void setupSimple(final int chunkSize) @@ -119,11 +124,12 @@ private void setupSimple(final int chunkSize) vals = new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16}; supplier = CompressedVSizeIntsIndexedSupplier.fromList( - Ints.asList(vals), + IntArrayList.wrap(vals), Ints.max(vals), chunkSize, ByteOrder.nativeOrder(), - compressionStrategy + compressionStrategy, + closer ); indexed = supplier.get(); @@ -142,14 +148,14 @@ private void makeWithSerde(final int chunkSize) throws IOException ByteArrayOutputStream baos = new ByteArrayOutputStream(); final CompressedVSizeIntsIndexedSupplier theSupplier = CompressedVSizeIntsIndexedSupplier.fromList( - Ints.asList(vals), Ints.max(vals), chunkSize, byteOrder, compressionStrategy + IntArrayList.wrap(vals), Ints.max(vals), chunkSize, byteOrder, compressionStrategy, closer ); - theSupplier.writeToChannel(Channels.newChannel(baos)); + theSupplier.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(theSupplier.getSerializedSize(), bytes.length); - supplier = CompressedVSizeIntsIndexedSupplier.fromByteBuffer(ByteBuffer.wrap(bytes), byteOrder, null); + supplier = CompressedVSizeIntsIndexedSupplier.fromByteBuffer(ByteBuffer.wrap(bytes), byteOrder); indexed = supplier.get(); } diff --git a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriterTest.java b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriterTest.java index 54c1fdb25ea8..ffba7eaddaf7 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriterTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedVSizeIntsIndexedWriterTest.java @@ -29,6 +29,10 @@ import io.druid.java.util.common.io.smoosh.Smoosh; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.writeout.WriteOutBytes; +import it.unimi.dsi.fastutil.ints.IntArrayList; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.After; @@ -41,8 +45,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; import java.util.List; import java.util.Random; import java.util.Set; @@ -53,13 +55,13 @@ public class CompressedVSizeIntsIndexedWriterTest { private static final int[] MAX_VALUES = new int[]{0xFF, 0xFFFF, 0xFFFFFF, 0x0FFFFFFF}; - private final IOPeon ioPeon = new TmpFileIOPeon(); - private final CompressedObjectStrategy.CompressionStrategy compressionStrategy; + private final SegmentWriteOutMedium segmentWriteOutMedium = new OffHeapMemorySegmentWriteOutMedium(); + private final CompressionStrategy compressionStrategy; private final ByteOrder byteOrder; private final Random rand = new Random(0); private int[] vals; public CompressedVSizeIntsIndexedWriterTest( - CompressedObjectStrategy.CompressionStrategy compressionStrategy, + CompressionStrategy compressionStrategy, ByteOrder byteOrder ) { @@ -71,7 +73,7 @@ public CompressedVSizeIntsIndexedWriterTest( public static Iterable compressionStrategiesAndByteOrders() { Set> combinations = Sets.cartesianProduct( - Sets.newHashSet(CompressedObjectStrategy.CompressionStrategy.noNoneValues()), + Sets.newHashSet(CompressionStrategy.noNoneValues()), Sets.newHashSet(ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN) ); @@ -96,7 +98,7 @@ public void setUp() throws Exception @After public void tearDown() throws Exception { - ioPeon.close(); + segmentWriteOutMedium.close(); } private void generateVals(final int totalSize, final int maxValue) throws IOException @@ -112,29 +114,36 @@ private void checkSerializedSizeAndData(int chunkSize) throws Exception FileSmoosher smoosher = new FileSmoosher(FileUtils.getTempDirectory()); CompressedVSizeIntsIndexedWriter writer = new CompressedVSizeIntsIndexedWriter( - ioPeon, "test", vals.length > 0 ? Ints.max(vals) : 0, chunkSize, byteOrder, compressionStrategy + segmentWriteOutMedium, + "test", + vals.length > 0 ? Ints.max(vals) : 0, + chunkSize, + byteOrder, + compressionStrategy ); CompressedVSizeIntsIndexedSupplier supplierFromList = CompressedVSizeIntsIndexedSupplier.fromList( - Ints.asList(vals), vals.length > 0 ? Ints.max(vals) : 0, chunkSize, byteOrder, compressionStrategy + IntArrayList.wrap(vals), + vals.length > 0 ? Ints.max(vals) : 0, + chunkSize, + byteOrder, + compressionStrategy, + segmentWriteOutMedium.getCloser() ); writer.open(); for (int val : vals) { writer.add(val); } - writer.close(); long writtenLength = writer.getSerializedSize(); - final WritableByteChannel outputChannel = Channels.newChannel(ioPeon.makeOutputStream("output")); - writer.writeToChannel(outputChannel, smoosher); - outputChannel.close(); + final WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); + writer.writeTo(writeOutBytes, smoosher); smoosher.close(); assertEquals(writtenLength, supplierFromList.getSerializedSize()); // read from ByteBuffer and check values CompressedVSizeIntsIndexedSupplier supplierFromByteBuffer = CompressedVSizeIntsIndexedSupplier.fromByteBuffer( - ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), - byteOrder, - null + ByteBuffer.wrap(IOUtils.toByteArray(writeOutBytes.asInputStream())), + byteOrder ); IndexedInts indexedInts = supplierFromByteBuffer.get(); for (int i = 0; i < vals.length; ++i) { @@ -177,20 +186,18 @@ private void checkV2SerializedSizeAndData(int chunkSize) throws Exception File tmpDirectory = FileUtils.getTempDirectory(); FileSmoosher smoosher = new FileSmoosher(tmpDirectory); - int maxValue = vals.length > 0 ? Ints.max(vals) : 0; - GenericIndexedWriter genericIndexed = new GenericIndexedWriter<>( - ioPeon, + GenericIndexedWriter genericIndexed = GenericIndexedWriter.ofCompressedByteBuffers( + segmentWriteOutMedium, "test", - CompressedByteBufferObjectStrategy.getBufferForOrder( - byteOrder, - compressionStrategy, - chunkSize * VSizeIndexedInts.getNumBytesForMax(maxValue) - + CompressedVSizeIntsIndexedSupplier.bufferPadding(VSizeIndexedInts.getNumBytesForMax(maxValue)) - ), + compressionStrategy, Longs.BYTES * 10000 ); CompressedVSizeIntsIndexedWriter writer = new CompressedVSizeIntsIndexedWriter( - vals.length > 0 ? Ints.max(vals) : 0, chunkSize, byteOrder, compressionStrategy, + segmentWriteOutMedium, + vals.length > 0 ? Ints.max(vals) : 0, + chunkSize, + byteOrder, + compressionStrategy, genericIndexed ); writer.open(); @@ -198,12 +205,11 @@ private void checkV2SerializedSizeAndData(int chunkSize) throws Exception writer.add(val); } - writer.close(); final SmooshedWriter channel = smoosher.addWithSmooshedWriter( "test", writer.getSerializedSize() ); - writer.writeToChannel(channel, smoosher); + writer.writeTo(channel, smoosher); channel.close(); smoosher.close(); @@ -211,8 +217,7 @@ private void checkV2SerializedSizeAndData(int chunkSize) throws Exception CompressedVSizeIntsIndexedSupplier supplierFromByteBuffer = CompressedVSizeIntsIndexedSupplier.fromByteBuffer( mapper.mapFile("test"), - byteOrder, - mapper + byteOrder ); IndexedInts indexedInts = supplierFromByteBuffer.get(); diff --git a/processing/src/test/java/io/druid/segment/data/CompressionStrategyTest.java b/processing/src/test/java/io/druid/segment/data/CompressionStrategyTest.java index 4867cd27689b..e94d53f2f7ab 100644 --- a/processing/src/test/java/io/druid/segment/data/CompressionStrategyTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressionStrategyTest.java @@ -21,26 +21,24 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; +import io.druid.java.util.common.io.Closer; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.List; import java.util.Random; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; -import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -52,11 +50,11 @@ public class CompressionStrategyTest public static Iterable compressionStrategies() { return Iterables.transform( - Arrays.asList(CompressedObjectStrategy.CompressionStrategy.noNoneValues()), - new Function() + Arrays.asList(CompressionStrategy.noNoneValues()), + new Function() { @Override - public Object[] apply(CompressedObjectStrategy.CompressionStrategy compressionStrategy) + public Object[] apply(CompressionStrategy compressionStrategy) { return new Object[]{compressionStrategy}; } @@ -64,9 +62,9 @@ public Object[] apply(CompressedObjectStrategy.CompressionStrategy compressionSt ); } - protected final CompressedObjectStrategy.CompressionStrategy compressionStrategy; + protected final CompressionStrategy compressionStrategy; - public CompressionStrategyTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy) + public CompressionStrategyTest(CompressionStrategy compressionStrategy) { this.compressionStrategy = compressionStrategy; } @@ -83,25 +81,27 @@ public static void setupClass() random.nextBytes(originalData); } - @Test - public void testBasicOperations() + private Closer closer; + + @Before + public void createCloser() { - ByteBuffer compressed = ByteBuffer.wrap(compressionStrategy.getCompressor().compress(originalData)); - ByteBuffer output = ByteBuffer.allocate(originalData.length); - compressionStrategy.getDecompressor().decompress(compressed, compressed.array().length, output); - byte[] checkArray = new byte[DATA_SIZER]; - output.get(checkArray); - Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); + closer = Closer.create(); } + @After + public void closeCloser() throws IOException + { + closer.close(); + } @Test - public void testOutputSizeKnownOperations() + public void testBasicOperations() { - ByteBuffer compressed = ByteBuffer.wrap(compressionStrategy.getCompressor().compress(originalData)); + ByteBuffer compressionOut = compressionStrategy.getCompressor().allocateOutBuffer(originalData.length, closer); + ByteBuffer compressed = compressionStrategy.getCompressor().compress(ByteBuffer.wrap(originalData), compressionOut); ByteBuffer output = ByteBuffer.allocate(originalData.length); - compressionStrategy.getDecompressor() - .decompress(compressed, compressed.array().length, output, originalData.length); + compressionStrategy.getDecompressor().decompress(compressed, compressed.remaining(), output); byte[] checkArray = new byte[DATA_SIZER]; output.get(checkArray); Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); @@ -110,9 +110,10 @@ public void testOutputSizeKnownOperations() @Test public void testDirectMemoryOperations() { - ByteBuffer compressed = ByteBuffer.wrap(compressionStrategy.getCompressor().compress(originalData)); + ByteBuffer compressionOut = compressionStrategy.getCompressor().allocateOutBuffer(originalData.length, closer); + ByteBuffer compressed = compressionStrategy.getCompressor().compress(ByteBuffer.wrap(originalData), compressionOut); ByteBuffer output = ByteBuffer.allocateDirect(originalData.length); - compressionStrategy.getDecompressor().decompress(compressed, compressed.array().length, output); + compressionStrategy.getDecompressor().decompress(compressed, compressed.remaining(), output); byte[] checkArray = new byte[DATA_SIZER]; output.get(checkArray); Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); @@ -139,9 +140,10 @@ public void testConcurrency() throws Exception @Override public Boolean call() throws Exception { - ByteBuffer compressed = ByteBuffer.wrap(compressionStrategy.getCompressor().compress(originalData)); + ByteBuffer compressionOut = compressionStrategy.getCompressor().allocateOutBuffer(originalData.length, closer); + ByteBuffer compressed = compressionStrategy.getCompressor().compress(ByteBuffer.wrap(originalData), compressionOut); ByteBuffer output = ByteBuffer.allocate(originalData.length); - compressionStrategy.getDecompressor().decompress(compressed, compressed.array().length, output); + compressionStrategy.getDecompressor().decompress(compressed, compressed.remaining(), output); byte[] checkArray = new byte[DATA_SIZER]; output.get(checkArray); Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); @@ -156,36 +158,4 @@ public Boolean call() throws Exception Assert.assertTrue((Boolean) result.get()); } } - - - @Test(timeout = 60000) - public void testKnownSizeConcurrency() throws Exception - { - final int numThreads = 20; - - ListeningExecutorService threadPoolExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numThreads)); - List> results = new ArrayList<>(); - for (int i = 0; i < numThreads; ++i) { - results.add( - threadPoolExecutor.submit( - new Runnable() - { - @Override - public void run() - { - ByteBuffer compressed = ByteBuffer.wrap(compressionStrategy.getCompressor().compress(originalData)); - ByteBuffer output = ByteBuffer.allocate(originalData.length); - // TODO: Lambdas would be nice here whenever we use Java 8 - compressionStrategy.getDecompressor() - .decompress(compressed, compressed.array().length, output, originalData.length); - byte[] checkArray = new byte[DATA_SIZER]; - output.get(checkArray); - Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray); - } - } - ) - ); - } - Futures.allAsList(results).get(); - } } diff --git a/processing/src/test/java/io/druid/segment/data/GenericIndexedStringWriterTest.java b/processing/src/test/java/io/druid/segment/data/GenericIndexedStringWriterTest.java new file mode 100644 index 000000000000..ec7f5d8e69b8 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/data/GenericIndexedStringWriterTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.data; + +import io.druid.segment.writeout.OnHeapMemorySegmentWriteOutMedium; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +public class GenericIndexedStringWriterTest +{ + @Test + public void testRandomAccess() throws IOException + { + OnHeapMemorySegmentWriteOutMedium segmentWriteOutMedium = new OnHeapMemorySegmentWriteOutMedium(); + GenericIndexedWriter writer = new GenericIndexedWriter<>( + segmentWriteOutMedium, + "test", + GenericIndexed.STRING_STRATEGY + ); + writer.open(); + writer.write(null); + List strings = new ArrayList<>(); + strings.add(null); + ThreadLocalRandom r = ThreadLocalRandom.current(); + for (int i = 0; i < 100_000; i++) { + byte[] bs = new byte[r.nextInt(1, 10)]; + r.nextBytes(bs); + String s = new String(bs, StandardCharsets.US_ASCII); + strings.add(s); + writer.write(s); + } + for (int i = 0; i < strings.size(); i++) { + Assert.assertEquals(strings.get(i), writer.get(i)); + } + } +} diff --git a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java index 48d5899afdb0..37ac72fdaaf1 100644 --- a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java @@ -126,7 +126,7 @@ private GenericIndexed serializeAndDeserialize(GenericIndexed in { ByteArrayOutputStream baos = new ByteArrayOutputStream(); final WritableByteChannel channel = Channels.newChannel(baos); - indexed.writeToChannel(channel); + indexed.writeTo(channel, null); channel.close(); final ByteBuffer byteBuffer = ByteBuffer.wrap(baos.toByteArray()); diff --git a/processing/src/test/java/io/druid/segment/data/IOPeonForTesting.java b/processing/src/test/java/io/druid/segment/data/IOPeonForTesting.java deleted file mode 100644 index e4043e776aa6..000000000000 --- a/processing/src/test/java/io/druid/segment/data/IOPeonForTesting.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.data; - -import com.google.common.collect.Maps; -import io.druid.java.util.common.StringUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Map; - -/** - */ -class IOPeonForTesting implements IOPeon -{ - Map outStreams = Maps.newHashMap(); - - @Override - public OutputStream makeOutputStream(String filename) throws IOException - { - ByteArrayOutputStream stream = outStreams.get(filename); - - if (stream == null) { - stream = new ByteArrayOutputStream(); - outStreams.put(filename, stream); - } - - return stream; - } - - @Override - public InputStream makeInputStream(String filename) throws IOException - { - ByteArrayOutputStream outStream = outStreams.get(filename); - - if (outStream == null) { - throw new FileNotFoundException(StringUtils.format("unknown file[%s]", filename)); - } - - return new ByteArrayInputStream(outStream.toByteArray()); - } - - @Override - public void close() throws IOException - { - outStreams.clear(); - } - - @Override - public File getFile(String filename) - { - return null; - } -} diff --git a/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java index c8ebcf1290b5..9bd87cdba2e4 100644 --- a/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java +++ b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java @@ -19,14 +19,12 @@ package io.druid.segment.data; -import com.google.common.primitives.Ints; import org.junit.Assert; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.nio.channels.Channels; -import java.util.List; /** */ @@ -52,7 +50,7 @@ public void testSerialization() throws Exception VSizeIndexedInts ints = VSizeIndexedInts.fromArray(array); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ints.writeToChannel(Channels.newChannel(baos)); + ints.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(ints.getSerializedSize(), bytes.length); @@ -64,16 +62,4 @@ public void testSerialization() throws Exception Assert.assertEquals(array[i], deserialized.get(i)); } } - - @Test - public void testGetBytesNoPaddingfromList() throws Exception - { - final int[] array = {1, 2, 4, 5, 6, 8, 9, 10}; - List list = Ints.asList(array); - int maxValue = Ints.max(array); - VSizeIndexedInts ints = VSizeIndexedInts.fromList(list, maxValue); - byte[] bytes1 = ints.getBytesNoPadding(); - byte[] bytes2 = VSizeIndexedInts.getBytesNoPaddingFromList(list, maxValue); - Assert.assertArrayEquals(bytes1, bytes2); - } } diff --git a/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsWriterTest.java b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsWriterTest.java index 4a0a4a2926ad..2649b4ca8eaf 100644 --- a/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsWriterTest.java +++ b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsWriterTest.java @@ -20,6 +20,10 @@ package io.druid.segment.data; import com.google.common.primitives.Ints; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; +import io.druid.segment.writeout.WriteOutBytes; +import io.druid.segment.writeout.SegmentWriteOutMedium; +import it.unimi.dsi.fastutil.ints.IntArrayList; import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; @@ -27,8 +31,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; import java.util.Random; import static org.junit.Assert.assertEquals; @@ -37,7 +39,7 @@ public class VSizeIndexedIntsWriterTest { private static final int[] MAX_VALUES = new int[]{0xFF, 0xFFFF, 0xFFFFFF, 0x0FFFFFFF}; - private final IOPeon ioPeon = new TmpFileIOPeon(); + private final SegmentWriteOutMedium segmentWriteOutMedium = new OffHeapMemorySegmentWriteOutMedium(); private final Random rand = new Random(0); private int[] vals; @@ -50,7 +52,7 @@ public void setUp() throws Exception @After public void tearDown() throws Exception { - ioPeon.close(); + segmentWriteOutMedium.close(); } private void generateVals(final int totalSize, final int maxValue) throws IOException @@ -64,28 +66,24 @@ private void generateVals(final int totalSize, final int maxValue) throws IOExce private void checkSerializedSizeAndData() throws Exception { int maxValue = vals.length == 0 ? 0 : Ints.max(vals); - VSizeIndexedIntsWriter writer = new VSizeIndexedIntsWriter( - ioPeon, "test", maxValue - ); + VSizeIndexedIntsWriter writer = new VSizeIndexedIntsWriter(segmentWriteOutMedium, maxValue); VSizeIndexedInts intsFromList = VSizeIndexedInts.fromList( - Ints.asList(vals), maxValue + IntArrayList.wrap(vals), maxValue ); writer.open(); for (int val : vals) { writer.add(val); } - writer.close(); long writtenLength = writer.getSerializedSize(); - final WritableByteChannel outputChannel = Channels.newChannel(ioPeon.makeOutputStream("output")); - writer.writeToChannel(outputChannel, null); - outputChannel.close(); + WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); + writer.writeTo(writeOutBytes, null); assertEquals(writtenLength, intsFromList.getSerializedSize()); // read from ByteBuffer and check values VSizeIndexedInts intsFromByteBuffer = VSizeIndexedInts.readFromByteBuffer( - ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))) + ByteBuffer.wrap(IOUtils.toByteArray(writeOutBytes.asInputStream())) ); assertEquals(vals.length, intsFromByteBuffer.size()); for (int i = 0; i < vals.length; ++i) { diff --git a/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java b/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java index 215a4d12f053..bb366937ecea 100644 --- a/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java @@ -60,7 +60,7 @@ public VSizeIndexedInts apply(int[] input) assertSame(someInts, indexed); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - indexed.writeToChannel(Channels.newChannel(baos)); + indexed.writeTo(Channels.newChannel(baos), null); final byte[] bytes = baos.toByteArray(); Assert.assertEquals(indexed.getSerializedSize(), bytes.length); diff --git a/processing/src/test/java/io/druid/segment/filter/AndFilterTest.java b/processing/src/test/java/io/druid/segment/filter/AndFilterTest.java index 2d8c2440249c..38facaf912f7 100644 --- a/processing/src/test/java/io/druid/segment/filter/AndFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/AndFilterTest.java @@ -58,12 +58,12 @@ public class AndFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "0")), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim1", "0")), - PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "0")), - PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "0")), - PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "0")), - PARSER.parse(ImmutableMap.of("dim0", "5", "dim1", "0")) + PARSER.parseBatch(ImmutableMap.of("dim0", "0", "dim1", "0")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1", "dim1", "0")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2", "dim1", "0")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3", "dim1", "0")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4", "dim1", "0")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5", "dim1", "0")).get(0) ); public AndFilterTest( diff --git a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java index b814e10da872..f38c49d7e6fe 100644 --- a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java @@ -34,6 +34,9 @@ import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.query.BitmapResultFactory; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.CountAggregatorFactory; @@ -50,12 +53,10 @@ import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; import io.druid.segment.IndexBuilder; -import io.druid.segment.IndexMerger; import io.druid.segment.IndexSpec; import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexStorageAdapter; import io.druid.segment.StorageAdapter; -import io.druid.segment.TestHelper; import io.druid.segment.VirtualColumn; import io.druid.segment.VirtualColumns; import io.druid.segment.column.ValueType; @@ -181,8 +182,9 @@ public static Collection makeConstructors() "roaring", new RoaringBitmapSerdeFactory(true) ); - final Map indexMergers = ImmutableMap.of( - "IndexMergerV9", TestHelper.getTestIndexMergerV9() + final Map segmentWriteOutMediumFactories = ImmutableMap.of( + "tmpFile segment write-out medium", TmpFileSegmentWriteOutMediumFactory.instance(), + "off-heap memory segment write-out medium", OffHeapMemorySegmentWriteOutMediumFactory.instance() ); final Map>> finishers = ImmutableMap.of( @@ -246,25 +248,23 @@ public void close() throws IOException ); for (Map.Entry bitmapSerdeFactoryEntry : bitmapSerdeFactories.entrySet()) { - for (Map.Entry indexMergerEntry : indexMergers.entrySet()) { - for (Map.Entry>> finisherEntry : finishers.entrySet()) { + for (Map.Entry segmentWriteOutMediumFactoryEntry : + segmentWriteOutMediumFactories.entrySet()) { + for (Map.Entry>> finisherEntry : + finishers.entrySet()) { for (boolean cnf : ImmutableList.of(false, true)) { for (boolean optimize : ImmutableList.of(false, true)) { final String testName = StringUtils.format( "bitmaps[%s], indexMerger[%s], finisher[%s], optimize[%s]", bitmapSerdeFactoryEntry.getKey(), - indexMergerEntry.getKey(), + segmentWriteOutMediumFactoryEntry.getKey(), finisherEntry.getKey(), optimize ); - final IndexBuilder indexBuilder = IndexBuilder.create() - .indexSpec(new IndexSpec( - bitmapSerdeFactoryEntry.getValue(), - null, - null, - null - )) - .indexMerger(indexMergerEntry.getValue()); + final IndexBuilder indexBuilder = IndexBuilder + .create() + .indexSpec(new IndexSpec(bitmapSerdeFactoryEntry.getValue(), null, null, null)) + .segmentWriteOutMediumFactory(segmentWriteOutMediumFactoryEntry.getValue()); constructors.add(new Object[]{testName, indexBuilder, finisherEntry.getValue(), cnf, optimize}); } diff --git a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java index 7b6eac561f14..3ade65b1572c 100644 --- a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java @@ -60,14 +60,14 @@ public class BoundFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("dim0", "5", "dim1", "abc")), - PARSER.parse(ImmutableMap.of("dim0", "6", "dim1", "-1000", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("dim0", "7", "dim1", "-10.012", "dim2", ImmutableList.of("d"))) + PARSER.parseBatch(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5", "dim1", "abc")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "6", "dim1", "-1000", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "7", "dim1", "-10.012", "dim2", ImmutableList.of("d"))).get(0) ); public BoundFilterTest( diff --git a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java index 13abd491f2a6..d1de40c05d4c 100644 --- a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java @@ -66,16 +66,16 @@ public class ColumnComparisonFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("1", "2"))), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("3"))), - PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "1", "dim2", ImmutableList.of("4", "5"))), - PARSER.parse(ImmutableMap.of("dim0", "5", "dim1", "5", "dim2", ImmutableList.of("4", "5"))), - PARSER.parse(ImmutableMap.of("dim0", "6", "dim1", "1")), - PARSER.parse(ImmutableMap.of("dim0", "7", "dim1", "a")), - PARSER.parse(ImmutableMap.of("dim0", "8", "dim1", 8L)), - PARSER.parse(ImmutableMap.of("dim0", "9", "dim1", 1.234f, "dim2", 1.234f)) + PARSER.parseBatch(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("1", "2"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("3"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4", "dim1", "1", "dim2", ImmutableList.of("4", "5"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5", "dim1", "5", "dim2", ImmutableList.of("4", "5"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "6", "dim1", "1")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "7", "dim1", "a")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "8", "dim1", 8L)).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "9", "dim1", 1.234f, "dim2", 1.234f)).get(0) ); public ColumnComparisonFilterTest( @@ -109,7 +109,7 @@ public void testColumnsWithoutNulls() assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( DefaultDimensionSpec.of("dim1"), DefaultDimensionSpec.of("dim2") - )), ImmutableList.of("1", "5", "9")); + )), ImmutableList.of("5", "9")); assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( DefaultDimensionSpec.of("dim0"), DefaultDimensionSpec.of("dim1"), @@ -128,7 +128,7 @@ public void testMissingColumnNotSpecifiedInDimensionList() assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( DefaultDimensionSpec.of("dim1"), DefaultDimensionSpec.of("dim6") - )), ImmutableList.of("0", "1")); + )), ImmutableList.of("0")); assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( DefaultDimensionSpec.of("dim2"), @@ -138,7 +138,7 @@ public void testMissingColumnNotSpecifiedInDimensionList() assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( DefaultDimensionSpec.of("dim1"), DefaultDimensionSpec.of("dim6") - )), ImmutableList.of("1")); + )), ImmutableList.of()); assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( DefaultDimensionSpec.of("dim2"), diff --git a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java index ce6e789f917a..f81aa913761f 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java @@ -82,7 +82,7 @@ public class ExpressionFilterTest extends BaseFilterTest ImmutableMap.of("dim0", "7", "dim1", 7L, "dim2", 7.0f, "dim3", "a"), ImmutableMap.of("dim0", "8", "dim1", 8L, "dim2", 8.0f, "dim3", 8L), ImmutableMap.of("dim0", "9", "dim1", 9L, "dim2", 9.0f, "dim3", 1.234f, "dim4", 1.234f) - ).stream().map(PARSER::parse).collect(Collectors.toList()); + ).stream().map(e -> PARSER.parseBatch(e).get(0)).collect(Collectors.toList()); public ExpressionFilterTest( String testName, diff --git a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java index b9ffd79a3813..b3942fe2883c 100644 --- a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java @@ -168,16 +168,16 @@ public DruidDoublePredicate makeDoublePredicate() ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("dim0", "5", "dim1", "abc")), - PARSER.parse(ImmutableMap.of("dim0", "6", "dim1", "B453B411", "dim2", ImmutableList.of("c", "d", "e"))), - PARSER.parse(ImmutableMap.of("dim0", "7", "dim1", "HELLO", "dim2", ImmutableList.of("foo"))), - PARSER.parse(ImmutableMap.of("dim0", "8", "dim1", "abc", "dim2", ImmutableList.of("bar"))), - PARSER.parse(ImmutableMap.of("dim0", "9", "dim1", "1", "dim2", ImmutableList.of("foo", "bar"))) + PARSER.parseBatch(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5", "dim1", "abc")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "6", "dim1", "B453B411", "dim2", ImmutableList.of("c", "d", "e"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "7", "dim1", "HELLO", "dim2", ImmutableList.of("foo"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "8", "dim1", "abc", "dim2", ImmutableList.of("bar"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "9", "dim1", "1", "dim2", ImmutableList.of("foo", "bar"))).get(0) ); public FilterPartitionTest( diff --git a/processing/src/test/java/io/druid/segment/filter/FloatAndDoubleFilteringTest.java b/processing/src/test/java/io/druid/segment/filter/FloatAndDoubleFilteringTest.java index f20f7597724c..598968de8b09 100644 --- a/processing/src/test/java/io/druid/segment/filter/FloatAndDoubleFilteringTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FloatAndDoubleFilteringTest.java @@ -93,12 +93,12 @@ public class FloatAndDoubleFilteringTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("ts", 1L, "dim0", "1", "flt", 1.0f, "dbl", 1.0d)), - PARSER.parse(ImmutableMap.of("ts", 2L, "dim0", "2", "flt", 2.0f, "dbl", 2.0d)), - PARSER.parse(ImmutableMap.of("ts", 3L, "dim0", "3", "flt", 3.0f, "dbl", 3.0d)), - PARSER.parse(ImmutableMap.of("ts", 4L, "dim0", "4", "flt", 4.0f, "dbl", 4.0d)), - PARSER.parse(ImmutableMap.of("ts", 5L, "dim0", "5", "flt", 5.0f, "dbl", 5.0d)), - PARSER.parse(ImmutableMap.of("ts", 6L, "dim0", "6", "flt", 6.0f, "dbl", 6.0d)) + PARSER.parseBatch(ImmutableMap.of("ts", 1L, "dim0", "1", "flt", 1.0f, "dbl", 1.0d)).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 2L, "dim0", "2", "flt", 2.0f, "dbl", 2.0d)).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 3L, "dim0", "3", "flt", 3.0f, "dbl", 3.0d)).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 4L, "dim0", "4", "flt", 4.0f, "dbl", 4.0d)).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 5L, "dim0", "5", "flt", 5.0f, "dbl", 5.0d)).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 6L, "dim0", "6", "flt", 6.0f, "dbl", 6.0d)).get(0) ); public FloatAndDoubleFilteringTest( diff --git a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java index 12122307d341..f2bc0dcf7c6a 100644 --- a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java @@ -64,12 +64,12 @@ public class InFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "a", "dim1", "", "dim2", ImmutableList.of("a", "b"))), - PARSER.parse(ImmutableMap.of("dim0", "b", "dim1", "10", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("dim0", "c", "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("dim0", "d", "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("dim0", "e", "dim1", "def", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("dim0", "f", "dim1", "abc")) + PARSER.parseBatch(ImmutableMap.of("dim0", "a", "dim1", "", "dim2", ImmutableList.of("a", "b"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "b", "dim1", "10", "dim2", ImmutableList.of())).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "c", "dim1", "2", "dim2", ImmutableList.of(""))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "d", "dim1", "1", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "e", "dim1", "def", "dim2", ImmutableList.of("c"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "f", "dim1", "abc")).get(0) ); public InFilterTest( diff --git a/processing/src/test/java/io/druid/segment/filter/InvalidFilteringTest.java b/processing/src/test/java/io/druid/segment/filter/InvalidFilteringTest.java index 87f30dd3efbb..686c1064defc 100644 --- a/processing/src/test/java/io/druid/segment/filter/InvalidFilteringTest.java +++ b/processing/src/test/java/io/druid/segment/filter/InvalidFilteringTest.java @@ -64,12 +64,12 @@ public class InvalidFilteringTest extends BaseFilterTest ) ); - private static final InputRow row0 = PARSER.parse(ImmutableMap.of("ts", 1L, "dim0", "1", "dim1", "", "dim2", ImmutableList.of("a", "b"))); - private static final InputRow row1 = PARSER.parse(ImmutableMap.of("ts", 2L, "dim0", "2", "dim1", "10", "dim2", ImmutableList.of())); - private static final InputRow row2 = PARSER.parse(ImmutableMap.of("ts", 3L, "dim0", "3", "dim1", "2", "dim2", ImmutableList.of(""))); - private static final InputRow row3 = PARSER.parse(ImmutableMap.of("ts", 4L, "dim0", "4", "dim1", "1", "dim2", ImmutableList.of("a"))); - private static final InputRow row4 = PARSER.parse(ImmutableMap.of("ts", 5L, "dim0", "5", "dim1", "def", "dim2", ImmutableList.of("c"))); - private static final InputRow row5 = PARSER.parse(ImmutableMap.of("ts", 6L, "dim0", "6", "dim1", "abc")); + private static final InputRow row0 = PARSER.parseBatch(ImmutableMap.of("ts", 1L, "dim0", "1", "dim1", "", "dim2", ImmutableList.of("a", "b"))).get(0); + private static final InputRow row1 = PARSER.parseBatch(ImmutableMap.of("ts", 2L, "dim0", "2", "dim1", "10", "dim2", ImmutableList.of())).get(0); + private static final InputRow row2 = PARSER.parseBatch(ImmutableMap.of("ts", 3L, "dim0", "3", "dim1", "2", "dim2", ImmutableList.of(""))).get(0); + private static final InputRow row3 = PARSER.parseBatch(ImmutableMap.of("ts", 4L, "dim0", "4", "dim1", "1", "dim2", ImmutableList.of("a"))).get(0); + private static final InputRow row4 = PARSER.parseBatch(ImmutableMap.of("ts", 5L, "dim0", "5", "dim1", "def", "dim2", ImmutableList.of("c"))).get(0); + private static final InputRow row5 = PARSER.parseBatch(ImmutableMap.of("ts", 6L, "dim0", "6", "dim1", "abc")).get(0); private static final List ROWS = ImmutableList.of( row0, diff --git a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java index 8182429dd1d5..00480e8e2feb 100644 --- a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java @@ -65,12 +65,12 @@ public class JavaScriptFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("dim0", "5", "dim1", "abc")) + PARSER.parseBatch(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5", "dim1", "abc")).get(0) ); public JavaScriptFilterTest( diff --git a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java index 9b9fb6a563f4..dd2c2c8504f2 100644 --- a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java @@ -57,12 +57,12 @@ public class LikeFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "")), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim1", "foo")), - PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "foobar")), - PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "bar")), - PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "foobarbaz")), - PARSER.parse(ImmutableMap.of("dim0", "5", "dim1", "foo%bar")) + PARSER.parseBatch(ImmutableMap.of("dim0", "0", "dim1", "")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1", "dim1", "foo")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2", "dim1", "foobar")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3", "dim1", "bar")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4", "dim1", "foobarbaz")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5", "dim1", "foo%bar")).get(0) ); public LikeFilterTest( diff --git a/processing/src/test/java/io/druid/segment/filter/LongFilteringTest.java b/processing/src/test/java/io/druid/segment/filter/LongFilteringTest.java index a4a7b4a68b17..10d111358120 100644 --- a/processing/src/test/java/io/druid/segment/filter/LongFilteringTest.java +++ b/processing/src/test/java/io/druid/segment/filter/LongFilteringTest.java @@ -86,16 +86,16 @@ public class LongFilteringTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("ts", 1L, "dim0", "1", "lng", 1L, "dim1", "", "dim2", ImmutableList.of("a", "b"))), - PARSER.parse(ImmutableMap.of("ts", 2L, "dim0", "2", "lng", 2L, "dim1", "10", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("ts", 3L, "dim0", "3", "lng", 3L, "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("ts", 4L, "dim0", "4", "lng", 4L, "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("ts", 5L, "dim0", "5", "lng", 5L, "dim1", "def", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("ts", 6L, "dim0", "6", "lng", 6L, "dim1", "abc")), - PARSER.parse(ImmutableMap.of("ts", 7L, "dim0", "7", "lng", 100000000L, "dim1", "xyz")), - PARSER.parse(ImmutableMap.of("ts", 8L, "dim0", "8", "lng", 100000001L, "dim1", "xyz")), - PARSER.parse(ImmutableMap.of("ts", 9L, "dim0", "9", "lng", -25L, "dim1", "ghi")), - PARSER.parse(ImmutableMap.of("ts", 10L, "dim0", "10", "lng", -100000001L, "dim1", "qqq")) + PARSER.parseBatch(ImmutableMap.of("ts", 1L, "dim0", "1", "lng", 1L, "dim1", "", "dim2", ImmutableList.of("a", "b"))).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 2L, "dim0", "2", "lng", 2L, "dim1", "10", "dim2", ImmutableList.of())).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 3L, "dim0", "3", "lng", 3L, "dim1", "2", "dim2", ImmutableList.of(""))).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 4L, "dim0", "4", "lng", 4L, "dim1", "1", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 5L, "dim0", "5", "lng", 5L, "dim1", "def", "dim2", ImmutableList.of("c"))).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 6L, "dim0", "6", "lng", 6L, "dim1", "abc")).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 7L, "dim0", "7", "lng", 100000000L, "dim1", "xyz")).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 8L, "dim0", "8", "lng", 100000001L, "dim1", "xyz")).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 9L, "dim0", "9", "lng", -25L, "dim1", "ghi")).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 10L, "dim0", "10", "lng", -100000001L, "dim1", "qqq")).get(0) ); public LongFilteringTest( diff --git a/processing/src/test/java/io/druid/segment/filter/NotFilterTest.java b/processing/src/test/java/io/druid/segment/filter/NotFilterTest.java index 6fdb7191fc8a..a25dd09fd5b2 100644 --- a/processing/src/test/java/io/druid/segment/filter/NotFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/NotFilterTest.java @@ -56,12 +56,12 @@ public class NotFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0")), - PARSER.parse(ImmutableMap.of("dim0", "1")), - PARSER.parse(ImmutableMap.of("dim0", "2")), - PARSER.parse(ImmutableMap.of("dim0", "3")), - PARSER.parse(ImmutableMap.of("dim0", "4")), - PARSER.parse(ImmutableMap.of("dim0", "5")) + PARSER.parseBatch(ImmutableMap.of("dim0", "0")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5")).get(0) ); public NotFilterTest( diff --git a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java index 7dbff91f50fa..0e441b74c6b0 100644 --- a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java @@ -63,12 +63,12 @@ public class RegexFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "abdef", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("dim0", "5", "dim1", "abc")) + PARSER.parseBatch(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4", "dim1", "abdef", "dim2", ImmutableList.of("c"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5", "dim1", "abc")).get(0) ); public RegexFilterTest( diff --git a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java index 140b8b78dcf9..4e04df39895f 100644 --- a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java @@ -65,12 +65,12 @@ public class SearchQueryFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "abdef", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("dim0", "5", "dim1", "abc")) + PARSER.parseBatch(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of())).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4", "dim1", "abdef", "dim2", ImmutableList.of("c"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5", "dim1", "abc")).get(0) ); public SearchQueryFilterTest( diff --git a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java index d45e45dd364f..835623c6e529 100644 --- a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java @@ -68,12 +68,12 @@ public class SelectorFilterTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"), "dim6", "2017-07-25")), - PARSER.parse(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of(), "dim6", "2017-07-25")), - PARSER.parse(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""), "dim6", "2017-05-25")), - PARSER.parse(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("dim0", "5", "dim1", "abc")) + PARSER.parseBatch(ImmutableMap.of("dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"), "dim6", "2017-07-25")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "1", "dim1", "10", "dim2", ImmutableList.of(), "dim6", "2017-07-25")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""), "dim6", "2017-05-25")).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))).get(0), + PARSER.parseBatch(ImmutableMap.of("dim0", "5", "dim1", "abc")).get(0) ); public SelectorFilterTest( diff --git a/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java index f2def054670f..3ff7337942cf 100644 --- a/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java @@ -32,6 +32,7 @@ import io.druid.java.util.common.Intervals; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.Granularities; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.QueryPlus; @@ -65,6 +66,7 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -84,8 +86,6 @@ public class SpatialFilterBonusTest new LongSumAggregatorFactory("val", "val") }; private static List DIMS = Lists.newArrayList("dim", "dim.geo"); - private static final IndexMerger INDEX_MERGER = TestHelper.getTestIndexMergerV9(); - private static final IndexIO INDEX_IO = TestHelper.getTestIndexIO(); private final Segment segment; @@ -97,23 +97,19 @@ public SpatialFilterBonusTest(Segment segment) @Parameterized.Parameters public static Collection constructorFeeder() throws IOException { - final IndexSpec indexSpec = new IndexSpec(); - final IncrementalIndex rtIndex = makeIncrementalIndex(); - final QueryableIndex mMappedTestIndex = makeQueryableIndex(indexSpec); - final QueryableIndex mergedRealtimeIndex = makeMergedQueryableIndex(indexSpec); - return Arrays.asList( - new Object[][]{ - { - new IncrementalIndexSegment(rtIndex, null) - }, - { - new QueryableIndexSegment(null, mMappedTestIndex) - }, - { - new QueryableIndexSegment(null, mergedRealtimeIndex) - } - } - ); + List argumentArrays = new ArrayList<>(); + for (SegmentWriteOutMediumFactory segmentWriteOutMediumFactory : SegmentWriteOutMediumFactory.builtInFactories()) { + IndexMerger indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); + IndexIO indexIO = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory); + final IndexSpec indexSpec = new IndexSpec(); + final IncrementalIndex rtIndex = makeIncrementalIndex(); + final QueryableIndex mMappedTestIndex = makeQueryableIndex(indexSpec, indexMerger, indexIO); + final QueryableIndex mergedRealtimeIndex = makeMergedQueryableIndex(indexSpec, indexMerger, indexIO); + argumentArrays.add(new Object[] {new IncrementalIndexSegment(rtIndex, null)}); + argumentArrays.add(new Object[] {new QueryableIndexSegment(null, mMappedTestIndex)}); + argumentArrays.add(new Object[] {new QueryableIndexSegment(null, mergedRealtimeIndex)}); + } + return argumentArrays; } private static IncrementalIndex makeIncrementalIndex() throws IOException @@ -246,7 +242,8 @@ private static IncrementalIndex makeIncrementalIndex() throws IOException return theIndex; } - private static QueryableIndex makeQueryableIndex(IndexSpec indexSpec) throws IOException + private static QueryableIndex makeQueryableIndex(IndexSpec indexSpec, IndexMerger indexMerger, IndexIO indexIO) + throws IOException { IncrementalIndex theIndex = makeIncrementalIndex(); File tmpFile = File.createTempFile("billy", "yay"); @@ -254,11 +251,15 @@ private static QueryableIndex makeQueryableIndex(IndexSpec indexSpec) throws IOE tmpFile.mkdirs(); tmpFile.deleteOnExit(); - INDEX_MERGER.persist(theIndex, tmpFile, indexSpec); - return INDEX_IO.loadIndex(tmpFile); + indexMerger.persist(theIndex, tmpFile, indexSpec, null); + return indexIO.loadIndex(tmpFile); } - private static QueryableIndex makeMergedQueryableIndex(final IndexSpec indexSpec) + private static QueryableIndex makeMergedQueryableIndex( + final IndexSpec indexSpec, + final IndexMerger indexMerger, + final IndexIO indexIO + ) { try { IncrementalIndex first = new IncrementalIndex.Builder() @@ -444,21 +445,22 @@ private static QueryableIndex makeMergedQueryableIndex(final IndexSpec indexSpec mergedFile.mkdirs(); mergedFile.deleteOnExit(); - INDEX_MERGER.persist(first, DATA_INTERVAL, firstFile, indexSpec); - INDEX_MERGER.persist(second, DATA_INTERVAL, secondFile, indexSpec); - INDEX_MERGER.persist(third, DATA_INTERVAL, thirdFile, indexSpec); + indexMerger.persist(first, DATA_INTERVAL, firstFile, indexSpec, null); + indexMerger.persist(second, DATA_INTERVAL, secondFile, indexSpec, null); + indexMerger.persist(third, DATA_INTERVAL, thirdFile, indexSpec, null); - QueryableIndex mergedRealtime = INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + QueryableIndex mergedRealtime = indexIO.loadIndex( + indexMerger.mergeQueryableIndex( Arrays.asList( - INDEX_IO.loadIndex(firstFile), - INDEX_IO.loadIndex(secondFile), - INDEX_IO.loadIndex(thirdFile) + indexIO.loadIndex(firstFile), + indexIO.loadIndex(secondFile), + indexIO.loadIndex(thirdFile) ), true, METRIC_AGGS, mergedFile, - indexSpec + indexSpec, + null ) ); diff --git a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java index 7e8b1871815c..31311cb15f2e 100644 --- a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java @@ -31,6 +31,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.FinalizeResultsQueryRunner; import io.druid.query.QueryPlus; @@ -73,8 +74,8 @@ @RunWith(Parameterized.class) public class SpatialFilterTest { - private static IndexMerger INDEX_MERGER = TestHelper.getTestIndexMergerV9(); - private static IndexIO INDEX_IO = TestHelper.getTestIndexIO(); + private static IndexMerger INDEX_MERGER = TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()); + private static IndexIO INDEX_IO = TestHelper.getTestIndexIO(OffHeapMemorySegmentWriteOutMediumFactory.instance()); public static final int NUM_POINTS = 5000; private static Interval DATA_INTERVAL = Intervals.of("2013-01-01/2013-01-07"); @@ -270,7 +271,7 @@ private static QueryableIndex makeQueryableIndex(IndexSpec indexSpec) throws IOE tmpFile.mkdirs(); tmpFile.deleteOnExit(); - INDEX_MERGER.persist(theIndex, tmpFile, indexSpec); + INDEX_MERGER.persist(theIndex, tmpFile, indexSpec, null); return INDEX_IO.loadIndex(tmpFile); } @@ -497,9 +498,9 @@ private static QueryableIndex makeMergedQueryableIndex(IndexSpec indexSpec) mergedFile.mkdirs(); mergedFile.deleteOnExit(); - INDEX_MERGER.persist(first, DATA_INTERVAL, firstFile, indexSpec); - INDEX_MERGER.persist(second, DATA_INTERVAL, secondFile, indexSpec); - INDEX_MERGER.persist(third, DATA_INTERVAL, thirdFile, indexSpec); + INDEX_MERGER.persist(first, DATA_INTERVAL, firstFile, indexSpec, null); + INDEX_MERGER.persist(second, DATA_INTERVAL, secondFile, indexSpec, null); + INDEX_MERGER.persist(third, DATA_INTERVAL, thirdFile, indexSpec, null); QueryableIndex mergedRealtime = INDEX_IO.loadIndex( INDEX_MERGER.mergeQueryableIndex( @@ -507,7 +508,8 @@ private static QueryableIndex makeMergedQueryableIndex(IndexSpec indexSpec) true, METRIC_AGGS, mergedFile, - indexSpec + indexSpec, + null ) ); diff --git a/processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java b/processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java index 40780d33a23f..5f68aaeba174 100644 --- a/processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java +++ b/processing/src/test/java/io/druid/segment/filter/TimeFilteringTest.java @@ -80,12 +80,12 @@ public class TimeFilteringTest extends BaseFilterTest ); private static final List ROWS = ImmutableList.of( - PARSER.parse(ImmutableMap.of("ts", 0L, "dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))), - PARSER.parse(ImmutableMap.of("ts", 1L, "dim0", "1", "dim1", "10", "dim2", ImmutableList.of())), - PARSER.parse(ImmutableMap.of("ts", 2L, "dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))), - PARSER.parse(ImmutableMap.of("ts", 3L, "dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))), - PARSER.parse(ImmutableMap.of("ts", 4L, "dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))), - PARSER.parse(ImmutableMap.of("ts", 5L, "dim0", "5", "dim1", "abc")) + PARSER.parseBatch(ImmutableMap.of("ts", 0L, "dim0", "0", "dim1", "", "dim2", ImmutableList.of("a", "b"))).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 1L, "dim0", "1", "dim1", "10", "dim2", ImmutableList.of())).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 2L, "dim0", "2", "dim1", "2", "dim2", ImmutableList.of(""))).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 3L, "dim0", "3", "dim1", "1", "dim2", ImmutableList.of("a"))).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 4L, "dim0", "4", "dim1", "def", "dim2", ImmutableList.of("c"))).get(0), + PARSER.parseBatch(ImmutableMap.of("ts", 5L, "dim0", "5", "dim1", "abc")).get(0) ); public TimeFilteringTest( diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexAdapterTest.java index 619a5e3e8ae7..0e7a4e91cb3d 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexAdapterTest.java @@ -23,8 +23,8 @@ import io.druid.segment.IndexableAdapter; import io.druid.segment.Rowboat; import io.druid.segment.data.BitmapValues; -import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.segment.data.IncrementalIndexTest; import org.junit.Assert; @@ -37,8 +37,8 @@ public class IncrementalIndexAdapterTest { private static final IndexSpec INDEX_SPEC = new IndexSpec( new ConciseBitmapSerdeFactory(), - CompressedObjectStrategy.CompressionStrategy.LZ4, - CompressedObjectStrategy.CompressionStrategy.LZ4, + CompressionStrategy.LZ4, + CompressionStrategy.LZ4, CompressionFactory.LongEncodingStrategy.LONGS ); diff --git a/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java b/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java index 908f4896c537..7f5bdc2c9437 100644 --- a/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java +++ b/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java @@ -175,7 +175,8 @@ protected Integer addToFacts( AtomicInteger numEntries, TimeAndDims key, ThreadLocal rowContainer, - Supplier rowSupplier + Supplier rowSupplier, + boolean skipMaxRowsInMemoryCheck // ignore for benchmark ) throws IndexSizeExceededException { diff --git a/processing/src/test/java/io/druid/segment/loading/SegmentizerFactoryTest.java b/processing/src/test/java/io/druid/segment/loading/SegmentizerFactoryTest.java index a05e4e983898..5df84c4ddd87 100644 --- a/processing/src/test/java/io/druid/segment/loading/SegmentizerFactoryTest.java +++ b/processing/src/test/java/io/druid/segment/loading/SegmentizerFactoryTest.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.druid.jackson.DefaultObjectMapper; import io.druid.jackson.SegmentizerModule; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.segment.IndexIO; import io.druid.segment.column.ColumnConfig; import org.junit.Assert; @@ -42,14 +43,18 @@ public void testFactory() throws IOException FileOutputStream fos = new FileOutputStream(factoryFile); ObjectMapper mapper = new DefaultObjectMapper(); mapper.registerModule(new SegmentizerModule()); - IndexIO indexIO = new IndexIO(mapper, new ColumnConfig() - { - @Override - public int columnCacheSizeBytes() - { - return 777; - } - }); + IndexIO indexIO = new IndexIO( + mapper, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), + new ColumnConfig() + { + @Override + public int columnCacheSizeBytes() + { + return 777; + } + } + ); mapper.setInjectableValues( new InjectableValues.Std().addValue( IndexIO.class, diff --git a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java index 3f2df176c16b..602868d48499 100644 --- a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java +++ b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java @@ -24,10 +24,10 @@ import com.metamx.common.StringUtils; import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogCollector; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.IOPeon; import io.druid.segment.data.ObjectStrategy; import java.nio.ByteBuffer; @@ -152,10 +152,10 @@ public int compare(HyperLogLogCollector o1, HyperLogLogCollector o2) } @Override - public GenericColumnSerializer getSerializer(IOPeon peon, String metric) + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String metric) { return LargeColumnSupportedComplexColumnSerializer.createWithColumnSize( - peon, + segmentWriteOutMedium, metric, this.getObjectStrategy(), Integer.MAX_VALUE diff --git a/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java b/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java index a7e7b98cf42d..de710e3c8a17 100644 --- a/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java +++ b/processing/src/test/java/io/druid/segment/serde/LargeColumnSupportedComplexColumnSerializerTest.java @@ -27,12 +27,12 @@ import io.druid.java.util.common.io.smoosh.Smoosh; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.io.smoosh.SmooshedWriter; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMedium; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ComplexColumn; import io.druid.segment.column.ValueType; -import io.druid.segment.data.IOPeon; -import io.druid.segment.data.TmpFileIOPeon; import org.apache.commons.io.FileUtils; import org.junit.Assert; import org.junit.Test; @@ -60,11 +60,11 @@ public void testSanity() throws IOException for (int aCase : cases) { File tmpFile = FileUtils.getTempDirectory(); HyperLogLogCollector baseCollector = HyperLogLogCollector.makeLatestCollector(); - try (IOPeon peon = new TmpFileIOPeon(); + try (SegmentWriteOutMedium segmentWriteOutMedium = new OffHeapMemorySegmentWriteOutMedium(); FileSmoosher v9Smoosher = new FileSmoosher(tmpFile)) { LargeColumnSupportedComplexColumnSerializer serializer = LargeColumnSupportedComplexColumnSerializer - .createWithColumnSize(peon, "test", serde.getObjectStrategy(), columnSize); + .createWithColumnSize(segmentWriteOutMedium, "test", serde.getObjectStrategy(), columnSize); serializer.open(); for (int i = 0; i < aCase; i++) { @@ -74,13 +74,12 @@ public void testSanity() throws IOException baseCollector.fold(collector); serializer.serialize(collector); } - serializer.close(); try (final SmooshedWriter channel = v9Smoosher.addWithSmooshedWriter( "test", serializer.getSerializedSize() )) { - serializer.writeToChannel(channel, v9Smoosher); + serializer.writeTo(channel, v9Smoosher); } } diff --git a/processing/src/test/java/io/druid/segment/writeout/WriteOutBytesTest.java b/processing/src/test/java/io/druid/segment/writeout/WriteOutBytesTest.java new file mode 100644 index 000000000000..6b62ad30c862 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/writeout/WriteOutBytesTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.writeout; + +import com.google.common.io.Files; +import com.google.common.primitives.Ints; +import io.druid.java.util.common.StringUtils; +import org.apache.commons.io.IOUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; + +@RunWith(Parameterized.class) +public class WriteOutBytesTest +{ + @Parameterized.Parameters + public static Collection constructorFeeder() throws IOException + { + return Arrays.asList( + new Object[] {new TmpFileSegmentWriteOutMedium(Files.createTempDir())}, + new Object[] {new OffHeapMemorySegmentWriteOutMedium()}, + new Object[] {new OnHeapMemorySegmentWriteOutMedium()} + ); + } + + private final SegmentWriteOutMedium segmentWriteOutMedium; + + public WriteOutBytesTest(SegmentWriteOutMedium segmentWriteOutMedium) + { + this.segmentWriteOutMedium = segmentWriteOutMedium; + } + + @Test + public void testWriteOutBytes() throws IOException + { + WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); + + writeOutBytes.write('1'); + verifyContents(writeOutBytes, "1"); + + writeOutBytes.writeInt(Ints.fromBytes((byte) '2', (byte) '3', (byte) '4', (byte) '5')); + verifyContents(writeOutBytes, "12345"); + + writeOutBytes.write(new byte[] {'a'}); + verifyContents(writeOutBytes, "12345a"); + + writeOutBytes.write(new byte[] {'a', 'b', 'c'}, 1, 1); + verifyContents(writeOutBytes, "12345ab"); + + ByteBuffer bb = ByteBuffer.wrap(new byte[]{'a', 'b', 'c'}); + bb.position(2); + writeOutBytes.write(bb); + Assert.assertEquals(3, bb.position()); + verifyContents(writeOutBytes, "12345abc"); + } + + private void verifyContents(WriteOutBytes writeOutBytes, String expected) throws IOException + { + Assert.assertEquals(expected, IOUtils.toString(writeOutBytes.asInputStream(), StandardCharsets.US_ASCII)); + ByteBuffer bb = ByteBuffer.allocate((int) writeOutBytes.size()); + writeOutBytes.readFully(0, bb); + bb.flip(); + Assert.assertEquals(expected, StringUtils.fromUtf8(bb)); + } + + @Test + public void testCrossBufferRandomAccess() throws IOException + { + WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); + for (int i = 0; i < ByteBufferWriteOutBytes.BUFFER_SIZE; i++) { + writeOutBytes.write('0'); + } + writeOutBytes.write('1'); + writeOutBytes.write('2'); + writeOutBytes.write('3'); + ByteBuffer bb = ByteBuffer.allocate(4); + writeOutBytes.readFully(ByteBufferWriteOutBytes.BUFFER_SIZE - 1, bb); + bb.flip(); + Assert.assertEquals("0123", StringUtils.fromUtf8(bb)); + } + + @Test(expected = BufferUnderflowException.class) + public void testReadFullyUnderflow() throws IOException + { + WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); + writeOutBytes.write('1'); + writeOutBytes.readFully(0, ByteBuffer.allocate(2)); + } + + @Test + public void testReadFullyEmptyAtTheEnd() throws IOException + { + WriteOutBytes writeOutBytes = segmentWriteOutMedium.makeWriteOutBytes(); + writeOutBytes.write('1'); + writeOutBytes.readFully(1, ByteBuffer.allocate(0)); + } +} diff --git a/server/src/main/java/io/druid/client/DirectDruidClient.java b/server/src/main/java/io/druid/client/DirectDruidClient.java index cd1705dbf3d1..5399a01de593 100644 --- a/server/src/main/java/io/druid/client/DirectDruidClient.java +++ b/server/src/main/java/io/druid/client/DirectDruidClient.java @@ -337,7 +337,7 @@ public ClientResponse handleChunk( public ClientResponse done(ClientResponse clientResponse) { long stopTimeNs = System.nanoTime(); - long nodeTimeNs = stopTimeNs - responseStartTimeNs; + long nodeTimeNs = stopTimeNs - requestStartTimeNs; final long nodeTimeMs = TimeUnit.NANOSECONDS.toMillis(nodeTimeNs); log.debug( "Completed queryId[%s] request to url[%s] with %,d bytes returned in %,d millis [%,f b/s].", diff --git a/server/src/main/java/io/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java b/server/src/main/java/io/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java index 00f29f68cec7..ee96b335a0ce 100644 --- a/server/src/main/java/io/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java +++ b/server/src/main/java/io/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java @@ -76,14 +76,14 @@ List getUsedSegmentsForIntervals(String dataSource, List *

* Note that a segment sequence may include segments with a variety of different intervals and versions. * - * @param dataSource dataSource for which to allocate a segment - * @param sequenceName name of the group of ingestion tasks producing a segment series - * @param previousSegmentId previous segment in the series; may be null or empty, meaning this is the first segment - * @param interval interval for which to allocate a segment - * @param maxVersion use this version if we have no better version to use. The returned segment identifier may - * have a version lower than this one, but will not have one higher. - * @param skipSegmentLineageCheck if false, perform lineage validation using previousSegmentId for this sequence. - * Should be set to false if replica tasks would index events in same order + * @param dataSource dataSource for which to allocate a segment + * @param sequenceName name of the group of ingestion tasks producing a segment series + * @param previousSegmentId previous segment in the series; may be null or empty, meaning this is the first segment + * @param interval interval for which to allocate a segment + * @param maxVersion use this version if we have no better version to use. The returned segment identifier may + * have a version lower than this one, but will not have one higher. + * @param skipSegmentLineageCheck if true, perform lineage validation using previousSegmentId for this sequence. + * Should be set to false if replica tasks would index events in same order * * @return the pending segment identifier, or null if it was impossible to allocate a new segment */ @@ -126,7 +126,8 @@ SegmentPublishResult announceHistoricalSegments( /** * Removes entry for 'dataSource' from the dataSource metadata table. * - * @param dataSource identifier + * @param dataSource identifier + * * @return true if the entry was deleted, false otherwise */ boolean deleteDataSourceMetadata(String dataSource); @@ -134,8 +135,9 @@ SegmentPublishResult announceHistoricalSegments( /** * Resets dataSourceMetadata entry for 'dataSource' to the one supplied. * - * @param dataSource identifier + * @param dataSource identifier * @param dataSourceMetadata value to set + * * @return true if the entry was reset, false otherwise */ boolean resetDataSourceMetadata(String dataSource, DataSourceMetadata dataSourceMetadata) throws IOException; diff --git a/server/src/main/java/io/druid/initialization/Initialization.java b/server/src/main/java/io/druid/initialization/Initialization.java index 81ef5e2fdc31..aeef72a0cb7a 100644 --- a/server/src/main/java/io/druid/initialization/Initialization.java +++ b/server/src/main/java/io/druid/initialization/Initialization.java @@ -64,6 +64,7 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; import io.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule; +import io.druid.segment.writeout.SegmentWriteOutMediumModule; import io.druid.server.emitter.EmitterModule; import io.druid.server.initialization.AuthenticatorMapperModule; import io.druid.server.initialization.AuthorizerMapperModule; @@ -360,6 +361,7 @@ public static Injector makeInjectorWithModules(final Injector baseInjector, Iter new AnnouncerModule(), new AWSModule(), new MetricsModule(), + new SegmentWriteOutMediumModule(), new ServerModule(), new DruidProcessingConfigModule(), new StorageNodeModule(), diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java b/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java index 1a04d42c632d..eca6620f17a4 100644 --- a/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java +++ b/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java @@ -458,7 +458,7 @@ public DataSegment map(int index, ResultSet r, StatementContext ctx) )); } catch (IOException e) { - log.makeAlert(e, "Failed to read segment from db."); + log.makeAlert(e, "Failed to read segment from db.").emit(); return null; } } diff --git a/server/src/main/java/io/druid/segment/indexing/RealtimeTuningConfig.java b/server/src/main/java/io/druid/segment/indexing/RealtimeTuningConfig.java index 5338055a4d8b..32c2f5ec962e 100644 --- a/server/src/main/java/io/druid/segment/indexing/RealtimeTuningConfig.java +++ b/server/src/main/java/io/druid/segment/indexing/RealtimeTuningConfig.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.io.Files; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.IndexSpec; import io.druid.segment.realtime.appenderator.AppenderatorConfig; import io.druid.segment.realtime.plumber.IntervalStartVersioningPolicy; @@ -33,6 +34,7 @@ import io.druid.timeline.partition.ShardSpec; import org.joda.time.Period; +import javax.annotation.Nullable; import java.io.File; /** @@ -57,7 +59,7 @@ private static File createNewBasePersistDirectory() } // Might make sense for this to be a builder - public static RealtimeTuningConfig makeDefaultTuningConfig(final File basePersistDirectory) + public static RealtimeTuningConfig makeDefaultTuningConfig(final @Nullable File basePersistDirectory) { return new RealtimeTuningConfig( defaultMaxRowsInMemory, @@ -74,7 +76,8 @@ public static RealtimeTuningConfig makeDefaultTuningConfig(final File basePersis 0, defaultReportParseExceptions, defaultHandoffConditionTimeout, - defaultAlertTimeout + defaultAlertTimeout, + null ); } @@ -92,6 +95,8 @@ public static RealtimeTuningConfig makeDefaultTuningConfig(final File basePersis private final boolean reportParseExceptions; private final long handoffConditionTimeout; private final long alertTimeout; + @Nullable + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; @JsonCreator public RealtimeTuningConfig( @@ -110,7 +115,8 @@ public RealtimeTuningConfig( @JsonProperty("mergeThreadPriority") int mergeThreadPriority, @JsonProperty("reportParseExceptions") Boolean reportParseExceptions, @JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout, - @JsonProperty("alertTimeout") Long alertTimeout + @JsonProperty("alertTimeout") Long alertTimeout, + @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { this.maxRowsInMemory = maxRowsInMemory == null ? defaultMaxRowsInMemory : maxRowsInMemory; @@ -138,6 +144,7 @@ public RealtimeTuningConfig( this.alertTimeout = alertTimeout == null ? defaultAlertTimeout : alertTimeout; Preconditions.checkArgument(this.alertTimeout >= 0, "alertTimeout must be >= 0"); + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } @Override @@ -240,6 +247,14 @@ public long getAlertTimeout() return alertTimeout; } + @Override + @JsonProperty + @Nullable + public SegmentWriteOutMediumFactory getSegmentWriteOutMediumFactory() + { + return segmentWriteOutMediumFactory; + } + public RealtimeTuningConfig withVersioningPolicy(VersioningPolicy policy) { return new RealtimeTuningConfig( @@ -257,7 +272,8 @@ public RealtimeTuningConfig withVersioningPolicy(VersioningPolicy policy) mergeThreadPriority, reportParseExceptions, handoffConditionTimeout, - alertTimeout + alertTimeout, + segmentWriteOutMediumFactory ); } @@ -278,7 +294,8 @@ public RealtimeTuningConfig withBasePersistDirectory(File dir) mergeThreadPriority, reportParseExceptions, handoffConditionTimeout, - alertTimeout + alertTimeout, + segmentWriteOutMediumFactory ); } } diff --git a/server/src/main/java/io/druid/segment/realtime/RealtimeManager.java b/server/src/main/java/io/druid/segment/realtime/RealtimeManager.java index d9428f483fc9..73bf34fb0b28 100644 --- a/server/src/main/java/io/druid/segment/realtime/RealtimeManager.java +++ b/server/src/main/java/io/druid/segment/realtime/RealtimeManager.java @@ -362,7 +362,8 @@ private boolean runFirehoseV2(FirehoseV2 firehose) throws Exception } catch (Exception e) { log.makeAlert(e, "Unknown exception, Ignoring and continuing.") - .addData("inputRow", inputRow); + .addData("inputRow", inputRow) + .emit(); } try { diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java index 79d24c56a48c..564ed4b9181d 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java @@ -53,6 +53,16 @@ public interface Appenderator extends QuerySegmentWalker, Closeable */ Object startJob(); + /** + * Same as {@link #add(SegmentIdentifier, InputRow, Supplier, boolean)}, with allowIncrementalPersists set to true + */ + default AppenderatorAddResult add(SegmentIdentifier identifier, InputRow row, Supplier committerSupplier) + throws IndexSizeExceededException, SegmentNotWritableException + { + return add(identifier, row, committerSupplier, true); + } + + /** * Add a row. Must not be called concurrently from multiple threads. *

@@ -65,16 +75,28 @@ public interface Appenderator extends QuerySegmentWalker, Closeable * The add, clear, persist, persistAll, and push methods should all be called from the same thread to keep the * metadata committed by Committer in sync. * - * @param identifier the segment into which this row should be added - * @param row the row to add - * @param committerSupplier supplier of a committer associated with all data that has been added, including this row + * @param identifier the segment into which this row should be added + * @param row the row to add + * @param committerSupplier supplier of a committer associated with all data that has been added, including this row + * if {@param allowIncrementalPersists} is set to false then this will not be used as no + * persist will be done automatically + * @param allowIncrementalPersists indicate whether automatic persist should be performed or not if required. + * If this flag is set to false then the return value should have + * {@link AppenderatorAddResult#isPersistRequired} set to true if persist was skipped + * because of this flag and it is assumed that the responsibility of calling + * {@link #persistAll(Committer)} is on the caller. * - * @return positive number indicating how many summarized rows exist in this segment so far + * @return {@link AppenderatorAddResult} * * @throws IndexSizeExceededException if this row cannot be added because it is too large * @throws SegmentNotWritableException if the requested segment is known, but has been closed */ - int add(SegmentIdentifier identifier, InputRow row, Supplier committerSupplier) + AppenderatorAddResult add( + SegmentIdentifier identifier, + InputRow row, + Supplier committerSupplier, + boolean allowIncrementalPersists + ) throws IndexSizeExceededException, SegmentNotWritableException; /** @@ -192,4 +214,39 @@ default ListenableFuture persistAll(Committer committer) * in background thread then it does not cause any problems. */ void closeNow(); + + /** + * Result of {@link Appenderator#add(SegmentIdentifier, InputRow, Supplier, boolean)} containing following information + * - SegmentIdentifier - identifier of segment to which rows are being added + * - int - positive number indicating how many summarized rows exist in this segment so far and + * - boolean - true if {@param allowIncrementalPersists} is set to false and persist is required; false otherwise + */ + class AppenderatorAddResult + { + private final SegmentIdentifier segmentIdentifier; + private final int numRowsInSegment; + private final boolean isPersistRequired; + + AppenderatorAddResult(SegmentIdentifier identifier, int numRowsInSegment, boolean isPersistRequired) + { + this.segmentIdentifier = identifier; + this.numRowsInSegment = numRowsInSegment; + this.isPersistRequired = isPersistRequired; + } + + SegmentIdentifier getSegmentIdentifier() + { + return segmentIdentifier; + } + + int getNumRowsInSegment() + { + return numRowsInSegment; + } + + boolean isPersistRequired() + { + return isPersistRequired; + } + } } diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorConfig.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorConfig.java index 4b2c99f644c0..286470cae761 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorConfig.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorConfig.java @@ -19,9 +19,11 @@ package io.druid.segment.realtime.appenderator; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.segment.IndexSpec; import org.joda.time.Period; +import javax.annotation.Nullable; import java.io.File; public interface AppenderatorConfig @@ -37,4 +39,7 @@ public interface AppenderatorConfig IndexSpec getIndexSpec(); File getBasePersistDirectory(); + + @Nullable + SegmentWriteOutMediumFactory getSegmentWriteOutMediumFactory(); } diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriver.java index b09380360003..82dc9ef6759a 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriver.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriver.java @@ -39,10 +39,10 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; -import io.druid.java.util.common.concurrent.Execs; import io.druid.data.input.Committer; import io.druid.data.input.InputRow; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.logger.Logger; import io.druid.query.SegmentDescriptor; import io.druid.segment.realtime.FireDepartmentMetrics; @@ -61,6 +61,7 @@ import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -250,31 +251,38 @@ public void clear() throws InterruptedException appenderator.clear(); } - /** - * Add a row. Must not be called concurrently from multiple threads. - * - * @param row the row to add - * @param sequenceName sequenceName for this row's segment - * @param committerSupplier supplier of a committer associated with all data that has been added, including this row - * - * @return segment to which this row was added, or null if segment allocator returned null for this row - * - * @throws IOException if there is an I/O error while allocating or writing to a segment - */ public AppenderatorDriverAddResult add( final InputRow row, final String sequenceName, final Supplier committerSupplier ) throws IOException { - return add(row, sequenceName, committerSupplier, false); + return add(row, sequenceName, committerSupplier, false, true); } + /** + * Add a row. Must not be called concurrently from multiple threads. + * + * @param row the row to add + * @param sequenceName sequenceName for this row's segment + * @param committerSupplier supplier of a committer associated with all data that has been added, including this row + * if {@param allowIncrementalPersists} is set to false then this will not be used + * @param skipSegmentLineageCheck if true, perform lineage validation using previousSegmentId for this sequence. + * Should be set to false if replica tasks would index events in same order + * @param allowIncrementalPersists whether to allow persist to happen when maxRowsInMemory or intermediate persist period + * threshold is hit + * + * @return {@link AppenderatorDriverAddResult} + * + * @throws IOException if there is an I/O error while allocating or writing to a segment + */ + public AppenderatorDriverAddResult add( final InputRow row, final String sequenceName, final Supplier committerSupplier, - final boolean skipSegmentLineageCheck + final boolean skipSegmentLineageCheck, + final boolean allowIncrementalPersists ) throws IOException { Preconditions.checkNotNull(row, "row"); @@ -285,8 +293,18 @@ public AppenderatorDriverAddResult add( if (identifier != null) { try { - final int numRowsInMemory = appenderator.add(identifier, row, wrapCommitterSupplier(committerSupplier)); - return AppenderatorDriverAddResult.ok(identifier, numRowsInMemory, appenderator.getTotalRowCount()); + final Appenderator.AppenderatorAddResult result = appenderator.add( + identifier, + row, + wrapCommitterSupplier(committerSupplier), + allowIncrementalPersists + ); + return AppenderatorDriverAddResult.ok( + identifier, + result.getNumRowsInSegment(), + appenderator.getTotalRowCount(), + result.isPersistRequired() + ); } catch (SegmentNotWritableException e) { throw new ISE(e, "WTF?! Segment[%s] not writable when it should have been.", identifier); @@ -299,7 +317,7 @@ public AppenderatorDriverAddResult add( /** * Persist all data indexed through this driver so far. Blocks until complete. *

- * Should be called after all data has been added through {@link #add(InputRow, String, Supplier)}. + * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. * * @param committer committer representing all data that has been added so far * @@ -322,6 +340,22 @@ public Object persist(final Committer committer) throws InterruptedException } } + /** + * Persist all data indexed through this driver so far. Returns a future of persisted commitMetadata. + *

+ * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. + * + * @param committer committer representing all data that has been added so far + * + * @return future containing commitMetadata persisted + */ + public ListenableFuture persistAsync(final Committer committer) + throws InterruptedException, ExecutionException + { + log.info("Persisting data asynchronously"); + return appenderator.persistAll(wrapCommitter(committer)); + } + /** * Register the segments in the given {@link SegmentsAndMetadata} to be handed off and execute a background task which * waits until the hand off completes. @@ -440,8 +474,10 @@ private SegmentIdentifier getActiveSegment(final DateTime timestamp, final Strin /** * Return a segment usable for "timestamp". May return null if no segment can be allocated. * - * @param row input row - * @param sequenceName sequenceName for potential segment allocation + * @param row input row + * @param sequenceName sequenceName for potential segment allocation + * @param skipSegmentLineageCheck if false, perform lineage validation using previousSegmentId for this sequence. + * Should be set to false if replica tasks would index events in same order * * @return identifier, or null * @@ -615,7 +651,7 @@ public void onFailure(Throwable t) * Execute a task in background to publish the given segments. The task blocks until complete. * Retries forever on transient failures, but may exit early on permanent failures. *

- * Should be called after all data has been added through {@link #add(InputRow, String, Supplier)}. + * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. * * @param publisher publisher to use for this set of segments * @param wrappedCommitter committer representing all data that has been added so far diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverAddResult.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverAddResult.java index 04a70173e99b..4838d58a227f 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverAddResult.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverAddResult.java @@ -25,38 +25,43 @@ import javax.annotation.Nullable; /** - * Result of {@link AppenderatorDriver#add(InputRow, String, Supplier)}. It contains the identifier of the - * segment which the InputRow is added to and the number of rows in that segment. + * Result of {@link AppenderatorDriver#add(InputRow, String, Supplier, boolean)}. It contains the identifier of the + * segment which the InputRow is added to, the number of rows in that segment and if persist is required because either + * maxRowsInMemory or intermediate persist period threshold is hit. */ public class AppenderatorDriverAddResult { private final SegmentIdentifier segmentIdentifier; private final int numRowsInSegment; private final long totalNumRowsInAppenderator; + private final boolean isPersistRequired; public static AppenderatorDriverAddResult ok( SegmentIdentifier segmentIdentifier, int numRowsInSegment, - long totalNumRowsInAppenderator + long totalNumRowsInAppenderator, + boolean isPersistRequired ) { - return new AppenderatorDriverAddResult(segmentIdentifier, numRowsInSegment, totalNumRowsInAppenderator); + return new AppenderatorDriverAddResult(segmentIdentifier, numRowsInSegment, totalNumRowsInAppenderator, isPersistRequired); } public static AppenderatorDriverAddResult fail() { - return new AppenderatorDriverAddResult(null, 0, 0); + return new AppenderatorDriverAddResult(null, 0, 0, false); } private AppenderatorDriverAddResult( @Nullable SegmentIdentifier segmentIdentifier, int numRowsInSegment, - long totalNumRowsInAppenderator + long totalNumRowsInAppenderator, + boolean isPersistRequired ) { this.segmentIdentifier = segmentIdentifier; this.numRowsInSegment = numRowsInSegment; this.totalNumRowsInAppenderator = totalNumRowsInAppenderator; + this.isPersistRequired = isPersistRequired; } public boolean isOk() @@ -78,4 +83,9 @@ public long getTotalNumRowsInAppenderator() { return totalNumRowsInAppenderator; } + + public boolean isPersistRequired() + { + return isPersistRequired; + } } diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java index 83dddae834e4..aec0f751a3b6 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java @@ -199,10 +199,11 @@ public Object startJob() } @Override - public int add( + public AppenderatorAddResult add( final SegmentIdentifier identifier, final InputRow row, - final Supplier committerSupplier + final Supplier committerSupplier, + final boolean allowIncrementalPersists ) throws IndexSizeExceededException, SegmentNotWritableException { if (!identifier.getDataSource().equals(schema.getDataSource())) { @@ -219,7 +220,7 @@ public int add( final int sinkRowsInMemoryAfterAdd; try { - sinkRowsInMemoryAfterAdd = sink.add(row); + sinkRowsInMemoryAfterAdd = sink.add(row, !allowIncrementalPersists); } catch (IndexSizeExceededException e) { // Uh oh, we can't do anything about this! We can't persist (commit metadata would be out of sync) and we @@ -237,14 +238,19 @@ public int add( rowsCurrentlyInMemory.addAndGet(numAddedRows); totalRows.addAndGet(numAddedRows); + boolean isPersistRequired = false; if (!sink.canAppendRow() || System.currentTimeMillis() > nextFlush || rowsCurrentlyInMemory.get() >= tuningConfig.getMaxRowsInMemory()) { - // persistAll clears rowsCurrentlyInMemory, no need to update it. - persistAll(committerSupplier.get()); + if (allowIncrementalPersists) { + // persistAll clears rowsCurrentlyInMemory, no need to update it. + persistAll(committerSupplier.get()); + } else { + isPersistRequired = true; + } } - return sink.getNumRows(); + return new AppenderatorAddResult(identifier, sink.getNumRows(), isPersistRequired); } @Override @@ -614,7 +620,8 @@ private DataSegment mergeAndPush(final SegmentIdentifier identifier, final Sink schema.getGranularitySpec().isRollup(), schema.getAggregators(), mergedTarget, - tuningConfig.getIndexSpec() + tuningConfig.getIndexSpec(), + tuningConfig.getSegmentWriteOutMediumFactory() ); } catch (Throwable t) { @@ -1145,7 +1152,8 @@ private int persistHydrant(FireHydrant indexToPersist, SegmentIdentifier identif indexToPersist.getIndex(), identifier.getInterval(), new File(persistDir, String.valueOf(indexToPersist.getCount())), - indexSpec + indexSpec, + tuningConfig.getSegmentWriteOutMediumFactory() ); indexToPersist.swapSegment( diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java index 5da2b59e1306..2d34065d5023 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java @@ -155,7 +155,7 @@ public int add(InputRow row, Supplier committerSupplier) throws Index final int numRows; try { - numRows = appenderator.add(identifier, row, committerSupplier); + numRows = appenderator.add(identifier, row, committerSupplier).getNumRowsInSegment(); lastCommitterSupplier = committerSupplier; return numRows; } diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentAllocator.java b/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentAllocator.java index f7abbd58d386..2cd8a94d596b 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentAllocator.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentAllocator.java @@ -28,9 +28,11 @@ public interface SegmentAllocator /** * Allocates a new segment for a given timestamp. * - * @param row the event which triggered this allocation request - * @param sequenceName sequenceName for this allocation - * @param previousSegmentId segment identifier returned on the previous call to allocate for your sequenceName + * @param row the event which triggered this allocation request + * @param sequenceName sequenceName for this allocation + * @param previousSegmentId segment identifier returned on the previous call to allocate for your sequenceName + * @param skipSegmentLineageCheck if true, perform lineage validation using previousSegmentId for this sequence. + * Should be set to false if replica tasks would index events in same order * * @return the pending segment identifier, or null if it was impossible to allocate a new segment */ diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/EventReceiverFirehoseFactory.java b/server/src/main/java/io/druid/segment/realtime/firehose/EventReceiverFirehoseFactory.java index 59044ae2c9a2..4343e873a564 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/EventReceiverFirehoseFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/EventReceiverFirehoseFactory.java @@ -228,7 +228,7 @@ public Response addAll( final List rows = Lists.newArrayList(); for (final Map event : events) { // Might throw an exception. We'd like that to happen now, instead of while adding to the row buffer. - rows.add(parser.parse(event)); + rows.addAll(parser.parseBatch(event)); } try { diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java b/server/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java index 456790533d0f..6927e46d91fb 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/IrcFirehoseFactory.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.ircclouds.irc.api.Callback; import com.ircclouds.irc.api.IRCApi; @@ -42,6 +43,7 @@ import javax.annotation.Nullable; import java.io.File; import java.io.IOException; +import java.util.Iterator; import java.util.List; import java.util.UUID; import java.util.concurrent.LinkedBlockingQueue; @@ -187,27 +189,33 @@ public void onFailure(Exception e) return new Firehose() { InputRow nextRow = null; + Iterator nextIterator = Iterators.emptyIterator(); @Override public boolean hasMore() { try { while (true) { - Pair nextMsg = queue.poll(100, TimeUnit.MILLISECONDS); if (closed) { return false; } - if (nextMsg == null) { - continue; - } - try { - nextRow = firehoseParser.parse(nextMsg); + + if (nextIterator.hasNext()) { + nextRow = nextIterator.next(); if (nextRow != null) { return true; } - } - catch (IllegalArgumentException iae) { - log.debug("ignoring invalid message in channel [%s]", nextMsg.rhs.getChannelName()); + } else { + Pair nextMsg = queue.poll(100, TimeUnit.MILLISECONDS); + if (nextMsg == null) { + continue; + } + try { + nextIterator = firehoseParser.parseBatch(nextMsg).iterator(); + } + catch (IllegalArgumentException iae) { + log.debug("ignoring invalid message in channel [%s]", nextMsg.rhs.getChannelName()); + } } } } diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/IrcInputRowParser.java b/server/src/main/java/io/druid/segment/realtime/firehose/IrcInputRowParser.java index 2b8dce8c03d2..017c94ae9cd4 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/IrcInputRowParser.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/IrcInputRowParser.java @@ -22,15 +22,16 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.collect.ImmutableList; import com.ircclouds.irc.api.domain.messages.ChannelPrivMsg; - import io.druid.data.input.InputRow; import io.druid.data.input.impl.InputRowParser; import io.druid.data.input.impl.ParseSpec; import io.druid.java.util.common.Pair; - import org.joda.time.DateTime; +import java.util.List; + /** *

Example Usage

*

@@ -86,9 +87,9 @@ public IrcDecoder getDecoder() } @Override - public InputRow parse(Pair msg) + public List parseBatch(Pair msg) { - return decoder.decodeMessage(msg.lhs, msg.rhs.getChannelName(), msg.rhs.getText()); + return ImmutableList.of(decoder.decodeMessage(msg.lhs, msg.rhs.getChannelName(), msg.rhs.getText())); } @JsonProperty diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumber.java b/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumber.java index 224d3c7395da..546181b1fb02 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumber.java @@ -29,8 +29,8 @@ import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.StringUtils; -import io.druid.java.util.common.granularity.Granularity; import io.druid.java.util.common.concurrent.ScheduledExecutors; +import io.druid.java.util.common.granularity.Granularity; import io.druid.query.QueryRunnerFactoryConglomerate; import io.druid.segment.IndexIO; import io.druid.segment.IndexMerger; diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java index 246952d71e4b..3a19ed632161 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java @@ -216,7 +216,7 @@ public int add(InputRow row, Supplier committerSupplier) throws Index return -1; } - final int numRows = sink.add(row); + final int numRows = sink.add(row, false); if (!sink.canAppendRow() || System.currentTimeMillis() > nextFlush) { persist(committerSupplier.get()); @@ -422,13 +422,13 @@ public void doRun() closer.register(segmentAndCloseable.rhs); } - mergedFile = indexMerger.mergeQueryableIndex( indexes, schema.getGranularitySpec().isRollup(), schema.getAggregators(), mergedTarget, - config.getIndexSpec() + config.getIndexSpec(), + config.getSegmentWriteOutMediumFactory() ); } catch (Throwable t) { @@ -942,7 +942,8 @@ protected int persistHydrant( indexToPersist.getIndex(), interval, new File(computePersistDir(schema, interval), String.valueOf(indexToPersist.getCount())), - indexSpec + indexSpec, + config.getSegmentWriteOutMediumFactory() ); indexToPersist.swapSegment( diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java b/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java index 5ac9b05cd6dd..634d8f514bd6 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java @@ -26,7 +26,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; - import io.druid.data.input.InputRow; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; @@ -140,7 +139,7 @@ public FireHydrant getCurrHydrant() return currHydrant; } - public int add(InputRow row) throws IndexSizeExceededException + public int add(InputRow row, boolean skipMaxRowsInMemoryCheck) throws IndexSizeExceededException { if (currHydrant == null) { throw new IAE("No currHydrant but given row[%s]", row); @@ -155,7 +154,7 @@ public int add(InputRow row) throws IndexSizeExceededException if (index == null) { return ADD_FAILED; // the hydrant was swapped without being replaced } - return index.add(row); + return index.add(row, skipMaxRowsInMemoryCheck); } } diff --git a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java index a95fbb7df8e6..1e84da5e20a9 100644 --- a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java @@ -376,12 +376,13 @@ public void run() if (failedSegments.size() > 0) { log.makeAlert("%,d errors seen while loading segments", failedSegments.size()) - .addData("failedSegments", failedSegments); + .addData("failedSegments", failedSegments) + .emit(); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - log.makeAlert(e, "LoadingInterrupted"); + log.makeAlert(e, "LoadingInterrupted").emit(); } backgroundSegmentAnnouncer.finishAnnouncing(); diff --git a/server/src/main/java/io/druid/server/security/Escalator.java b/server/src/main/java/io/druid/server/security/Escalator.java index 021572ed3f0a..c3a9c19aa7d5 100644 --- a/server/src/main/java/io/druid/server/security/Escalator.java +++ b/server/src/main/java/io/druid/server/security/Escalator.java @@ -23,14 +23,14 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.metamx.http.client.HttpClient; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = NoopEscalator.class) -@JsonSubTypes(value = { - @JsonSubTypes.Type(name = "noop", value = NoopEscalator.class), -}) /** * This interface provides methods needed for escalating internal system requests with priveleged authentication * credentials. Each Escalator is associated with a specific authentication scheme, like Authenticators. */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = NoopEscalator.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "noop", value = NoopEscalator.class), +}) public interface Escalator { /** diff --git a/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java index e464c241a81c..3061bfb177c8 100644 --- a/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java @@ -169,9 +169,9 @@ public void testTransformSpec() throws Exception // Test hack that produces a StringInputRowParser. final StringInputRowParser parser = (StringInputRowParser) schema.getParser(); - final InputRow row1bb = parser.parse( + final InputRow row1bb = parser.parseBatch( ByteBuffer.wrap("{\"time\":\"2000-01-01\",\"dimA\":\"foo\"}".getBytes(Charsets.UTF_8)) - ); + ).get(0); Assert.assertEquals(DateTimes.of("2000-01-01"), row1bb.getTimestamp()); Assert.assertEquals("foo", row1bb.getRaw("dimA")); Assert.assertEquals("foofoo", row1bb.getRaw("expr")); @@ -181,9 +181,9 @@ public void testTransformSpec() throws Exception Assert.assertEquals("foo", row1string.getRaw("dimA")); Assert.assertEquals("foofoo", row1string.getRaw("expr")); - final InputRow row2 = parser.parse( + final InputRow row2 = parser.parseBatch( ByteBuffer.wrap("{\"time\":\"2000-01-01\",\"dimA\":\"x\"}".getBytes(Charsets.UTF_8)) - ); + ).get(0); Assert.assertNull(row2); } diff --git a/server/src/test/java/io/druid/segment/indexing/TransformSpecTest.java b/server/src/test/java/io/druid/segment/indexing/TransformSpecTest.java index 5ddc716c76cf..ec72561aeb00 100644 --- a/server/src/test/java/io/druid/segment/indexing/TransformSpecTest.java +++ b/server/src/test/java/io/druid/segment/indexing/TransformSpecTest.java @@ -80,7 +80,7 @@ public void testTransforms() ); final InputRowParser> parser = transformSpec.decorate(PARSER); - final InputRow row = parser.parse(ROW1); + final InputRow row = parser.parseBatch(ROW1).get(0); Assert.assertNotNull(row); Assert.assertEquals(DateTimes.of("2000-01-01").getMillis(), row.getTimestampFromEpoch()); @@ -108,7 +108,7 @@ public void testTransformOverwriteField() ); final InputRowParser> parser = transformSpec.decorate(PARSER); - final InputRow row = parser.parse(ROW1); + final InputRow row = parser.parseBatch(ROW1).get(0); Assert.assertNotNull(row); Assert.assertEquals(DateTimes.of("2000-01-01").getMillis(), row.getTimestampFromEpoch()); @@ -139,8 +139,8 @@ public void testFilterOnTransforms() ); final InputRowParser> parser = transformSpec.decorate(PARSER); - Assert.assertNotNull(parser.parse(ROW1)); - Assert.assertNull(parser.parse(ROW2)); + Assert.assertNotNull(parser.parseBatch(ROW1).get(0)); + Assert.assertNull(parser.parseBatch(ROW2).get(0)); } @Test @@ -154,7 +154,7 @@ public void testTransformTimeFromOtherFields() ); final InputRowParser> parser = transformSpec.decorate(PARSER); - final InputRow row = parser.parse(ROW1); + final InputRow row = parser.parseBatch(ROW1).get(0); Assert.assertNotNull(row); Assert.assertEquals(DateTimes.of("1970-01-01T05:00:00Z"), row.getTimestamp()); @@ -172,7 +172,7 @@ public void testTransformTimeFromTime() ); final InputRowParser> parser = transformSpec.decorate(PARSER); - final InputRow row = parser.parse(ROW1); + final InputRow row = parser.parseBatch(ROW1).get(0); Assert.assertNotNull(row); Assert.assertEquals(DateTimes.of("2000-01-01T01:00:00Z"), row.getTimestamp()); diff --git a/server/src/test/java/io/druid/segment/loading/SegmentLoaderLocalCacheManagerTest.java b/server/src/test/java/io/druid/segment/loading/SegmentLoaderLocalCacheManagerTest.java index ed10455f3370..ffacdb5f6a96 100644 --- a/server/src/test/java/io/druid/segment/loading/SegmentLoaderLocalCacheManagerTest.java +++ b/server/src/test/java/io/druid/segment/loading/SegmentLoaderLocalCacheManagerTest.java @@ -28,6 +28,9 @@ import com.metamx.emitter.EmittingLogger; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Intervals; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.segment.TestHelper; import io.druid.server.metrics.NoopServiceEmitter; import io.druid.timeline.DataSegment; @@ -37,21 +40,36 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; +import java.io.IOException; +import java.util.Collection; import java.util.List; +@RunWith(Parameterized.class) public class SegmentLoaderLocalCacheManagerTest { + @Parameterized.Parameters + public static Collection constructorFeeder() throws IOException + { + return ImmutableList.of( + new Object[] {TmpFileSegmentWriteOutMediumFactory.instance()}, + new Object[] {OffHeapMemorySegmentWriteOutMediumFactory.instance()} + ); + } + @Rule public final TemporaryFolder tmpFolder = new TemporaryFolder(); private final ObjectMapper jsonMapper; + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; private File localSegmentCacheFolder; private SegmentLoaderLocalCacheManager manager; - public SegmentLoaderLocalCacheManagerTest() + public SegmentLoaderLocalCacheManagerTest(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) { jsonMapper = new DefaultObjectMapper(); jsonMapper.registerSubtypes(new NamedType(LocalLoadSpec.class, "local")); @@ -61,6 +79,7 @@ public SegmentLoaderLocalCacheManagerTest() new LocalDataSegmentPuller() ) ); + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } @Before @@ -76,7 +95,7 @@ public void setUp() throws Exception locations.add(locationConfig); manager = new SegmentLoaderLocalCacheManager( - TestHelper.getTestIndexIO(), + TestHelper.getTestIndexIO(segmentWriteOutMediumFactory), new SegmentLoaderConfig().withLocations(locations), jsonMapper ); @@ -150,7 +169,7 @@ public void testRetrySuccessAtFirstLocation() throws Exception locations.add(locationConfig2); manager = new SegmentLoaderLocalCacheManager( - TestHelper.getTestIndexIO(), + TestHelper.getTestIndexIO(segmentWriteOutMediumFactory), new SegmentLoaderConfig().withLocations(locations), jsonMapper ); @@ -203,7 +222,7 @@ public void testRetrySuccessAtSecondLocation() throws Exception locations.add(locationConfig2); manager = new SegmentLoaderLocalCacheManager( - TestHelper.getTestIndexIO(), + TestHelper.getTestIndexIO(segmentWriteOutMediumFactory), new SegmentLoaderConfig().withLocations(locations), jsonMapper ); @@ -258,7 +277,7 @@ public void testRetryAllFail() throws Exception locations.add(locationConfig2); manager = new SegmentLoaderLocalCacheManager( - TestHelper.getTestIndexIO(), + TestHelper.getTestIndexIO(segmentWriteOutMediumFactory), new SegmentLoaderConfig().withLocations(locations), jsonMapper ); @@ -312,7 +331,7 @@ public void testEmptyToFullOrder() throws Exception locations.add(locationConfig2); manager = new SegmentLoaderLocalCacheManager( - TestHelper.getTestIndexIO(), + TestHelper.getTestIndexIO(segmentWriteOutMediumFactory), new SegmentLoaderConfig().withLocations(locations), jsonMapper ); diff --git a/server/src/test/java/io/druid/segment/realtime/FireDepartmentTest.java b/server/src/test/java/io/druid/segment/realtime/FireDepartmentTest.java index 137b75b451fb..ec023fb108bf 100644 --- a/server/src/test/java/io/druid/segment/realtime/FireDepartmentTest.java +++ b/server/src/test/java/io/druid/segment/realtime/FireDepartmentTest.java @@ -29,6 +29,7 @@ import io.druid.data.input.impl.TimestampSpec; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.granularity.Granularities; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.segment.TestHelper; @@ -110,8 +111,8 @@ public void testSerde() throws Exception null, null, null, - TestHelper.getTestIndexMergerV9(), - TestHelper.getTestIndexIO(), + TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance()), + TestHelper.getTestIndexIO(OffHeapMemorySegmentWriteOutMediumFactory.instance()), MapCache.create(0), NO_CACHE_CONFIG, TestHelper.getJsonMapper() diff --git a/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java b/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java index 77046fc5de3e..17ad17256a8e 100644 --- a/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java +++ b/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java @@ -213,6 +213,7 @@ public FirehoseV2 connect(InputRowParser parser, Object arg1) throws IOException 0, null, null, + null, null ); plumber = new TestPlumber(new Sink( @@ -271,6 +272,7 @@ public FirehoseV2 connect(InputRowParser parser, Object arg1) throws IOException 0, null, null, + null, null ); @@ -289,6 +291,7 @@ public FirehoseV2 connect(InputRowParser parser, Object arg1) throws IOException 0, null, null, + null, null ); @@ -1059,7 +1062,7 @@ public int add(InputRow row, Supplier committerSupplier) throws Index return -1; } - return sink.add(row); + return sink.add(row, false); } public Sink getSink(long timestamp) diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverFailTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverFailTest.java index 44142ce3af69..1cc19f4c1458 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverFailTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverFailTest.java @@ -304,12 +304,13 @@ public Object startJob() } @Override - public int add( - SegmentIdentifier identifier, InputRow row, Supplier committerSupplier + public AppenderatorAddResult add( + SegmentIdentifier identifier, InputRow row, Supplier committerSupplier, boolean allowIncrementalPersists ) throws IndexSizeExceededException, SegmentNotWritableException { rows.computeIfAbsent(identifier, k -> new ArrayList<>()).add(row); - return ++numRows; + numRows++; + return new AppenderatorAddResult(identifier, numRows, false); } @Override diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorPlumberTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorPlumberTest.java index 3a6dc6dcfb0a..d7b0aa4a6477 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorPlumberTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorPlumberTest.java @@ -81,6 +81,7 @@ EasyMock. anyObject(), 0, false, null, + null, null ); diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java index 10d314eed88f..8a5bbda46c19 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java @@ -83,13 +83,22 @@ public void testSimpleIngestion() throws Exception // add commitMetadata.put("x", "1"); - Assert.assertEquals(1, appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier)); + Assert.assertEquals(1, + appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier) + .getNumRowsInSegment() + ); commitMetadata.put("x", "2"); - Assert.assertEquals(2, appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier)); + Assert.assertEquals(2, + appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier) + .getNumRowsInSegment() + ); commitMetadata.put("x", "3"); - Assert.assertEquals(1, appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 4), committerSupplier)); + Assert.assertEquals(1, + appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 4), committerSupplier) + .getNumRowsInSegment() + ); // getSegments Assert.assertEquals(IDENTIFIERS.subList(0, 2), sorted(appenderator.getSegments())); @@ -188,6 +197,52 @@ public void run() } } + @Test + public void testMaxRowsInMemoryDisallowIncrementalPersists() throws Exception + { + try (final AppenderatorTester tester = new AppenderatorTester(3, false)) { + final Appenderator appenderator = tester.getAppenderator(); + final AtomicInteger eventCount = new AtomicInteger(0); + final Supplier committerSupplier = () -> { + final Object metadata = ImmutableMap.of("eventCount", eventCount.get()); + + return new Committer() + { + @Override + public Object getMetadata() + { + return metadata; + } + + @Override + public void run() + { + // Do nothing + } + }; + }; + + Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.startJob(); + Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier, false); + Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier, false); + Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier, false); + Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.add(IDENTIFIERS.get(0), IR("2000", "baz", 1), committerSupplier, false); + Assert.assertEquals(3, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 1), committerSupplier, false); + Assert.assertEquals(4, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier, false); + Assert.assertEquals(5, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.persist(ImmutableList.of(IDENTIFIERS.get(1)), committerSupplier.get()); + Assert.assertEquals(3, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.close(); + Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); + } + } @Test public void testRestoreFromDisk() throws Exception { diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java index 766fa19709b7..555e05d20673 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java @@ -33,6 +33,7 @@ import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.granularity.Granularities; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.DefaultQueryRunnerFactoryConglomerate; import io.druid.query.IntervalChunkingQueryRunnerDecorator; import io.druid.query.Query; @@ -146,6 +147,7 @@ public AppenderatorTester( 0, null, null, + null, null ); @@ -154,6 +156,7 @@ public AppenderatorTester( indexIO = new IndexIO( objectMapper, + OffHeapMemorySegmentWriteOutMediumFactory.instance(), new ColumnConfig() { @Override @@ -163,7 +166,7 @@ public int columnCacheSizeBytes() } } ); - indexMerger = new IndexMergerV9(objectMapper, indexIO); + indexMerger = new IndexMergerV9(objectMapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); emitter = new ServiceEmitter( "test", diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/DefaultOfflineAppenderatorFactoryTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/DefaultOfflineAppenderatorFactoryTest.java index 39bc73eaa570..08157eaf7f61 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/DefaultOfflineAppenderatorFactoryTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/DefaultOfflineAppenderatorFactoryTest.java @@ -147,6 +147,7 @@ public int columnCacheSizeBytes() 0, null, null, + null, null ); diff --git a/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java b/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java index 51bd505ea93f..e9d9a7579b48 100644 --- a/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java +++ b/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java @@ -33,6 +33,9 @@ import io.druid.hll.HyperLogLogCollector; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; @@ -52,12 +55,17 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; +import java.io.IOException; +import java.util.Collection; import java.util.List; /** */ +@RunWith(Parameterized.class) public class IngestSegmentFirehoseTest { private static final DimensionsSpec DIMENSIONS_SPEC = new DimensionsSpec( @@ -88,11 +96,26 @@ public class IngestSegmentFirehoseTest new HyperUniquesAggregatorFactory("unique_hosts", "unique_hosts") ); + @Parameterized.Parameters + public static Collection constructorFeeder() throws IOException + { + return ImmutableList.of( + new Object[] {TmpFileSegmentWriteOutMediumFactory.instance()}, + new Object[] {OffHeapMemorySegmentWriteOutMediumFactory.instance()} + ); + } + @Rule public final TemporaryFolder tempFolder = new TemporaryFolder(); - private IndexIO indexIO = TestHelper.getTestIndexIO(); - private IndexMerger indexMerger = TestHelper.getTestIndexMergerV9(); + private final IndexIO indexIO; + private final IndexMerger indexMerger; + + public IngestSegmentFirehoseTest(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) + { + indexIO = TestHelper.getTestIndexIO(segmentWriteOutMediumFactory); + indexMerger = TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory); + } @Test public void testReadFromIndexAndWriteAnotherIndex() throws Exception @@ -207,7 +230,7 @@ private void createTestIndex(File segmentDir) throws Exception for (String line : rows) { index.add(parser.parse(line)); } - indexMerger.persist(index, segmentDir, new IndexSpec()); + indexMerger.persist(index, segmentDir, new IndexSpec(), null); } } } diff --git a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java index 3bd9bfe19ac7..c29531699522 100644 --- a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java +++ b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java @@ -39,6 +39,9 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.query.DefaultQueryRunnerFactoryConglomerate; import io.druid.query.Query; import io.druid.query.QueryRunnerFactory; @@ -84,7 +87,24 @@ @RunWith(Parameterized.class) public class RealtimePlumberSchoolTest { + @Parameterized.Parameters(name = "rejectionPolicy = {0}, segmentWriteOutMediumFactory = {1}") + public static Collection constructorFeeder() throws IOException + { + final RejectionPolicyFactory[] rejectionPolicies = new RejectionPolicyFactory[]{ + new NoopRejectionPolicyFactory(), + new MessageTimeRejectionPolicyFactory() + }; + + final List constructors = Lists.newArrayList(); + for (RejectionPolicyFactory rejectionPolicy : rejectionPolicies) { + constructors.add(new Object[]{rejectionPolicy, OffHeapMemorySegmentWriteOutMediumFactory.instance()}); + constructors.add(new Object[]{rejectionPolicy, TmpFileSegmentWriteOutMediumFactory.instance()}); + } + return constructors; + } + private final RejectionPolicyFactory rejectionPolicy; + private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; private RealtimePlumber plumber; private RealtimePlumberSchool realtimePlumberSchool; private DataSegmentAnnouncer announcer; @@ -99,24 +119,10 @@ public class RealtimePlumberSchoolTest private FireDepartmentMetrics metrics; private File tmpDir; - public RealtimePlumberSchoolTest(RejectionPolicyFactory rejectionPolicy) + public RealtimePlumberSchoolTest(RejectionPolicyFactory rejectionPolicy, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) { this.rejectionPolicy = rejectionPolicy; - } - - @Parameterized.Parameters(name = "rejectionPolicy = {0}") - public static Collection constructorFeeder() throws IOException - { - final RejectionPolicyFactory[] rejectionPolicies = new RejectionPolicyFactory[]{ - new NoopRejectionPolicyFactory(), - new MessageTimeRejectionPolicyFactory() - }; - - final List constructors = Lists.newArrayList(); - for (RejectionPolicyFactory rejectionPolicy : rejectionPolicies) { - constructors.add(new Object[]{rejectionPolicy}); - } - return constructors; + this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } @Before @@ -204,6 +210,7 @@ public void setUp() throws Exception 0, false, null, + null, null ); @@ -215,8 +222,8 @@ public void setUp() throws Exception segmentPublisher, handoffNotifierFactory, MoreExecutors.sameThreadExecutor(), - TestHelper.getTestIndexMergerV9(), - TestHelper.getTestIndexIO(), + TestHelper.getTestIndexMergerV9(segmentWriteOutMediumFactory), + TestHelper.getTestIndexIO(segmentWriteOutMediumFactory), MapCache.create(0), FireDepartmentTest.NO_CACHE_CONFIG, TestHelper.getJsonMapper() diff --git a/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java b/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java index 81b42027a5e8..5b3f6cc4dd83 100644 --- a/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java +++ b/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java @@ -74,6 +74,7 @@ public void testSwap() throws Exception 0, null, null, + null, null ); final Sink sink = new Sink( @@ -129,7 +130,8 @@ public int compareTo(Row o) { return 0; } - } + }, + false ); FireHydrant currHydrant = sink.getCurrHydrant(); @@ -182,7 +184,8 @@ public int compareTo(Row o) { return 0; } - } + }, + false ); Assert.assertEquals(currHydrant, swapHydrant); diff --git a/services/src/test/java/io/druid/cli/validate/DruidJsonValidatorTest.java b/services/src/test/java/io/druid/cli/validate/DruidJsonValidatorTest.java index 06ae9b7c5d0d..49518d6483dd 100644 --- a/services/src/test/java/io/druid/cli/validate/DruidJsonValidatorTest.java +++ b/services/src/test/java/io/druid/cli/validate/DruidJsonValidatorTest.java @@ -182,6 +182,7 @@ public Plumber findPlumber( 0, true, null, + null, null ) ), diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java index 06dc5c03e58b..af99862f5e84 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java @@ -25,7 +25,11 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.ISE; import io.druid.java.util.common.granularity.Granularity; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.ExprType; +import io.druid.math.expr.Parser; +import io.druid.query.expression.TimestampFloorExprMacro; import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.TimeFormatExtractionFn; import io.druid.query.filter.AndDimFilter; @@ -332,14 +336,55 @@ private static DimFilter toSimpleLeafFilter( flip = true; } + // Flip operator, maybe. + final SqlKind flippedKind; + + if (flip) { + switch (kind) { + case EQUALS: + case NOT_EQUALS: + flippedKind = kind; + break; + case GREATER_THAN: + flippedKind = SqlKind.LESS_THAN; + break; + case GREATER_THAN_OR_EQUAL: + flippedKind = SqlKind.LESS_THAN_OR_EQUAL; + break; + case LESS_THAN: + flippedKind = SqlKind.GREATER_THAN; + break; + case LESS_THAN_OR_EQUAL: + flippedKind = SqlKind.GREATER_THAN_OR_EQUAL; + break; + default: + throw new ISE("WTF?! Kind[%s] not expected here", kind); + } + } else { + flippedKind = kind; + } + // rhs must be a literal if (rhs.getKind() != SqlKind.LITERAL) { return null; } - // lhs must be translatable to a SimpleExtraction to be simple-filterable + // Translate lhs to a DruidExpression. final DruidExpression lhsExpression = toDruidExpression(plannerContext, rowSignature, lhs); - if (lhsExpression == null || !lhsExpression.isSimpleExtraction()) { + if (lhsExpression == null) { + return null; + } + + // Special handling for filters on FLOOR(__time TO granularity). + final Granularity queryGranularity = toQueryGranularity(lhsExpression, plannerContext.getExprMacroTable()); + if (queryGranularity != null) { + // lhs is FLOOR(__time TO granularity); rhs must be a timestamp + final long rhsMillis = Calcites.calciteDateTimeLiteralToJoda(rhs, plannerContext.getTimeZone()).getMillis(); + return buildTimeFloorFilter(Column.TIME_COLUMN_NAME, queryGranularity, flippedKind, rhsMillis); + } + + // In the general case, lhs must be translatable to a SimpleExtraction to be simple-filterable. + if (!lhsExpression.isSimpleExtraction()) { return null; } @@ -362,28 +407,29 @@ private static DimFilter toSimpleLeafFilter( // Create a BoundRefKey that strips the extractionFn and compares __time as a number. final BoundRefKey boundRefKey = new BoundRefKey(column, null, StringComparators.NUMERIC); - if (kind == SqlKind.EQUALS) { - return rhsAligned - ? Bounds.interval(boundRefKey, rhsInterval) - : Filtration.matchNothing(); - } else if (kind == SqlKind.NOT_EQUALS) { - return rhsAligned - ? new NotDimFilter(Bounds.interval(boundRefKey, rhsInterval)) - : Filtration.matchEverything(); - } else if ((!flip && kind == SqlKind.GREATER_THAN) || (flip && kind == SqlKind.LESS_THAN)) { - return Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); - } else if ((!flip && kind == SqlKind.GREATER_THAN_OR_EQUAL) || (flip && kind == SqlKind.LESS_THAN_OR_EQUAL)) { - return rhsAligned - ? Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) - : Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); - } else if ((!flip && kind == SqlKind.LESS_THAN) || (flip && kind == SqlKind.GREATER_THAN)) { - return rhsAligned - ? Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) - : Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); - } else if ((!flip && kind == SqlKind.LESS_THAN_OR_EQUAL) || (flip && kind == SqlKind.GREATER_THAN_OR_EQUAL)) { - return Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); - } else { - throw new IllegalStateException("WTF?! Shouldn't have got here..."); + switch (flippedKind) { + case EQUALS: + return rhsAligned + ? Bounds.interval(boundRefKey, rhsInterval) + : Filtration.matchNothing(); + case NOT_EQUALS: + return rhsAligned + ? new NotDimFilter(Bounds.interval(boundRefKey, rhsInterval)) + : Filtration.matchEverything(); + case GREATER_THAN: + return Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); + case GREATER_THAN_OR_EQUAL: + return rhsAligned + ? Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) + : Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); + case LESS_THAN: + return rhsAligned + ? Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) + : Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); + case LESS_THAN_OR_EQUAL: + return Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); + default: + throw new IllegalStateException("WTF?! Shouldn't have got here..."); } } } @@ -412,20 +458,27 @@ private static DimFilter toSimpleLeafFilter( final DimFilter filter; // Always use BoundDimFilters, to simplify filter optimization later (it helps to remember the comparator). - if (kind == SqlKind.EQUALS) { - filter = Bounds.equalTo(boundRefKey, val); - } else if (kind == SqlKind.NOT_EQUALS) { - filter = new NotDimFilter(Bounds.equalTo(boundRefKey, val)); - } else if ((!flip && kind == SqlKind.GREATER_THAN) || (flip && kind == SqlKind.LESS_THAN)) { - filter = Bounds.greaterThan(boundRefKey, val); - } else if ((!flip && kind == SqlKind.GREATER_THAN_OR_EQUAL) || (flip && kind == SqlKind.LESS_THAN_OR_EQUAL)) { - filter = Bounds.greaterThanOrEqualTo(boundRefKey, val); - } else if ((!flip && kind == SqlKind.LESS_THAN) || (flip && kind == SqlKind.GREATER_THAN)) { - filter = Bounds.lessThan(boundRefKey, val); - } else if ((!flip && kind == SqlKind.LESS_THAN_OR_EQUAL) || (flip && kind == SqlKind.GREATER_THAN_OR_EQUAL)) { - filter = Bounds.lessThanOrEqualTo(boundRefKey, val); - } else { - throw new IllegalStateException("WTF?! Shouldn't have got here..."); + switch (flippedKind) { + case EQUALS: + filter = Bounds.equalTo(boundRefKey, val); + break; + case NOT_EQUALS: + filter = new NotDimFilter(Bounds.equalTo(boundRefKey, val)); + break; + case GREATER_THAN: + filter = Bounds.greaterThan(boundRefKey, val); + break; + case GREATER_THAN_OR_EQUAL: + filter = Bounds.greaterThanOrEqualTo(boundRefKey, val); + break; + case LESS_THAN: + filter = Bounds.lessThan(boundRefKey, val); + break; + case LESS_THAN_OR_EQUAL: + filter = Bounds.lessThanOrEqualTo(boundRefKey, val); + break; + default: + throw new IllegalStateException("WTF?! Shouldn't have got here..."); } return filter; @@ -480,4 +533,86 @@ private static DimFilter toExpressionLeafFilter( ? null : new ExpressionDimFilter(druidExpression.getExpression(), plannerContext.getExprMacroTable()); } + + /** + * Converts an expression to a Granularity, if possible. This is possible if, and only if, the expression + * is a timestamp_floor function on the __time column with literal parameters for period, origin, and timeZone. + * + * @return granularity or null if not possible + */ + @Nullable + public static Granularity toQueryGranularity(final DruidExpression expression, final ExprMacroTable macroTable) + { + final TimestampFloorExprMacro.TimestampFloorExpr expr = asTimestampFloorExpr(expression, macroTable); + + if (expr == null) { + return null; + } + + final Expr arg = expr.getArg(); + final Granularity granularity = expr.getGranularity(); + + if (Column.TIME_COLUMN_NAME.equals(Parser.getIdentifierIfIdentifier(arg))) { + return granularity; + } else { + return null; + } + } + + @Nullable + public static TimestampFloorExprMacro.TimestampFloorExpr asTimestampFloorExpr( + final DruidExpression expression, + final ExprMacroTable macroTable + ) + { + final Expr expr = Parser.parse(expression.getExpression(), macroTable); + + if (expr instanceof TimestampFloorExprMacro.TimestampFloorExpr) { + return (TimestampFloorExprMacro.TimestampFloorExpr) expr; + } else { + return null; + } + } + + /** + * Build a filter for an expression like FLOOR(column TO granularity) [operator] rhsMillis + */ + private static DimFilter buildTimeFloorFilter( + final String column, + final Granularity granularity, + final SqlKind operatorKind, + final long rhsMillis + ) + { + final BoundRefKey boundRefKey = new BoundRefKey(column, null, StringComparators.NUMERIC); + final Interval rhsInterval = granularity.bucket(DateTimes.utc(rhsMillis)); + + // Is rhs aligned on granularity boundaries? + final boolean rhsAligned = rhsInterval.getStartMillis() == rhsMillis; + + switch (operatorKind) { + case EQUALS: + return rhsAligned + ? Bounds.interval(boundRefKey, rhsInterval) + : Filtration.matchNothing(); + case NOT_EQUALS: + return rhsAligned + ? new NotDimFilter(Bounds.interval(boundRefKey, rhsInterval)) + : Filtration.matchEverything(); + case GREATER_THAN: + return Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); + case GREATER_THAN_OR_EQUAL: + return rhsAligned + ? Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) + : Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); + case LESS_THAN: + return rhsAligned + ? Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) + : Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); + case LESS_THAN_OR_EQUAL: + return Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis())); + default: + throw new IllegalStateException("WTF?! Shouldn't have got here..."); + } + } } diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CastOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CastOperatorConversion.java index 7cc7acaf4374..e572cc18dbb1 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CastOperatorConversion.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CastOperatorConversion.java @@ -126,7 +126,8 @@ public DruidExpression toDruidExpression( // Floor to day when casting to DATE. return TimeFloorOperatorConversion.applyTimestampFloor( typeCastExpression, - new PeriodGranularity(Period.days(1), null, plannerContext.getTimeZone()) + new PeriodGranularity(Period.days(1), null, plannerContext.getTimeZone()), + plannerContext.getExprMacroTable() ); } else { return typeCastExpression; @@ -153,7 +154,8 @@ private static DruidExpression castCharToDateTime( if (toType == SqlTypeName.DATE) { return TimeFloorOperatorConversion.applyTimestampFloor( timestampExpression, - new PeriodGranularity(Period.days(1), null, plannerContext.getTimeZone()) + new PeriodGranularity(Period.days(1), null, plannerContext.getTimeZone()), + plannerContext.getExprMacroTable() ); } else if (toType == SqlTypeName.TIMESTAMP) { return timestampExpression; diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/FloorOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/FloorOperatorConversion.java index d2674a3d0e60..9d75a2942d3d 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/FloorOperatorConversion.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/FloorOperatorConversion.java @@ -73,7 +73,11 @@ public DruidExpression toDruidExpression( return null; } - return TimeFloorOperatorConversion.applyTimestampFloor(druidExpression, granularity); + return TimeFloorOperatorConversion.applyTimestampFloor( + druidExpression, + granularity, + plannerContext.getExprMacroTable() + ); } else { // WTF? FLOOR with 3 arguments? return null; diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/TimeFloorOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/TimeFloorOperatorConversion.java index 3dde8c42f678..7acb2c11297d 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/TimeFloorOperatorConversion.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/TimeFloorOperatorConversion.java @@ -20,10 +20,12 @@ package io.druid.sql.calcite.expression.builtin; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import io.druid.java.util.common.granularity.PeriodGranularity; +import io.druid.math.expr.ExprMacroTable; +import io.druid.query.expression.TimestampFloorExprMacro; import io.druid.sql.calcite.expression.DruidExpression; import io.druid.sql.calcite.expression.Expressions; -import io.druid.sql.calcite.expression.ExtractionFns; import io.druid.sql.calcite.expression.OperatorConversions; import io.druid.sql.calcite.expression.SqlOperatorConversion; import io.druid.sql.calcite.planner.Calcites; @@ -42,8 +44,8 @@ import org.joda.time.DateTimeZone; import org.joda.time.Period; -import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; public class TimeFloorOperatorConversion implements SqlOperatorConversion @@ -58,28 +60,52 @@ public class TimeFloorOperatorConversion implements SqlOperatorConversion public static DruidExpression applyTimestampFloor( final DruidExpression input, - final PeriodGranularity granularity + final PeriodGranularity granularity, + final ExprMacroTable macroTable ) { Preconditions.checkNotNull(input, "input"); Preconditions.checkNotNull(granularity, "granularity"); - return input.map( - simpleExtraction -> simpleExtraction.cascade(ExtractionFns.fromQueryGranularity(granularity)), - expression -> DruidExpression.functionCall( - "timestamp_floor", - Arrays.asList( - expression, - DruidExpression.stringLiteral(granularity.getPeriod().toString()), - DruidExpression.numberLiteral( - granularity.getOrigin() == null ? null : granularity.getOrigin().getMillis() - ), - DruidExpression.stringLiteral(granularity.getTimeZone().toString()) - ).stream().map(DruidExpression::fromExpression).collect(Collectors.toList()) - ) + // Collapse floor chains if possible. Useful for constructs like CAST(FLOOR(__time TO QUARTER) AS DATE). + if (granularity.getPeriod().equals(Period.days(1))) { + final TimestampFloorExprMacro.TimestampFloorExpr floorExpr = Expressions.asTimestampFloorExpr( + input, + macroTable + ); + + if (floorExpr != null) { + final PeriodGranularity inputGranularity = floorExpr.getGranularity(); + if (Objects.equals(inputGranularity.getTimeZone(), granularity.getTimeZone()) + && Objects.equals(inputGranularity.getOrigin(), granularity.getOrigin()) + && periodIsDayMultiple(inputGranularity.getPeriod())) { + return input; + } + } + } + + return DruidExpression.fromFunctionCall( + "timestamp_floor", + ImmutableList.of( + input.getExpression(), + DruidExpression.stringLiteral(granularity.getPeriod().toString()), + DruidExpression.numberLiteral( + granularity.getOrigin() == null ? null : granularity.getOrigin().getMillis() + ), + DruidExpression.stringLiteral(granularity.getTimeZone().toString()) + ).stream().map(DruidExpression::fromExpression).collect(Collectors.toList()) ); } + private static boolean periodIsDayMultiple(final Period period) + { + return period.getMillis() == 0 + && period.getSeconds() == 0 + && period.getMinutes() == 0 + && period.getHours() == 0 + && (period.getDays() > 0 || period.getWeeks() > 0 || period.getMonths() > 0 || period.getYears() > 0); + } + @Override public SqlOperator calciteOperator() { @@ -117,7 +143,7 @@ public DruidExpression toDruidExpression( ? DateTimeZone.forID(RexLiteral.stringValue(operands.get(3))) : plannerContext.getTimeZone(); final PeriodGranularity granularity = new PeriodGranularity(period, origin, timeZone); - return applyTimestampFloor(druidExpressions.get(0), granularity); + return applyTimestampFloor(druidExpressions.get(0), granularity, plannerContext.getExprMacroTable()); } else { // Granularity is dynamic return DruidExpression.fromFunctionCall("timestamp_floor", druidExpressions); diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/io/druid/sql/calcite/rel/DruidQuery.java index b4f2c14caa91..bca4481992ff 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/DruidQuery.java @@ -64,7 +64,6 @@ import io.druid.sql.calcite.aggregation.DimensionExpression; import io.druid.sql.calcite.expression.DruidExpression; import io.druid.sql.calcite.expression.Expressions; -import io.druid.sql.calcite.expression.ExtractionFns; import io.druid.sql.calcite.filtration.Filtration; import io.druid.sql.calcite.planner.Calcites; import io.druid.sql.calcite.planner.PlannerContext; @@ -537,13 +536,15 @@ private static boolean postAggregatorDirectColumnIsOk( return toExprType.equals(fromExprType); } - public VirtualColumns getVirtualColumns(final ExprMacroTable macroTable) + public VirtualColumns getVirtualColumns(final ExprMacroTable macroTable, final boolean includeDimensions) { final List retVal = new ArrayList<>(); if (grouping != null) { - for (DimensionExpression dimensionExpression : grouping.getDimensions()) { - retVal.addAll(dimensionExpression.getVirtualColumns(macroTable)); + if (includeDimensions) { + for (DimensionExpression dimensionExpression : grouping.getDimensions()) { + retVal.addAll(dimensionExpression.getVirtualColumns(macroTable)); + } } for (Aggregation aggregation : grouping.getAggregations()) { @@ -653,14 +654,15 @@ public TimeseriesQuery toTimeseriesQuery() queryGranularity = Granularities.ALL; descending = false; } else if (grouping.getDimensions().size() == 1) { - final DimensionSpec dimensionSpec = Iterables.getOnlyElement(grouping.getDimensions()).toDimensionSpec(); - final Granularity gran = ExtractionFns.toQueryGranularity(dimensionSpec.getExtractionFn()); + final DimensionExpression dimensionExpression = Iterables.getOnlyElement(grouping.getDimensions()); + queryGranularity = Expressions.toQueryGranularity( + dimensionExpression.getDruidExpression(), + plannerContext.getExprMacroTable() + ); - if (gran == null || !dimensionSpec.getDimension().equals(Column.TIME_COLUMN_NAME)) { + if (queryGranularity == null) { // Timeseries only applies if the single dimension is granular __time. return null; - } else { - queryGranularity = gran; } if (limitSpec != null) { @@ -677,7 +679,7 @@ public TimeseriesQuery toTimeseriesQuery() // wouldn't matter anyway). final OrderByColumnSpec firstOrderBy = limitSpec.getColumns().get(0); - if (firstOrderBy.getDimension().equals(dimensionSpec.getOutputName())) { + if (firstOrderBy.getDimension().equals(dimensionExpression.getOutputName())) { // Order by time. descending = firstOrderBy.getDirection() == OrderByColumnSpec.Direction.DESCENDING; } else { @@ -703,7 +705,7 @@ public TimeseriesQuery toTimeseriesQuery() dataSource, filtration.getQuerySegmentSpec(), descending, - getVirtualColumns(plannerContext.getExprMacroTable()), + getVirtualColumns(plannerContext.getExprMacroTable(), false), filtration.getDimFilter(), queryGranularity, grouping.getAggregatorFactories(), @@ -768,7 +770,7 @@ public TopNQuery toTopNQuery() return new TopNQuery( dataSource, - getVirtualColumns(plannerContext.getExprMacroTable()), + getVirtualColumns(plannerContext.getExprMacroTable(), true), dimensionSpec, topNMetricSpec, limitSpec.getLimit(), @@ -798,7 +800,7 @@ public GroupByQuery toGroupByQuery() return new GroupByQuery( dataSource, filtration.getQuerySegmentSpec(), - getVirtualColumns(plannerContext.getExprMacroTable()), + getVirtualColumns(plannerContext.getExprMacroTable(), true), filtration.getDimFilter(), Granularities.ALL, grouping.getDimensionSpecs(), @@ -927,7 +929,7 @@ public SelectQuery toSelectQuery() Granularities.ALL, ImmutableList.of(new DefaultDimensionSpec(dummyColumn, dummyColumn)), metrics.stream().sorted().distinct().collect(Collectors.toList()), - getVirtualColumns(plannerContext.getExprMacroTable()), + getVirtualColumns(plannerContext.getExprMacroTable(), true), pagingSpec, ImmutableSortedMap.copyOf(plannerContext.getQueryContext()) ); diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 93d44fccadef..21d04bce1cc3 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -59,7 +59,6 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.RegexDimExtractionFn; import io.druid.query.extraction.SubstringDimExtractionFn; -import io.druid.query.extraction.TimeFormatExtractionFn; import io.druid.query.filter.AndDimFilter; import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; @@ -1631,41 +1630,87 @@ public void testGroupByCaseWhen() throws Exception } @Test - public void testNullEmptyStringEqualityBackwardsCompatibility() throws Exception + public void testIsNullString() throws Exception { - if(!NullHandlingHelper.useDefaultValuesForNull()){ - // This test only makes sense when nulls and empty strings are considered equal. - return; - } - // Doesn't conform to the SQL standard, but it's how we do it. - // This example is used in the sql.md doc. + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE NULLIF(dim2, 'a') IS NULL", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,isnull(\"dim2\"))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + NullHandlingHelper.useDefaultValuesForNull() ? + // Matches everything but "abc" + new Object[]{5L} : + // match only null values + new Object[]{4L} + ) + ); + } + + @Test + public void testEmptyStringEquality() throws Exception + { + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE NULLIF(dim2, 'a') = ''", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," + + (NullHandlingHelper.useDefaultValuesForNull() ? "1" : "0") + + ",(\"dim2\" == ''))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + NullHandlingHelper.useDefaultValuesForNull() ? + // Matches everything but "abc" + new Object[]{5L} : + // match only empty string + new Object[]{1L} + ) + ); + } + + @Test + public void testNullStringEquality() throws Exception + { + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE NULLIF(dim2, 'a') = null", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," + + (NullHandlingHelper.useDefaultValuesForNull() ? "1" : "0") + + ",(\"dim2\" == null))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + NullHandlingHelper.useDefaultValuesForNull() ? + // Matches everything but "abc" + ImmutableList.of(new Object[]{5L}) : + // null is not eqaual to null or any other value + ImmutableList.of() + ); - final ImmutableList wheres = ImmutableList.of( - "NULLIF(dim2, 'a') = ''", - "NULLIF(dim2, 'a') IS NULL" - ); - - for (String where : wheres) { - testQuery( - "SELECT COUNT(*)\n" - + "FROM druid.foo\n" - + "WHERE " + where, - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,(\"dim2\" == ''))")) - .aggregators(AGGS(new CountAggregatorFactory("a0"))) - .context(TIMESERIES_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - // Matches everything but "abc" - new Object[]{5L} - ) - ); - } } @Test @@ -3371,6 +3416,9 @@ public void testTimeseriesWithTimeFilterOnLongColumnUsingMillisToTimestamp() thr .setDataSource(CalciteTests.DATASOURCE1) .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) + .setVirtualColumns( + EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"cnt\",'P1Y',null,'UTC')", ValueType.LONG) + ) .setDimFilter( BOUND( "cnt", @@ -3382,14 +3430,7 @@ public void testTimeseriesWithTimeFilterOnLongColumnUsingMillisToTimestamp() thr StringComparators.NUMERIC ) ) - .setDimensions(DIMS( - new ExtractionDimensionSpec( - "cnt", - "d0", - ValueType.LONG, - new TimeFormatExtractionFn(null, null, null, Granularities.YEAR, true) - ) - )) + .setDimensions(DIMS(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG))) .setAggregatorSpecs(AGGS(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) .build() @@ -3925,13 +3966,15 @@ public void testNestedGroupBy() throws Exception ) .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) + .setVirtualColumns( + EXPRESSION_VIRTUAL_COLUMN( + "_d0:v", + "timestamp_floor(\"a0\",'PT1H',null,'UTC')", + ValueType.LONG + ) + ) .setDimensions(DIMS( - new ExtractionDimensionSpec( - "a0", - "_d0", - ValueType.LONG, - new TimeFormatExtractionFn(null, null, null, Granularities.HOUR, true) - ), + new DefaultDimensionSpec("_d0:v", "_d0", ValueType.LONG), new DefaultDimensionSpec("d0", "_d1", ValueType.STRING) )) .setAggregatorSpecs(AGGS( @@ -5259,14 +5302,16 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Excepti .setDataSource(CalciteTests.DATASOURCE1) .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) + .setVirtualColumns( + EXPRESSION_VIRTUAL_COLUMN( + "d0:v", + "timestamp_floor(\"__time\",'P1Y',null,'UTC')", + ValueType.LONG + ) + ) .setDimensions( DIMS( - new ExtractionDimensionSpec( - "__time", - "d0", - ValueType.LONG, - new TimeFormatExtractionFn(null, null, null, Granularities.YEAR, true) - ), + new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG), new DefaultDimensionSpec("dim2", "d1") ) ) @@ -5845,6 +5890,32 @@ public void testTimeseriesUsingCastAsDate() throws Exception ); } + @Test + public void testTimeseriesUsingFloorPlusCastAsDate() throws Exception + { + testQuery( + "SELECT SUM(cnt), dt FROM (\n" + + " SELECT CAST(FLOOR(__time TO QUARTER) AS DATE) AS dt,\n" + + " cnt FROM druid.foo\n" + + ") AS x\n" + + "GROUP BY dt\n" + + "ORDER BY dt", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(new PeriodGranularity(Period.months(3), null, DateTimeZone.UTC)) + .aggregators(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{3L, D("2000-01-01")}, + new Object[]{3L, D("2001-01-01")} + ) + ); + } + @Test public void testTimeseriesDescending() throws Exception { @@ -6051,16 +6122,14 @@ public void testTimeseriesWithLimitNoTopN() throws Exception .setDataSource(CalciteTests.DATASOURCE1) .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) - .setDimensions( - DIMS( - new ExtractionDimensionSpec( - "__time", - "d0", - ValueType.LONG, - new TimeFormatExtractionFn(null, null, null, Granularities.MONTH, true) - ) + .setVirtualColumns( + EXPRESSION_VIRTUAL_COLUMN( + "d0:v", + "timestamp_floor(\"__time\",'P1M',null,'UTC')", + ValueType.LONG ) ) + .setDimensions(DIMS(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG))) .setAggregatorSpecs(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec( new DefaultLimitSpec( @@ -6099,14 +6168,10 @@ public void testTimeseriesWithLimit() throws Exception .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) - .dimension( - new ExtractionDimensionSpec( - "__time", - "d0", - ValueType.LONG, - new TimeFormatExtractionFn(null, null, null, Granularities.MONTH, true) - ) + .virtualColumns( + EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG) ) + .dimension(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG)) .aggregators(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) .metric(new DimensionTopNMetricSpec(null, StringComparators.NUMERIC)) .threshold(1) @@ -6136,14 +6201,10 @@ public void testTimeseriesWithOrderByAndLimit() throws Exception .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) - .dimension( - new ExtractionDimensionSpec( - "__time", - "d0", - ValueType.LONG, - new TimeFormatExtractionFn(null, null, null, Granularities.MONTH, true) - ) + .virtualColumns( + EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG) ) + .dimension(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG)) .aggregators(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) .metric(new DimensionTopNMetricSpec(null, StringComparators.NUMERIC)) .threshold(1) @@ -6169,15 +6230,17 @@ public void testGroupByTimeAndOtherDimension() throws Exception .setDataSource(CalciteTests.DATASOURCE1) .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) + .setVirtualColumns( + EXPRESSION_VIRTUAL_COLUMN( + "d1:v", + "timestamp_floor(\"__time\",'P1M',null,'UTC')", + ValueType.LONG + ) + ) .setDimensions( DIMS( new DefaultDimensionSpec("dim2", "d0"), - new ExtractionDimensionSpec( - "__time", - "d1", - ValueType.LONG, - new TimeFormatExtractionFn(null, null, null, Granularities.MONTH, true) - ) + new DefaultDimensionSpec("d1:v", "d1", ValueType.LONG) ) ) .setAggregatorSpecs(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) @@ -6341,11 +6404,13 @@ public void testUsingSubqueryAsFilterOnTwoColumns() throws Exception newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) - .filters(OR(SELECTOR("dim1", "def", null), - AND( - SELECTOR("dim1", "def", null), - SELECTOR("dim2", "abc", null) - ))) + .filters(OR( + SELECTOR("dim1", "def", null), + AND( + SELECTOR("dim1", "def", null), + SELECTOR("dim2", "abc", null) + ) + )) .columns("__time", "cnt", "dim1", "dim2") .resultFormat(ScanQuery.RESULT_FORMAT_COMPACTED_LIST) .context(QUERY_CONTEXT_DEFAULT) diff --git a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java index 7f4994b745ca..15abe5590b37 100644 --- a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java @@ -22,12 +22,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.druid.java.util.common.DateTimes; -import io.druid.java.util.common.granularity.Granularities; -import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.math.expr.ExprEval; import io.druid.math.expr.Parser; import io.druid.query.extraction.RegexDimExtractionFn; -import io.druid.query.extraction.TimeFormatExtractionFn; import io.druid.segment.column.ValueType; import io.druid.server.security.AuthTestUtils; import io.druid.sql.calcite.expression.builtin.DateTruncOperatorConversion; @@ -395,9 +392,7 @@ public void testTimeFloor() timestampLiteral(DateTimes.of("2000-02-03T04:05:06Z")), rexBuilder.makeLiteral("PT1H") ), - DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H'," - + DruidExpression.nullLiteral() - + ",'UTC')"), + DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H',null,'UTC')"), DateTimes.of("2000-02-03T04:00:00").getMillis() ); @@ -409,19 +404,7 @@ public void testTimeFloor() rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.TIMESTAMP)), rexBuilder.makeLiteral("America/Los_Angeles") ), - DruidExpression.of( - SimpleExtraction.of( - "t", - new TimeFormatExtractionFn( - null, - null, - null, - new PeriodGranularity(Period.days(1), null, LOS_ANGELES), - true - ) - ), - "timestamp_floor(\"t\",'P1D'," + DruidExpression.nullLiteral() + ",'America/Los_Angeles')" - ), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D',null,'America/Los_Angeles')"), DateTimes.of("2000-02-02T08:00:00").getMillis() ); } @@ -437,13 +420,7 @@ public void testOtherTimeFloor() inputRef("t"), rexBuilder.makeFlag(TimeUnitRange.YEAR) ), - DruidExpression.of( - SimpleExtraction.of( - "t", - new TimeFormatExtractionFn(null, null, null, Granularities.YEAR, true) - ), - "timestamp_floor(\"t\",'P1Y'," + DruidExpression.nullLiteral() + ",'UTC')" - ), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1Y',null,'UTC')"), DateTimes.of("2000").getMillis() ); } @@ -459,7 +436,7 @@ public void testOtherTimeCeil() inputRef("t"), rexBuilder.makeFlag(TimeUnitRange.YEAR) ), - DruidExpression.fromExpression("timestamp_ceil(\"t\",'P1Y'," + DruidExpression.nullLiteral() + ",'UTC')"), + DruidExpression.fromExpression("timestamp_ceil(\"t\",'P1Y',null,'UTC')"), DateTimes.of("2001").getMillis() ); } @@ -741,10 +718,7 @@ public void testCastAsDate() typeFactory.createSqlType(SqlTypeName.DATE), inputRef("t") ), - DruidExpression.of( - SimpleExtraction.of("t", new TimeFormatExtractionFn(null, null, null, Granularities.DAY, true)), - "timestamp_floor(\"t\",'P1D'," + DruidExpression.nullLiteral() + ",'UTC')" - ), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D',null,'UTC')"), DateTimes.of("2000-02-03").getMillis() ); @@ -785,10 +759,7 @@ public void testCastFromDate() inputRef("t") ) ), - DruidExpression.of( - SimpleExtraction.of("t", new TimeFormatExtractionFn(null, null, null, Granularities.DAY, true)), - "timestamp_floor(\"t\",'P1D'," + DruidExpression.nullLiteral() + ",'UTC')" - ), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D',null,'UTC')"), DateTimes.of("2000-02-03").getMillis() ); } diff --git a/sql/src/test/java/io/druid/sql/calcite/schema/DruidSchemaTest.java b/sql/src/test/java/io/druid/sql/calcite/schema/DruidSchemaTest.java index dfdef39e8362..96e5dba2870d 100644 --- a/sql/src/test/java/io/druid/sql/calcite/schema/DruidSchemaTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/schema/DruidSchemaTest.java @@ -24,13 +24,13 @@ import com.google.common.collect.ImmutableSet; import io.druid.data.input.InputRow; import io.druid.java.util.common.Intervals; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.segment.IndexBuilder; import io.druid.segment.QueryableIndex; -import io.druid.segment.TestHelper; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.server.security.NoopEscalator; import io.druid.sql.calcite.planner.Calcites; @@ -88,7 +88,7 @@ public void setUp() throws Exception final File tmpDir = temporaryFolder.newFolder(); final QueryableIndex index1 = IndexBuilder.create() .tmpDir(new File(tmpDir, "1")) - .indexMerger(TestHelper.getTestIndexMergerV9()) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( new IncrementalIndexSchema.Builder() .withMetrics( @@ -104,7 +104,7 @@ public void setUp() throws Exception final QueryableIndex index2 = IndexBuilder.create() .tmpDir(new File(tmpDir, "2")) - .indexMerger(TestHelper.getTestIndexMergerV9()) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( new IncrementalIndexSchema.Builder() .withMetrics(new LongSumAggregatorFactory("m1", "m1")) diff --git a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java index 8757151ea033..40900b3102eb 100644 --- a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java @@ -44,6 +44,7 @@ import io.druid.guice.ExpressionModule; import io.druid.guice.annotations.Json; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.DefaultGenericQueryMetricsFactory; import io.druid.query.DefaultQueryRunnerFactoryConglomerate; import io.druid.query.DruidProcessingConfig; @@ -422,26 +423,29 @@ public static ObjectMapper getJsonMapper() public static SpecificSegmentsQuerySegmentWalker createMockWalker(final File tmpDir) { - final QueryableIndex index1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .indexMerger(TestHelper.getTestIndexMergerV9()) - .schema(INDEX_SCHEMA) - .rows(ROWS1) - .buildMMappedIndex(); - - final QueryableIndex index2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "2")) - .indexMerger(TestHelper.getTestIndexMergerV9()) - .schema(INDEX_SCHEMA) - .rows(ROWS2) - .buildMMappedIndex(); - - final QueryableIndex forbiddenIndex = IndexBuilder.create() - .tmpDir(new File(tmpDir, "forbidden")) - .indexMerger(TestHelper.getTestIndexMergerV9()) - .schema(INDEX_SCHEMA) - .rows(FORBIDDEN_ROWS) - .buildMMappedIndex(); + final QueryableIndex index1 = IndexBuilder + .create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema(INDEX_SCHEMA) + .rows(ROWS1) + .buildMMappedIndex(); + + final QueryableIndex index2 = IndexBuilder + .create() + .tmpDir(new File(tmpDir, "2")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema(INDEX_SCHEMA) + .rows(ROWS2) + .buildMMappedIndex(); + + final QueryableIndex forbiddenIndex = IndexBuilder + .create() + .tmpDir(new File(tmpDir, "forbidden")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema(INDEX_SCHEMA) + .rows(FORBIDDEN_ROWS) + .buildMMappedIndex(); return new SpecificSegmentsQuerySegmentWalker(queryRunnerFactoryConglomerate()).add( DataSegment.builder() @@ -528,18 +532,18 @@ public static DruidSchema createMockSchema( public static InputRow createRow(final ImmutableMap map) { - return PARSER.parse((Map) map); + return PARSER.parseBatch((Map) map).get(0); } public static InputRow createRow(final Object t, final String dim1, final String dim2, final double m1) { - return PARSER.parse( + return PARSER.parseBatch( ImmutableMap.of( "t", new DateTime(t, ISOChronology.getInstanceUTC()).getMillis(), "dim1", dim1, "dim2", dim2, "m1", m1 ) - ); + ).get(0); } } From 41f6261d71b6510ae21263dd434f5925dd1747c8 Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 8 Dec 2017 03:42:47 +0530 Subject: [PATCH 37/50] fix strict compilation --- .../main/java/io/druid/query/aggregation/AggregateCombiner.java | 1 + .../io/druid/query/groupby/RowBasedColumnSelectorFactory.java | 1 + processing/src/main/java/io/druid/segment/DimensionSelector.java | 1 + .../io/druid/segment/column/IndexedDoublesGenericColumn.java | 1 + .../main/java/io/druid/segment/virtual/ExpressionSelectors.java | 1 + 5 files changed, 5 insertions(+) diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java index 2bb418359a52..797f44843837 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java @@ -76,6 +76,7 @@ default boolean isNull() return false; } + @Override default void inspectRuntimeShape(RuntimeShapeInspector inspector) { // Usually AggregateCombiner has nothing to inspect diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java index c55ff3735b45..ff93c5562ba2 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java @@ -404,6 +404,7 @@ public boolean isNull() return false; } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("row", row); diff --git a/processing/src/main/java/io/druid/segment/DimensionSelector.java b/processing/src/main/java/io/druid/segment/DimensionSelector.java index b65fb8ce6b05..8e30f944b558 100644 --- a/processing/src/main/java/io/druid/segment/DimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelector.java @@ -173,6 +173,7 @@ default Object getObject() * This method is not the default implementation of {@link #getObject()} to minimize the chance that implementations * "forget" to override it with more optimized version. */ + @Deprecated @Nullable default Object defaultGetObject() { diff --git a/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java index 0c50e00d9baa..414d82e1af47 100644 --- a/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedDoublesGenericColumn.java @@ -73,6 +73,7 @@ public boolean isNull(int rowNum) return nullValueBitmap.get(rowNum); } + @Override public void close() { column.close(); diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index b2e95ae3eea6..9700ef0c6235 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -83,6 +83,7 @@ public boolean isNull() return baseSelector.getObject().isNull(); } + @Override public float getFloat() { return baseSelector.getFloat(); From d36fcca2643d7881410e35b342697ecdf0ddbc0b Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 11 Dec 2017 02:19:54 +0530 Subject: [PATCH 38/50] more review comments --- .../main/java/io/druid/data/input/Rows.java | 2 +- .../druid/common/config/NullHandlingUtil.java | 16 +++++--- .../DistinctCountAggregator.java | 37 +++++++++---------- .../DistinctCountBufferAggregator.java | 30 +++++++-------- .../io/druid/segment/NullHandlingHelper.java | 29 ++++++++++----- 5 files changed, 62 insertions(+), 52 deletions(-) diff --git a/api/src/main/java/io/druid/data/input/Rows.java b/api/src/main/java/io/druid/data/input/Rows.java index 6d72093a8211..7448e10e373d 100644 --- a/api/src/main/java/io/druid/data/input/Rows.java +++ b/api/src/main/java/io/druid/data/input/Rows.java @@ -94,7 +94,7 @@ public static List objectToStrings(final Object inputValue) * @throws ParseException if the column cannot be converted to a number */ @Nullable - public static Number objectToNumber(final String name, final Object inputValue) + public static Number objectToNumber(final String name, @Nullable final Object inputValue) { if (inputValue == null) { return null; diff --git a/common/src/main/java/io/druid/common/config/NullHandlingUtil.java b/common/src/main/java/io/druid/common/config/NullHandlingUtil.java index e053c0947ba8..df7eacbc7419 100644 --- a/common/src/main/java/io/druid/common/config/NullHandlingUtil.java +++ b/common/src/main/java/io/druid/common/config/NullHandlingUtil.java @@ -28,11 +28,13 @@ public class NullHandlingUtil { private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; - // INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. - // See NullHandlingModule for details. - // It does not take effect in all unit tests since we don't use Guice Injection. - // For tests default system property is supposed to be used only in tests + /** + * INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. + * See {@link io.druid.guice.NullHandlingModule} for details. + * It does not take effect in all unit tests since we don't use Guice Injection. + * For tests default system property is supposed to be used only in tests + */ @Inject private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) @@ -43,12 +45,14 @@ public static boolean useDefaultValuesForNull() return INSTANCE.isUseDefaultValuesForNull(); } - public static @Nullable String nullToEmptyIfNeeded(@Nullable String value) + @Nullable + public static String nullToEmptyIfNeeded(@Nullable String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; } - public static @Nullable String emptyToNullIfNeeded(@Nullable String value) + @Nullable + public static String emptyToNullIfNeeded(@Nullable String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java index 31d860359c69..d802b96307eb 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java @@ -23,12 +23,14 @@ import io.druid.query.aggregation.Aggregator; import io.druid.segment.DimensionSelector; import io.druid.segment.NullHandlingHelper; +import io.druid.segment.data.IndexedInts; public class DistinctCountAggregator implements Aggregator { - + private static final int UNKNOWN_ID = -1; private final DimensionSelector selector; private final MutableBitmap mutableBitmap; + private final int idForNull; public DistinctCountAggregator( DimensionSelector selector, @@ -37,18 +39,24 @@ public DistinctCountAggregator( { this.selector = selector; this.mutableBitmap = mutableBitmap; + this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : UNKNOWN_ID; } @Override public void aggregate() { - - boolean countNulls = !selector.nameLookupPossibleInAdvance() || NullHandlingHelper.useDefaultValuesForNull(); - selector.getRow().forEach(index -> { - if (countNulls || selector.lookupName(index) != null) { + IndexedInts row = selector.getRow(); + for (int i = 0; i < row.size(); i++) { + int index = row.get(i); + if (NullHandlingHelper.useDefaultValuesForNull() || isNotNull(index)) { mutableBitmap.add(index); } - }); + } + } + + private boolean isNotNull(int index) + { + return selector.nameLookupPossibleInAdvance() ? index != idForNull : selector.lookupName(index) != null; } @Override @@ -60,13 +68,13 @@ public void reset() @Override public Object get() { - return countValues(); + return mutableBitmap.size(); } @Override public float getFloat() { - return (float) countValues(); + return (float) mutableBitmap.size(); } @Override @@ -78,21 +86,12 @@ public void close() @Override public long getLong() { - return (long) countValues(); + return (long) mutableBitmap.size(); } @Override public double getDouble() { - return (double) countValues(); - } - - private int countValues() - { - if (!selector.nameLookupPossibleInAdvance() || NullHandlingHelper.useDefaultValuesForNull()) { - return mutableBitmap.size(); - } - int nullId = selector.idLookup().lookupId(null); - return mutableBitmap.get(nullId) ? mutableBitmap.size() - 1 : mutableBitmap.size(); + return (double) mutableBitmap.size(); } } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index ab5b899a2195..e9fd222a4e6d 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -33,14 +33,18 @@ public class DistinctCountBufferAggregator implements BufferAggregator { + private static final int UNKNOWN_ID = -1; private final DimensionSelector selector; private final Int2ObjectMap mutableBitmapCollection = new Int2ObjectOpenHashMap<>(); + private final int idForNull; + public DistinctCountBufferAggregator( DimensionSelector selector ) { this.selector = selector; + this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : UNKNOWN_ID; } @Override @@ -56,9 +60,8 @@ public void aggregate(ByteBuffer buf, int position) MutableBitmap mutableBitmap = getMutableBitmap(position); IndexedInts row = selector.getRow(); for (int i = 0; i < row.size(); i++) { - int index = row.get(i); - if (countNulls || selector.lookupName(index) != null) { + if (NullHandlingHelper.useDefaultValuesForNull() || isNotNull(index)) { mutableBitmap.add(index); } } @@ -75,28 +78,33 @@ private MutableBitmap getMutableBitmap(int position) return mutableBitmap; } + private boolean isNotNull(int index) + { + return selector.nameLookupPossibleInAdvance() ? index != idForNull : selector.lookupName(index) != null; + } + @Override public Object get(ByteBuffer buf, int position) { - return countValues(buf, position); + return buf.getLong(position); } @Override public float getFloat(ByteBuffer buf, int position) { - return (float) countValues(buf, position); + return (float) buf.getLong(position); } @Override public long getLong(ByteBuffer buf, int position) { - return countValues(buf, position); + return buf.getLong(position); } @Override public double getDouble(ByteBuffer buf, int position) { - return (double) countValues(buf, position); + return (double) buf.getLong(position); } @Override @@ -110,14 +118,4 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("selector", selector); } - - private long countValues(ByteBuffer buf, int position) - { - MutableBitmap mutableBitmap = getMutableBitmap(position); - if (!selector.nameLookupPossibleInAdvance() || NullHandlingHelper.useDefaultValuesForNull()) { - return mutableBitmap.size(); - } - int nullId = selector.idLookup().lookupId(null); - return mutableBitmap.get(nullId) ? mutableBitmap.size() - 1 : mutableBitmap.size(); - } } diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java index 1e762843af58..bd561dd661a7 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/processing/src/main/java/io/druid/segment/NullHandlingHelper.java @@ -30,6 +30,8 @@ import io.druid.query.aggregation.NullableAggregator; import io.druid.query.aggregation.NullableBufferAggregator; +import javax.annotation.Nullable; + public class NullHandlingHelper { private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; @@ -40,10 +42,12 @@ public class NullHandlingHelper public static final Float ZERO_FLOAT = 0.0f; public static final Long ZERO_LONG = 0L; - // INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. - // See NullHandlingModule for details. - // It does not take effect in all unit tests since we don't use Guice Injection. - // For tests default system property is supposed to be used only in tests + /** + * INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. + * See {@link io.druid.guice.NullHandlingModule} for details. + * It does not take effect in all unit tests since we don't use Guice Injection. + * For tests default system property is supposed to be used only in tests + */ @Inject private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) @@ -54,32 +58,37 @@ public static boolean useDefaultValuesForNull() return INSTANCE.isUseDefaultValuesForNull(); } - public static String nullToEmptyIfNeeded(String value) + @Nullable + public static String nullToEmptyIfNeeded(@Nullable String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; } - public static String emptyToNullIfNeeded(String value) + @Nullable + public static String emptyToNullIfNeeded(@Nullable String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; } - public static boolean isNullOrEquivalent(String value) + public static boolean isNullOrEquivalent(@Nullable String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; } - public static Long nullToZeroIfNeeded(Long value) + @Nullable + public static Long nullToZeroIfNeeded(@Nullable Long value) { return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_LONG : value; } - public static Double nullToZeroIfNeeded(Double value) + @Nullable + public static Double nullToZeroIfNeeded(@Nullable Double value) { return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_DOUBLE : value; } - public static Float nullToZeroIfNeeded(Float value) + @Nullable + public static Float nullToZeroIfNeeded(@Nullable Float value) { return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_FLOAT : value; } From cf4fa7e89a7d3a35c687062a0d1831620bd90399 Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 11 Dec 2017 02:58:41 +0530 Subject: [PATCH 39/50] Add order by clause to make test deterministic and pass in travis. --- .../test/java/io/druid/sql/calcite/CalciteQueryTest.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 21d04bce1cc3..2559c051151f 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -5368,7 +5368,7 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Excepti public void testGroupByStringLength() throws Exception { testQuery( - "SELECT CHARACTER_LENGTH(dim1), COUNT(*) FROM druid.foo GROUP BY CHARACTER_LENGTH(dim1)", + "SELECT CHARACTER_LENGTH(dim1), COUNT(*) FROM druid.foo GROUP BY CHARACTER_LENGTH(dim1) ORDER BY CHARACTER_LENGTH(dim1)", ImmutableList.of( GroupByQuery.builder() .setDataSource(CalciteTests.DATASOURCE1) @@ -5378,6 +5378,11 @@ public void testGroupByStringLength() throws Exception .setDimensions(DIMS(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG))) .setAggregatorSpecs(AGGS(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) + .setLimitSpec(new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec( + "d0", + OrderByColumnSpec.Direction.ASCENDING, + StringComparators.NUMERIC + )), null)) .build() ), ImmutableList.of( From 1fdb0a52135c0e37277a00b956a17567a2f1c0f9 Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 18 Dec 2017 00:46:19 +0530 Subject: [PATCH 40/50] More review comments 1) Refactor Aggregator Null Handling 2) Merge multiple null handling utils and renaming with other misc review comments --- .../io/druid/common/config/NullHandling.java | 58 ++-------- .../druid/common/config/NullHandlingUtil.java | 59 ---------- .../main/java/io/druid/math/expr/Expr.java | 10 +- .../java/io/druid/math/expr/ExprEval.java | 4 +- .../java/io/druid/math/expr/Function.java | 16 +-- .../java/io/druid/math/expr/EvalTest.java | 4 +- .../java/io/druid/math/expr/FunctionTest.java | 4 +- .../DistinctCountAggregator.java | 7 +- .../DistinctCountBufferAggregator.java | 9 +- .../DistinctCountGroupByQueryTest.java | 4 +- .../java/io/druid/indexer/InputRowSerde.java | 10 +- .../io/druid/indexer/InputRowSerdeTest.java | 32 +++--- .../common/task/RealtimeIndexTaskTest.java | 6 +- .../druid/java/util/common/StringUtils.java | 6 + .../io/druid/guice/NullHandlingModule.java | 7 +- .../druid/query/aggregation/Aggregator.java | 2 +- .../query/aggregation/BufferAggregator.java | 2 +- .../DoubleMaxAggregatorFactory.java | 15 +-- .../DoubleMinAggregatorFactory.java | 15 +-- .../DoubleSumAggregatorFactory.java | 15 +-- .../FloatMaxAggregatorFactory.java | 15 +-- .../FloatMinAggregatorFactory.java | 15 +-- .../FloatSumAggregatorFactory.java | 15 +-- .../aggregation/LongMaxAggregatorFactory.java | 22 ++-- .../aggregation/LongMinAggregatorFactory.java | 22 ++-- .../aggregation/LongSumAggregatorFactory.java | 21 ++-- .../NullableAggregatorFactory.java | 70 ++++++++++++ .../SimpleDoubleAggregatorFactory.java | 7 +- .../SimpleFloatAggregatorFactory.java | 7 +- ...alityAggregatorColumnSelectorStrategy.java | 6 +- ...alityAggregatorColumnSelectorStrategy.java | 6 +- ...alityAggregatorColumnSelectorStrategy.java | 6 +- ...alityAggregatorColumnSelectorStrategy.java | 10 +- .../first/DoubleFirstAggregatorFactory.java | 38 ++++--- .../first/FloatFirstAggregatorFactory.java | 33 +++--- .../first/LongFirstAggregatorFactory.java | 34 +++--- .../last/DoubleLastAggregatorFactory.java | 34 +++--- .../last/FloatLastAggregatorFactory.java | 34 +++--- .../last/LongLastAggregatorFactory.java | 34 +++--- .../post/ArithmeticPostAggregator.java | 4 +- .../post/DoubleGreatestPostAggregator.java | 4 +- .../post/DoubleLeastPostAggregator.java | 4 +- .../post/LongGreatestPostAggregator.java | 4 +- .../post/LongLeastPostAggregator.java | 4 +- .../dimension/ListFilteredDimensionSpec.java | 6 +- .../dimension/RegexFilteredDimensionSpec.java | 6 +- .../druid/query/expression/LikeExprMacro.java | 4 +- .../expression/RegexpExtractExprMacro.java | 6 +- .../extraction/FunctionalExtraction.java | 8 +- .../extraction/IdentityExtractionFn.java | 6 +- .../extraction/JavaScriptExtractionFn.java | 4 +- .../query/extraction/LowerExtractionFn.java | 4 +- .../extraction/MatchingDimExtractionFn.java | 4 +- .../extraction/RegexDimExtractionFn.java | 6 +- .../SearchQuerySpecDimExtractionFn.java | 4 +- .../query/extraction/StrlenExtractionFn.java | 4 +- .../extraction/SubstringDimExtractionFn.java | 4 +- .../query/extraction/TimeDimExtractionFn.java | 4 +- .../query/extraction/UpperExtractionFn.java | 4 +- .../io/druid/query/filter/InDimFilter.java | 6 +- .../io/druid/query/filter/LikeDimFilter.java | 4 +- .../druid/query/filter/SelectorDimFilter.java | 4 +- .../RowBasedColumnSelectorFactory.java | 12 +- .../epinephelinae/RowBasedGrouperHelper.java | 10 +- ...ngStringGroupByColumnSelectorStrategy.java | 4 +- .../StringGroupByColumnSelectorStrategy.java | 4 +- .../query/lookup/LookupExtractionFn.java | 4 +- .../java/io/druid/query/search/SearchHit.java | 4 +- .../druid/segment/AggregatorNullHandling.java | 41 +++++++ .../BaseNullableColumnValueSelector.java | 7 -- .../BaseObjectColumnValueSelector.java | 2 +- .../ColumnSelectorBitmapIndexSelector.java | 3 +- .../segment/ConstantDimensionSelector.java | 3 +- .../druid/segment/DimensionHandlerUtils.java | 7 +- .../druid/segment/DoubleDimensionIndexer.java | 3 +- .../java/io/druid/segment/IndexMerger.java | 3 +- .../druid/segment/NullDimensionSelector.java | 3 +- .../druid/segment/StringDimensionIndexer.java | 3 +- .../io/druid/segment/data/GenericIndexed.java | 6 +- .../io/druid/segment/filter/BoundFilter.java | 10 +- .../segment/filter/ExpressionFilter.java | 4 +- .../io/druid/segment/filter/LikeFilter.java | 4 +- .../virtual/ExpressionObjectSelector.java | 21 ++-- .../segment/virtual/ExpressionSelectors.java | 6 +- .../NullHandlingHelperInjectionTest.java | 8 +- .../io/druid/query/SchemaEvolutionTest.java | 8 +- .../aggregation/DoubleMaxAggregationTest.java | 4 +- .../aggregation/DoubleMinAggregationTest.java | 4 +- .../aggregation/FilteredAggregatorTest.java | 8 +- .../aggregation/LongMaxAggregationTest.java | 4 +- .../aggregation/LongMinAggregationTest.java | 4 +- .../CardinalityAggregatorTest.java | 16 +-- .../first/DoubleFirstAggregationTest.java | 6 +- .../first/FloatFirstAggregationTest.java | 6 +- .../first/LongFirstAggregationTest.java | 6 +- .../last/DoubleLastAggregationTest.java | 6 +- .../last/FloatLastAggregationTest.java | 6 +- .../last/LongLastAggregationTest.java | 6 +- .../extraction/FunctionalExtractionTest.java | 16 +-- .../JavaScriptExtractionFnTest.java | 4 +- .../extraction/LowerExtractionFnTest.java | 4 +- .../MatchingDimExtractionFnTest.java | 4 +- .../extraction/RegexDimExtractionFnTest.java | 18 +-- .../extraction/StrlenExtractionFnTest.java | 4 +- .../extraction/TimeDimExtractionFnTest.java | 4 +- .../extraction/UpperExtractionFnTest.java | 4 +- .../query/groupby/GroupByQueryRunnerTest.java | 40 +++---- .../epinephelinae/BufferHashGrouperTest.java | 10 +- .../LimitedBufferHashGrouperTest.java | 10 +- .../StreamingMergeSortedGrouperTest.java | 4 +- .../LookupExtractionFnExpectationsTest.java | 4 +- .../query/search/SearchQueryRunnerTest.java | 4 +- .../timeseries/TimeseriesQueryRunnerTest.java | 22 ++-- .../druid/query/topn/TopNQueryRunnerTest.java | 6 +- .../ConstantDimensionSelectorTest.java | 3 +- .../segment/IndexMergerNullHandlingTest.java | 11 +- .../io/druid/segment/IndexMergerTestBase.java | 17 +-- .../druid/segment/SchemalessTestFullTest.java | 29 ++--- .../segment/SchemalessTestSimpleTest.java | 5 +- .../segment/data/GenericIndexedTest.java | 6 +- .../druid/segment/filter/BoundFilterTest.java | 16 +-- .../filter/ColumnComparisonFilterTest.java | 4 +- .../segment/filter/ExpressionFilterTest.java | 8 +- .../segment/filter/FilterPartitionTest.java | 20 ++-- .../io/druid/segment/filter/InFilterTest.java | 10 +- .../segment/filter/JavaScriptFilterTest.java | 6 +- .../druid/segment/filter/LikeFilterTest.java | 4 +- .../druid/segment/filter/RegexFilterTest.java | 8 +- .../segment/filter/SearchQueryFilterTest.java | 8 +- .../segment/filter/SelectorFilterTest.java | 16 +-- .../IncrementalIndexStorageAdapterTest.java | 4 +- .../incremental/IncrementalIndexTest.java | 4 +- .../virtual/ExpressionVirtualColumnTest.java | 16 +-- .../query/dimension/LookupDimensionSpec.java | 4 +- .../druid/query/expression/ExprMacroTest.java | 4 +- .../sql/calcite/expression/Expressions.java | 4 +- .../io/druid/sql/calcite/rel/QueryMaker.java | 4 +- .../druid/sql/avatica/DruidStatementTest.java | 8 +- .../druid/sql/calcite/CalciteQueryTest.java | 106 +++++++++--------- .../sql/calcite/http/SqlResourceTest.java | 4 +- 140 files changed, 801 insertions(+), 764 deletions(-) rename processing/src/main/java/io/druid/segment/NullHandlingHelper.java => common/src/main/java/io/druid/common/config/NullHandling.java (61%) delete mode 100644 common/src/main/java/io/druid/common/config/NullHandlingUtil.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java create mode 100644 processing/src/main/java/io/druid/segment/AggregatorNullHandling.java diff --git a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java b/common/src/main/java/io/druid/common/config/NullHandling.java similarity index 61% rename from processing/src/main/java/io/druid/segment/NullHandlingHelper.java rename to common/src/main/java/io/druid/common/config/NullHandling.java index bd561dd661a7..643eeafb7b58 100644 --- a/processing/src/main/java/io/druid/segment/NullHandlingHelper.java +++ b/common/src/main/java/io/druid/common/config/NullHandling.java @@ -17,22 +17,14 @@ * under the License. */ -package io.druid.segment; +package io.druid.common.config; import com.google.common.base.Strings; -import com.google.common.base.Supplier; import com.google.inject.Inject; -import io.druid.common.config.NullValueHandlingConfig; -import io.druid.query.aggregation.AggregateCombiner; -import io.druid.query.aggregation.Aggregator; -import io.druid.query.aggregation.BufferAggregator; -import io.druid.query.aggregation.NullableAggregateCombiner; -import io.druid.query.aggregation.NullableAggregator; -import io.druid.query.aggregation.NullableBufferAggregator; import javax.annotation.Nullable; -public class NullHandlingHelper +public class NullHandling { private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; @@ -61,13 +53,18 @@ public static boolean useDefaultValuesForNull() @Nullable public static String nullToEmptyIfNeeded(@Nullable String value) { - return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; + return useDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; } @Nullable public static String emptyToNullIfNeeded(@Nullable String value) { - return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; + return useDefaultValuesForNull() ? Strings.emptyToNull(value) : value; + } + + public static String defaultValue() + { + return useDefaultValuesForNull() ? "" : null; } public static boolean isNullOrEquivalent(@Nullable String value) @@ -92,41 +89,4 @@ public static Float nullToZeroIfNeeded(@Nullable Float value) { return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_FLOAT : value; } - - public static Aggregator getNullableAggregator(Aggregator aggregator, BaseNullableColumnValueSelector selector) - { - return INSTANCE.isUseDefaultValuesForNull() ? aggregator : new NullableAggregator(aggregator, selector); - } - - public static BufferAggregator getNullableAggregator( - BufferAggregator aggregator, - BaseNullableColumnValueSelector selector - ) - { - return INSTANCE.isUseDefaultValuesForNull() ? aggregator : new NullableBufferAggregator(aggregator, selector); - } - - public static AggregateCombiner getNullableCombiner(AggregateCombiner combiner) - { - return INSTANCE.isUseDefaultValuesForNull() ? combiner : new NullableAggregateCombiner(combiner); - } - - public static int extraAggregatorBytes() - { - return NullHandlingHelper.useDefaultValuesForNull() ? 0 : Byte.BYTES; - } - - public static Supplier getNullableSupplier( - final Supplier supplier, - final BaseNullableColumnValueSelector selector - ) - { - return INSTANCE.isUseDefaultValuesForNull() ? - supplier : () -> { - if (selector.isNull()) { - return null; - } - return supplier.get(); - }; - } } diff --git a/common/src/main/java/io/druid/common/config/NullHandlingUtil.java b/common/src/main/java/io/druid/common/config/NullHandlingUtil.java deleted file mode 100644 index df7eacbc7419..000000000000 --- a/common/src/main/java/io/druid/common/config/NullHandlingUtil.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.common.config; - -import com.google.common.base.Strings; -import com.google.inject.Inject; - -import javax.annotation.Nullable; - -public class NullHandlingUtil -{ - private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; - - - /** - * INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. - * See {@link io.druid.guice.NullHandlingModule} for details. - * It does not take effect in all unit tests since we don't use Guice Injection. - * For tests default system property is supposed to be used only in tests - */ - @Inject - private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( - Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) - ); - - public static boolean useDefaultValuesForNull() - { - return INSTANCE.isUseDefaultValuesForNull(); - } - - @Nullable - public static String nullToEmptyIfNeeded(@Nullable String value) - { - return INSTANCE.isUseDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; - } - - @Nullable - public static String emptyToNullIfNeeded(@Nullable String value) - { - return INSTANCE.isUseDefaultValuesForNull() ? Strings.emptyToNull(value) : value; - } -} diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index b11c619fc423..a6ed91e4e0f3 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -22,7 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.math.LongMath; import com.google.common.primitives.Ints; -import io.druid.common.config.NullHandlingUtil; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Comparators; @@ -124,7 +124,7 @@ class StringExpr extends ConstantExpr public StringExpr(String value) { - this.value = NullHandlingUtil.emptyToNullIfNeeded(value); + this.value = NullHandling.emptyToNullIfNeeded(value); } @Nullable @@ -365,7 +365,7 @@ public ExprEval eval(ObjectBinding bindings) // Result of any Binary expressions is null if any of the argument is null. // e.g "select null * 2 as c;" or "select null + 1 as c;" will return null as per Standard SQL spec. - if (!NullHandlingUtil.useDefaultValuesForNull() && (leftVal.isNull() || rightVal.isNull())) { + if (!NullHandling.useDefaultValuesForNull() && (leftVal.isNull() || rightVal.isNull())) { return ExprEval.of(null); } @@ -498,8 +498,8 @@ class BinPlusExpr extends BinaryEvalOpExprBase @Override protected ExprEval evalString(@Nullable String left, @Nullable String right) { - return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded(left) - + NullHandlingUtil.nullToEmptyIfNeeded(right)); + return ExprEval.of(NullHandling.nullToEmptyIfNeeded(left) + + NullHandling.nullToEmptyIfNeeded(right)); } @Override diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 650fccb5d1cd..188a2c71bc24 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -22,7 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; import com.google.common.primitives.Ints; -import io.druid.common.config.NullHandlingUtil; +import io.druid.common.config.NullHandling; import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.IAE; @@ -233,7 +233,7 @@ private static class StringExprEval extends ExprEval { private StringExprEval(String value) { - super(NullHandlingUtil.emptyToNullIfNeeded(value)); + super(NullHandling.emptyToNullIfNeeded(value)); } @Override diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index b326d4bbc3c0..80f45e98655d 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -20,7 +20,7 @@ package io.druid.math.expr; import com.google.common.base.Strings; -import io.druid.common.config.NullHandlingUtil; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.StringUtils; @@ -75,7 +75,7 @@ abstract class SingleParamMath extends SingleParam @Override protected final ExprEval eval(ExprEval param) { - if (!NullHandlingUtil.useDefaultValuesForNull() && param.isNull()) { + if (!NullHandling.useDefaultValuesForNull() && param.isNull()) { return ExprEval.of(null); } if (param.type() == ExprType.LONG) { @@ -903,12 +903,12 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final StringBuilder builder = new StringBuilder(Strings.nullToEmpty(args.get(0).eval(bindings).asString())); for (int i = 1; i < args.size(); i++) { final String s = args.get(i).eval(bindings).asString(); - if (!NullHandlingUtil.useDefaultValuesForNull() && s == null) { + if (!NullHandling.useDefaultValuesForNull() && s == null) { // Result of concatenation is null if any of the Values is null. // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. return ExprEval.of(null); } else { - builder.append(NullHandlingUtil.nullToEmptyIfNeeded(s)); + builder.append(NullHandling.nullToEmptyIfNeeded(s)); } } return ExprEval.of(builder.toString()); @@ -992,7 +992,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } else { // If starting index of substring is greater then the length of string, the result will be a zero length string. // e.g. 'select substring("abc", 4,5) as c;' will return an empty string - return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded((String) null)); + return ExprEval.of(NullHandling.defaultValue()); } } } @@ -1016,7 +1016,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String pattern = args.get(1).eval(bindings).asString(); final String replacement = args.get(2).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded(null)); + return ExprEval.of(NullHandling.defaultValue()); } return ExprEval.of( arg.replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) @@ -1041,7 +1041,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded((String) null)); + return ExprEval.of(NullHandling.defaultValue()); } return ExprEval.of(StringUtils.toLowerCase(arg)); } @@ -1064,7 +1064,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandlingUtil.nullToEmptyIfNeeded((String) null)); + return ExprEval.of(NullHandling.defaultValue()); } return ExprEval.of(StringUtils.toUpperCase(arg)); } diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java index c3458bb8ac00..c63a016b8203 100644 --- a/common/src/test/java/io/druid/math/expr/EvalTest.java +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -20,7 +20,7 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; -import io.druid.common.config.NullHandlingUtil; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -140,7 +140,7 @@ public void testLongEval() Assert.assertEquals(1271055781L, evalLong("unix_timestamp('2010-04-12T07:03:01')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01+09:00')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01.419+09:00')", bindings)); - if (NullHandlingUtil.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals("NULL", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); } else { Assert.assertEquals("", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); diff --git a/common/src/test/java/io/druid/math/expr/FunctionTest.java b/common/src/test/java/io/druid/math/expr/FunctionTest.java index 27a98ef1982c..99c153da550c 100644 --- a/common/src/test/java/io/druid/math/expr/FunctionTest.java +++ b/common/src/test/java/io/druid/math/expr/FunctionTest.java @@ -20,7 +20,7 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; -import io.druid.common.config.NullHandlingUtil; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -56,7 +56,7 @@ public void testCaseSearched() public void testConcat() { assertExpr("concat(x,' ',y)", "foo 2"); - if (NullHandlingUtil.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertExpr("concat(x,' ',nonexistent,' ',y)", "foo 2"); } else { assertExpr("concat(x,' ',nonexistent,' ',y)", null); diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java index d802b96307eb..657feac74099 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java @@ -22,12 +22,11 @@ import io.druid.collections.bitmap.MutableBitmap; import io.druid.query.aggregation.Aggregator; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.segment.data.IndexedInts; public class DistinctCountAggregator implements Aggregator { - private static final int UNKNOWN_ID = -1; private final DimensionSelector selector; private final MutableBitmap mutableBitmap; private final int idForNull; @@ -39,7 +38,7 @@ public DistinctCountAggregator( { this.selector = selector; this.mutableBitmap = mutableBitmap; - this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : UNKNOWN_ID; + this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : -1; } @Override @@ -48,7 +47,7 @@ public void aggregate() IndexedInts row = selector.getRow(); for (int i = 0; i < row.size(); i++) { int index = row.get(i); - if (NullHandlingHelper.useDefaultValuesForNull() || isNotNull(index)) { + if (NullHandling.useDefaultValuesForNull() || isNotNull(index)) { mutableBitmap.add(index); } } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index e9fd222a4e6d..5f3d6acbb9f8 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -21,10 +21,11 @@ import io.druid.collections.bitmap.MutableBitmap; import io.druid.collections.bitmap.WrappedRoaringBitmap; +import io.druid.common.config.NullHandling; import io.druid.query.aggregation.BufferAggregator; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.segment.data.IndexedInts; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; @@ -33,7 +34,6 @@ public class DistinctCountBufferAggregator implements BufferAggregator { - private static final int UNKNOWN_ID = -1; private final DimensionSelector selector; private final Int2ObjectMap mutableBitmapCollection = new Int2ObjectOpenHashMap<>(); private final int idForNull; @@ -44,7 +44,7 @@ public DistinctCountBufferAggregator( ) { this.selector = selector; - this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : UNKNOWN_ID; + this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : -1; } @Override @@ -56,12 +56,11 @@ public void init(ByteBuffer buf, int position) @Override public void aggregate(ByteBuffer buf, int position) { - boolean countNulls = !selector.nameLookupPossibleInAdvance() || NullHandlingHelper.useDefaultValuesForNull(); MutableBitmap mutableBitmap = getMutableBitmap(position); IndexedInts row = selector.getRow(); for (int i = 0; i < row.size(); i++) { int index = row.get(i); - if (NullHandlingHelper.useDefaultValuesForNull() || isNotNull(index)) { + if (NullHandling.useDefaultValuesForNull() || isNotNull(index)) { mutableBitmap.add(index); } } diff --git a/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java b/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java index 5dbf8b929f0b..4c49c821520f 100644 --- a/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java +++ b/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java @@ -37,7 +37,7 @@ import io.druid.query.groupby.orderby.OrderByColumnSpec; import io.druid.query.groupby.strategy.GroupByStrategySelector; import io.druid.segment.IncrementalIndexSegment; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.segment.Segment; import io.druid.segment.TestHelper; import io.druid.segment.incremental.IncrementalIndex; @@ -186,7 +186,7 @@ public void testGroupByWithDistinctCountAgg() throws Exception ); List expectedResults; - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( "1970-01-01T00:00:00.000Z", diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index 86f002f9b0b9..e3f85992b483 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -49,8 +49,8 @@ public class InputRowSerde { private static final Logger log = new Logger(InputRowSerde.class); - private static final byte TRUE_BYTE = (byte) 1; - private static final byte FALSE_BYTE = (byte) 0; + private static final byte NULL_BYTE = (byte) 1; + private static final byte NON_NULL_BYTE = (byte) 0; public static final byte[] toBytes(final InputRow row, AggregatorFactory[] aggs, boolean reportParseExceptions) { @@ -107,9 +107,9 @@ public InputRow get() String t = aggFactory.getTypeName(); if (agg.isNull()) { - out.writeByte(TRUE_BYTE); + out.writeByte(NULL_BYTE); } else { - out.writeByte(FALSE_BYTE); + out.writeByte(NON_NULL_BYTE); if (t.equals("float")) { out.writeFloat(agg.getFloat()); } else if (t.equals("long")) { @@ -224,7 +224,7 @@ public static final InputRow fromBytes(byte[] data, AggregatorFactory[] aggs) String metric = readString(in); String type = getType(metric, aggs, i); byte metricNullability = in.readByte(); - if (metricNullability == TRUE_BYTE) { + if (metricNullability == NULL_BYTE) { // metric value is null. continue; } diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java index 0f150e9bd0a5..daad96fca838 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java @@ -33,7 +33,7 @@ import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; @@ -88,6 +88,18 @@ public void testSerde() EasyMock.expectLastCall().times(1); EasyMock.replay(mockedNullAggregator); + final AggregatorFactory mockedAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); + EasyMock.expect(mockedAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedAggregator); + EasyMock.expect(mockedAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); + EasyMock.expect(mockedAggregatorFactory.getName()).andReturn("mockedAggregator").anyTimes(); + + final AggregatorFactory mockedNullAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); + EasyMock.expect(mockedNullAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedNullAggregator); + EasyMock.expect(mockedNullAggregatorFactory.getName()).andReturn("mockedNullAggregator").anyTimes(); + EasyMock.expect(mockedNullAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); + + EasyMock.replay(mockedAggregatorFactory, mockedNullAggregatorFactory); + InputRow in = new MapBasedInputRow( timestamp, dims, @@ -100,20 +112,8 @@ public void testSerde() new LongSumAggregatorFactory("m2out", "m2"), new HyperUniquesAggregatorFactory("m3out", "m3"), new LongSumAggregatorFactory("unparseable", "m3"), // Unparseable from String to Long - new DoubleSumAggregatorFactory("mockedAggregator", "m4") { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - return mockedAggregator; - } - }, - new DoubleSumAggregatorFactory("mockedNullAggregator", "m5") { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - return mockedNullAggregator; - } - } + mockedAggregatorFactory, + mockedNullAggregatorFactory }; byte[] data = InputRowSerde.toBytes(in, aggregatorFactories, false); // Ignore Unparseable aggregator @@ -125,7 +125,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) Assert.assertEquals(ImmutableList.of("d1v"), out.getDimension("d1")); Assert.assertEquals(ImmutableList.of("d2v1", "d2v2"), out.getDimension("d2")); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0.0f, out.getMetric("agg_non_existing").floatValue(), 0.00001); } else { Assert.assertNull(out.getMetric("agg_non_existing")); diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index f7fb51847443..a5b17f0bfc48 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -101,7 +101,7 @@ import io.druid.query.timeseries.TimeseriesQueryQueryToolChest; import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; import io.druid.query.timeseries.TimeseriesResultValue; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.segment.TestHelper; import io.druid.segment.indexing.DataSchema; import io.druid.segment.indexing.RealtimeIOConfig; @@ -427,7 +427,7 @@ public void testTransformSpec() throws Exception // Do some queries. Assert.assertEquals(1, sumMetric(task, null, "rows").longValue()); Assert.assertEquals(1, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows").longValue()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows").longValue()); } else { Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); @@ -1121,7 +1121,7 @@ public Long sumMetric(final Task task, final DimFilter filter, final String metr Lists.>newArrayList() ); if (results.isEmpty()) { - return NullHandlingHelper.nullToZeroIfNeeded((Long) null); + return NullHandling.nullToZeroIfNeeded((Long) null); } return results.get(0).getValue().getLongMetric(metric); } diff --git a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java index e55d5260a142..0d7d84b1c544 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java @@ -94,6 +94,12 @@ public static String fromUtf8(final ByteBuffer buffer, final int numBytes) return fromUtf8(bytes); } + /** + * Reads numBytes bytes from buffer and converts that to a utf-8 string + * @param buffer buffer to read bytes from + * @param numBytes number of bytes to read + * @return returns null if numBytes is -1 otherwise utf-8 string represetation of bytes read + */ @Nullable public static String fromUtf8Nullable(final ByteBuffer buffer, final int numBytes) { diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java index b4f448031e07..7216ca363822 100644 --- a/processing/src/main/java/io/druid/guice/NullHandlingModule.java +++ b/processing/src/main/java/io/druid/guice/NullHandlingModule.java @@ -21,8 +21,7 @@ import com.google.inject.Binder; import com.google.inject.Module; -import io.druid.common.config.NullHandlingUtil; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.common.config.NullValueHandlingConfig; /** @@ -33,8 +32,8 @@ public class NullHandlingModule implements Module public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.generic", NullValueHandlingConfig.class); - binder.requestStaticInjection(NullHandlingHelper.class); - binder.requestStaticInjection(NullHandlingUtil.class); + binder.requestStaticInjection(NullHandling.class); + binder.requestStaticInjection(NullHandling.class); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java index feb61eb529e7..85b2b6c9f9b8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java @@ -63,7 +63,7 @@ default double getDouble() /** * returns true if the Aggregator supports returning null values and the aggregated value is Null. * The default implementation always return false to enable smooth backward compatibility, re-implement if your aggregator is nullable. - * For backwards compatibility, isNull() may return false even if get() returns null. Users of this method should account for this case. + * For backwards compatibility, isNull() may return false even if {@link Aggregator#get()} returns null. Users of this method should account for this case. */ default boolean isNull() { diff --git a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java index 2c253ab1d810..c1dba19228c4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java @@ -195,7 +195,7 @@ default void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, By * @param position offset within the byte buffer at which the aggregate value is stored * * @return true if the aggrgeated value is null otherwise false. - * For backwards compatibility, isNull() may return false even if get() returns null. Users of this method should account for this case. + * For backwards compatibility, isNull() may return false even if {@link BufferAggregator#get(ByteBuffer, int)} returns null. Users of this method should account for this case. */ default boolean isNull(ByteBuffer buf, int position) { diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index 1f895e9fbe1d..4d02de7cbec6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -22,11 +22,12 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -54,26 +55,26 @@ public DoubleMaxAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( metricFactory, Double.NEGATIVE_INFINITY ); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new DoubleMaxAggregator(doubleColumnSelector), doubleColumnSelector ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( metricFactory, Double.NEGATIVE_INFINITY ); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new DoubleMaxBufferAggregator(doubleColumnSelector), doubleColumnSelector ); @@ -93,9 +94,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { - return NullHandlingHelper.getNullableCombiner(new DoubleMaxAggregateCombiner()); + return new DoubleMaxAggregateCombiner(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index 115785cdfc5a..7869bb1f2ce3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -22,11 +22,12 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -54,26 +55,26 @@ public DoubleMinAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( metricFactory, Double.POSITIVE_INFINITY ); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new DoubleMinAggregator(doubleColumnSelector), doubleColumnSelector ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( metricFactory, Double.POSITIVE_INFINITY ); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new DoubleMinBufferAggregator(doubleColumnSelector), doubleColumnSelector ); @@ -93,9 +94,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { - return NullHandlingHelper.getNullableCombiner(new DoubleMinAggregateCombiner()); + return new DoubleMinAggregateCombiner(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index 1c2ce62e58fb..313a2676fdba 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -22,11 +22,12 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -55,20 +56,20 @@ public DoubleSumAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new DoubleSumAggregator(doubleColumnSelector), doubleColumnSelector ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new DoubleSumBufferAggregator(doubleColumnSelector), doubleColumnSelector ); @@ -88,9 +89,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { - return NullHandlingHelper.getNullableCombiner(new DoubleSumAggregateCombiner()); + return new DoubleSumAggregateCombiner(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java index 8f2098fc1c38..589c1def93e7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -22,11 +22,12 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -54,23 +55,23 @@ public FloatMaxAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( metricFactory, Float.NEGATIVE_INFINITY ); - return NullHandlingHelper.getNullableAggregator(new FloatMaxAggregator(floatColumnSelector), floatColumnSelector); + return Pair.of(new FloatMaxAggregator(floatColumnSelector), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( metricFactory, Float.NEGATIVE_INFINITY ); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new FloatMaxBufferAggregator(floatColumnSelector), floatColumnSelector ); @@ -90,9 +91,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { - return NullHandlingHelper.getNullableCombiner(new DoubleMaxAggregateCombiner()); + return new DoubleMaxAggregateCombiner(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java index 34fc2f3dba97..f97d7c5ba29f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java @@ -22,11 +22,12 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -54,23 +55,23 @@ public FloatMinAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( metricFactory, Float.POSITIVE_INFINITY ); - return NullHandlingHelper.getNullableAggregator(new FloatMinAggregator(floatColumnSelector), floatColumnSelector); + return Pair.of(new FloatMinAggregator(floatColumnSelector), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( metricFactory, Float.POSITIVE_INFINITY ); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new FloatMinBufferAggregator(floatColumnSelector), floatColumnSelector ); @@ -90,9 +91,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { - return NullHandlingHelper.getNullableCombiner(new DoubleMinAggregateCombiner()); + return new DoubleMinAggregateCombiner(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java index 79e118fa6914..b6c7725cf970 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java @@ -22,11 +22,12 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -54,17 +55,17 @@ public FloatSumAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f); - return NullHandlingHelper.getNullableAggregator(new FloatSumAggregator(floatColumnSelector), floatColumnSelector); + return Pair.of(new FloatSumAggregator(floatColumnSelector), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new FloatSumBufferAggregator(floatColumnSelector), floatColumnSelector ); @@ -84,9 +85,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { - return NullHandlingHelper.getNullableCombiner(new DoubleSumAggregateCombiner()); + return new DoubleSumAggregateCombiner(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index 25774971bbbc..07699519f29e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -24,13 +24,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -42,7 +43,7 @@ /** */ -public class LongMaxAggregatorFactory extends AggregatorFactory +public class LongMaxAggregatorFactory extends NullableAggregatorFactory { private final String name; private final String fieldName; @@ -75,17 +76,17 @@ public LongMaxAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return NullHandlingHelper.getNullableAggregator(new LongMaxAggregator(longColumnSelector), longColumnSelector); + return Pair.of(new LongMaxAggregator(longColumnSelector), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new LongMaxBufferAggregator(longColumnSelector), longColumnSelector ); @@ -122,9 +123,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { - LongAggregateCombiner combiner = new LongAggregateCombiner() + return new LongAggregateCombiner() { private long max; @@ -146,7 +147,6 @@ public long getLong() return max; } }; - return NullHandlingHelper.getNullableCombiner(combiner); } @Override @@ -231,9 +231,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Longs.BYTES + NullHandlingHelper.extraAggregatorBytes(); + return Longs.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 1cce860afbb5..6fc331210b22 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -25,13 +25,14 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -43,7 +44,7 @@ /** */ -public class LongMinAggregatorFactory extends AggregatorFactory +public class LongMinAggregatorFactory extends NullableAggregatorFactory { private final String name; @@ -78,17 +79,17 @@ public LongMinAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return NullHandlingHelper.getNullableAggregator(new LongMinAggregator(longColumnSelector), longColumnSelector); + return Pair.of(new LongMinAggregator(longColumnSelector), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new LongMinBufferAggregator(longColumnSelector), longColumnSelector ); @@ -125,9 +126,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { - LongAggregateCombiner combiner = new LongAggregateCombiner() + return new LongAggregateCombiner() { private long min; @@ -149,7 +150,6 @@ public long getLong() return min; } }; - return NullHandlingHelper.getNullableCombiner(combiner); } @Override @@ -234,9 +234,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Longs.BYTES + NullHandlingHelper.extraAggregatorBytes(); + return Longs.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 42e0a5180ca7..219911bc06bf 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -24,12 +24,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import java.nio.ByteBuffer; import java.util.Arrays; @@ -40,7 +41,7 @@ /** */ -public class LongSumAggregatorFactory extends AggregatorFactory +public class LongSumAggregatorFactory extends NullableAggregatorFactory { private final String name; private final String fieldName; @@ -73,17 +74,17 @@ public LongSumAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return NullHandlingHelper.getNullableAggregator(new LongSumAggregator(longColumnSelector), longColumnSelector); + return Pair.of(new LongSumAggregator(longColumnSelector), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return NullHandlingHelper.getNullableAggregator( + return Pair.of( new LongSumBufferAggregator(longColumnSelector), longColumnSelector ); @@ -119,9 +120,9 @@ public Object combine(Object lhs, Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { - return NullHandlingHelper.getNullableCombiner(new LongSumAggregateCombiner()); + return new LongSumAggregateCombiner(); } @Override @@ -206,9 +207,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Longs.BYTES + NullHandlingHelper.extraAggregatorBytes(); + return Longs.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java new file mode 100644 index 000000000000..698d3afa0157 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + + +import io.druid.common.config.NullHandling; +import io.druid.java.util.common.Pair; +import io.druid.segment.BaseNullableColumnValueSelector; +import io.druid.segment.ColumnSelectorFactory; + +/** + */ +public abstract class NullableAggregatorFactory extends AggregatorFactory +{ + + @Override + final public Aggregator factorize(ColumnSelectorFactory metricFactory) + { + Pair pair = factorize2( + metricFactory); + return NullHandling.useDefaultValuesForNull() ? pair.lhs : new NullableAggregator(pair.lhs, pair.rhs); + } + + protected abstract Pair factorize2(ColumnSelectorFactory metricfactory); + + @Override + final public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + { + Pair pair = factorizeBuffered2( + metricFactory); + return NullHandling.useDefaultValuesForNull() ? pair.lhs : new NullableBufferAggregator(pair.lhs, pair.rhs); + } + + protected abstract Pair factorizeBuffered2(ColumnSelectorFactory metricfactory); + + + @Override + final public AggregateCombiner makeAggregateCombiner() + { + AggregateCombiner combiner = makeAggregateCombiner2(); + return NullHandling.useDefaultValuesForNull() ? combiner : new NullableAggregateCombiner(combiner); + } + + protected abstract AggregateCombiner makeAggregateCombiner2(); + + @Override + final public int getMaxIntermediateSize() + { + return getMaxIntermediateSize2() + (NullHandling.useDefaultValuesForNull() ? 0 : Byte.BYTES); + } + + protected abstract int getMaxIntermediateSize2(); +} diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index ce304abb847e..fff7a299042e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -26,7 +26,6 @@ import io.druid.math.expr.Parser; import io.druid.segment.BaseDoubleColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import java.util.Collections; @@ -34,7 +33,7 @@ import java.util.List; import java.util.Objects; -public abstract class SimpleDoubleAggregatorFactory extends AggregatorFactory +public abstract class SimpleDoubleAggregatorFactory extends NullableAggregatorFactory { protected final String name; protected final String fieldName; @@ -92,9 +91,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Double.BYTES + NullHandlingHelper.extraAggregatorBytes(); + return Double.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java index f9a5dd29de7e..b920301828dc 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -25,14 +25,13 @@ import io.druid.math.expr.Parser; import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Objects; -public abstract class SimpleFloatAggregatorFactory extends AggregatorFactory +public abstract class SimpleFloatAggregatorFactory extends NullableAggregatorFactory { protected final String name; protected final String fieldName; @@ -88,9 +87,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Float.BYTES + NullHandlingHelper.extraAggregatorBytes(); + return Float.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java index 11bec1edf3e7..457e471739f4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java @@ -20,10 +20,10 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseDoubleColumnValueSelector; -import io.druid.segment.NullHandlingHelper; public class DoubleCardinalityAggregatorColumnSelectorStrategy @@ -32,7 +32,7 @@ public class DoubleCardinalityAggregatorColumnSelectorStrategy @Override public void hashRow(BaseDoubleColumnValueSelector dimSelector, Hasher hasher) { - if (NullHandlingHelper.useDefaultValuesForNull() || !dimSelector.isNull()) { + if (NullHandling.useDefaultValuesForNull() || !dimSelector.isNull()) { hasher.putDouble(dimSelector.getDouble()); } } @@ -40,7 +40,7 @@ public void hashRow(BaseDoubleColumnValueSelector dimSelector, Hasher hasher) @Override public void hashValues(BaseDoubleColumnValueSelector dimSelector, HyperLogLogCollector collector) { - if (NullHandlingHelper.useDefaultValuesForNull() || !dimSelector.isNull()) { + if (NullHandling.useDefaultValuesForNull() || !dimSelector.isNull()) { collector.add(CardinalityAggregator.hashFn.hashLong(Double.doubleToLongBits(dimSelector.getDouble())).asBytes()); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java index c22b1477eacb..1761cab9b6d9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java @@ -20,10 +20,10 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseFloatColumnValueSelector; -import io.druid.segment.NullHandlingHelper; public class FloatCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy @@ -31,7 +31,7 @@ public class FloatCardinalityAggregatorColumnSelectorStrategy @Override public void hashRow(BaseFloatColumnValueSelector selector, Hasher hasher) { - if (NullHandlingHelper.useDefaultValuesForNull() || !selector.isNull()) { + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { hasher.putFloat(selector.getFloat()); } } @@ -39,7 +39,7 @@ public void hashRow(BaseFloatColumnValueSelector selector, Hasher hasher) @Override public void hashValues(BaseFloatColumnValueSelector selector, HyperLogLogCollector collector) { - if (NullHandlingHelper.useDefaultValuesForNull() || !selector.isNull()) { + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { collector.add(CardinalityAggregator.hashFn.hashInt(Float.floatToIntBits(selector.getFloat())).asBytes()); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java index bdfda2f6a8f5..dd836d16be68 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java @@ -20,10 +20,10 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseLongColumnValueSelector; -import io.druid.segment.NullHandlingHelper; public class LongCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy @@ -31,7 +31,7 @@ public class LongCardinalityAggregatorColumnSelectorStrategy @Override public void hashRow(BaseLongColumnValueSelector dimSelector, Hasher hasher) { - if (NullHandlingHelper.useDefaultValuesForNull() || !dimSelector.isNull()) { + if (NullHandling.useDefaultValuesForNull() || !dimSelector.isNull()) { hasher.putLong(dimSelector.getLong()); } } @@ -39,7 +39,7 @@ public void hashRow(BaseLongColumnValueSelector dimSelector, Hasher hasher) @Override public void hashValues(BaseLongColumnValueSelector dimSelector, HyperLogLogCollector collector) { - if (NullHandlingHelper.useDefaultValuesForNull() || !dimSelector.isNull()) { + if (NullHandling.useDefaultValuesForNull() || !dimSelector.isNull()) { collector.add(CardinalityAggregator.hashFn.hashLong(dimSelector.getLong()).asBytes()); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java index 0fb0b80bc6f2..f604a59401d4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java @@ -20,10 +20,10 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.IndexedInts; import java.util.Arrays; @@ -42,7 +42,7 @@ public void hashRow(DimensionSelector dimSelector, Hasher hasher) // nothing to add to hasher if size == 0, only handle size == 1 and size != 0 cases. if (size == 1) { final String value = dimSelector.lookupName(row.get(0)); - if (NullHandlingHelper.useDefaultValuesForNull() || value != null) { + if (NullHandling.useDefaultValuesForNull() || value != null) { hasher.putUnencodedChars(nullToSpecial(value)); } } else if (size != 0) { @@ -50,12 +50,12 @@ public void hashRow(DimensionSelector dimSelector, Hasher hasher) final String[] values = new String[size]; for (int i = 0; i < size; ++i) { final String value = dimSelector.lookupName(row.get(i)); - if (!NullHandlingHelper.useDefaultValuesForNull() && !hasNonNullValue && value != null) { + if (!NullHandling.useDefaultValuesForNull() && !hasNonNullValue && value != null) { hasNonNullValue = true; } values[i] = nullToSpecial(value); } - if (NullHandlingHelper.useDefaultValuesForNull() || hasNonNullValue) { + if (NullHandling.useDefaultValuesForNull() || hasNonNullValue) { // Values need to be sorted to ensure consistent multi-value ordering across different segments Arrays.sort(values); for (int i = 0; i < size; ++i) { @@ -75,7 +75,7 @@ public void hashValues(DimensionSelector dimSelector, HyperLogLogCollector colle for (int i = 0; i < row.size(); i++) { int index = row.get(i); final String value = dimSelector.lookupName(index); - if (NullHandlingHelper.useDefaultValuesForNull() || value != null) { + if (NullHandling.useDefaultValuesForNull() || value != null) { collector.add(CardinalityAggregator.hashFn.hashUnencodedChars(nullToSpecial(value)).asBytes()); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index b8bd33525cf9..f79f3059df9c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -26,17 +26,19 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import javax.annotation.Nullable; @@ -47,7 +49,7 @@ import java.util.Map; import java.util.Objects; -public class DoubleFirstAggregatorFactory extends AggregatorFactory +public class DoubleFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Doubles.compare( ((SerializablePair) o1).rhs, @@ -78,20 +80,20 @@ public DoubleFirstAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new DoubleFirstAggregator( + BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of(new DoubleFirstAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), columnValueSelector ), columnValueSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new DoubleFirstBufferAggregator( + BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of(new DoubleFirstBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), columnValueSelector ), columnValueSelector); @@ -117,7 +119,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("DoubleFirstAggregatorFactory is not supported during ingestion for rollup"); } @@ -128,10 +130,10 @@ public AggregatorFactory getCombiningFactory() return new DoubleFirstAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new DoubleFirstAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new DoubleFirstAggregator(null, null) { @Override public void aggregate() @@ -146,10 +148,10 @@ public void aggregate() } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new DoubleFirstBufferAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new DoubleFirstBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -232,9 +234,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Long.BYTES + Double.BYTES + NullHandlingHelper.extraAggregatorBytes(); + return Long.BYTES + Double.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index dce31870bb36..53ac58d3e8fd 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -26,17 +26,18 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import javax.annotation.Nullable; @@ -47,7 +48,7 @@ import java.util.Map; import java.util.Objects; -public class FloatFirstAggregatorFactory extends AggregatorFactory +public class FloatFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Doubles.compare( ((SerializablePair) o1).rhs, @@ -76,20 +77,20 @@ public FloatFirstAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new FloatFirstAggregator( + return Pair.of(new FloatFirstAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), columnValueSelector ), columnValueSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new FloatFirstBufferAggregator( + return Pair.of(new FloatFirstBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), columnValueSelector ), columnValueSelector); @@ -115,7 +116,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("FloatFirstAggregatorFactory is not supported during ingestion for rollup"); } @@ -126,10 +127,10 @@ public AggregatorFactory getCombiningFactory() return new FloatFirstAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new FloatFirstAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new FloatFirstAggregator(null, null) { @Override public void aggregate() @@ -144,10 +145,10 @@ public void aggregate() } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new FloatFirstBufferAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new FloatFirstBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -226,9 +227,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Long.BYTES + Float.BYTES + NullHandlingHelper.extraAggregatorBytes(); + return Long.BYTES + Float.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 30eb7929087b..984951ef29c4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -25,17 +25,19 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseLongColumnValueSelector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; import javax.annotation.Nullable; @@ -45,7 +47,7 @@ import java.util.List; import java.util.Map; -public class LongFirstAggregatorFactory extends AggregatorFactory +public class LongFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Longs.compare( ((SerializablePair) o1).rhs, @@ -69,20 +71,20 @@ public LongFirstAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new LongFirstAggregator( + return Pair.of(new LongFirstAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), longColumnSelector ), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new LongFirstBufferAggregator( + return Pair.of(new LongFirstBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), longColumnSelector ), longColumnSelector); @@ -108,7 +110,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("LongFirstAggregatorFactory is not supported during ingestion for rollup"); } @@ -119,10 +121,10 @@ public AggregatorFactory getCombiningFactory() return new LongFirstAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new LongFirstAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new LongFirstAggregator(null, null) { @Override public void aggregate() @@ -137,10 +139,10 @@ public void aggregate() } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new LongFirstBufferAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new LongFirstBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -220,9 +222,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Long.BYTES * 2 + NullHandlingHelper.extraAggregatorBytes(); + return Long.BYTES * 2; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index b5e4969fe481..5b5b76f4d667 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -25,19 +25,21 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseDoubleColumnValueSelector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; import javax.annotation.Nullable; @@ -48,7 +50,7 @@ import java.util.Map; import java.util.Objects; -public class DoubleLastAggregatorFactory extends AggregatorFactory +public class DoubleLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; @@ -69,20 +71,20 @@ public DoubleLastAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new DoubleLastAggregator( + return Pair.of(new DoubleLastAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), doubleColumnSelector ), doubleColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new DoubleLastBufferAggregator( + return Pair.of(new DoubleLastBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), doubleColumnSelector ), doubleColumnSelector); @@ -108,7 +110,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("DoubleLastAggregatorFactory is not supported during ingestion for rollup"); } @@ -119,10 +121,10 @@ public AggregatorFactory getCombiningFactory() return new DoubleLastAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new DoubleLastAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new DoubleLastAggregator(null, null) { @Override public void aggregate() @@ -137,10 +139,10 @@ public void aggregate() } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new DoubleLastBufferAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new DoubleLastBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -223,9 +225,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Long.BYTES + Double.BYTES + NullHandlingHelper.extraAggregatorBytes(); + return Long.BYTES + Double.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index f7eaffbbe8ca..7864d0e30dba 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -25,19 +25,21 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.aggregation.first.FloatFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseFloatColumnValueSelector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; import javax.annotation.Nullable; @@ -48,7 +50,7 @@ import java.util.Map; import java.util.Objects; -public class FloatLastAggregatorFactory extends AggregatorFactory +public class FloatLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; @@ -67,20 +69,20 @@ public FloatLastAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new FloatLastAggregator( + return Pair.of(new FloatLastAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), floatColumnSelector ), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new FloatLastBufferAggregator( + return Pair.of(new FloatLastBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), floatColumnSelector ), floatColumnSelector); @@ -106,7 +108,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("FloatLastAggregatorFactory is not supported during ingestion for rollup"); } @@ -117,10 +119,10 @@ public AggregatorFactory getCombiningFactory() return new FloatLastAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new FloatLastAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new FloatLastAggregator(null, null) { @Override public void aggregate() @@ -135,10 +137,10 @@ public void aggregate() } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new FloatLastBufferAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new FloatLastBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -219,9 +221,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Long.BYTES + Float.BYTES + NullHandlingHelper.extraAggregatorBytes(); + return Long.BYTES + Float.BYTES; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 88f859b7817a..38380072ff62 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -24,19 +24,21 @@ import com.google.common.base.Preconditions; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseLongColumnValueSelector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; import javax.annotation.Nullable; @@ -47,7 +49,7 @@ import java.util.Map; import java.util.Objects; -public class LongLastAggregatorFactory extends AggregatorFactory +public class LongLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; private final String name; @@ -65,20 +67,20 @@ public LongLastAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new LongLastAggregator( + return Pair.of(new LongLastAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), longColumnSelector ), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return NullHandlingHelper.getNullableAggregator(new LongLastBufferAggregator( + return Pair.of(new LongLastBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), longColumnSelector ), longColumnSelector); @@ -104,7 +106,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("LongLastAggregatorFactory is not supported during ingestion for rollup"); } @@ -115,10 +117,10 @@ public AggregatorFactory getCombiningFactory() return new LongLastAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new LongLastAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new LongLastAggregator(null, null) { @Override public void aggregate() @@ -133,10 +135,10 @@ public void aggregate() } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return NullHandlingHelper.getNullableAggregator(new LongLastBufferAggregator(null, null) + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of(new LongLastBufferAggregator(null, null) { @Override public void aggregate(ByteBuffer buf, int position) @@ -215,9 +217,9 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { - return Long.BYTES * 2 + NullHandlingHelper.extraAggregatorBytes(); + return Long.BYTES * 2; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java index e1aaa73664eb..87b25a63021e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -24,13 +24,13 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; import io.druid.segment.DimensionHandlerUtils; -import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -111,7 +111,7 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - Double retVal = NullHandlingHelper.useDefaultValuesForNull() ? DimensionHandlerUtils.ZERO_DOUBLE : null; + Double retVal = NullHandling.useDefaultValuesForNull() ? DimensionHandlerUtils.ZERO_DOUBLE : null; if (fieldsIter.hasNext()) { Number nextVal = (Number) fieldsIter.next().compute(values); if (nextVal == null) { diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index b85b236008ec..95f23fc05e9f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -23,12 +23,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; -import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -76,7 +76,7 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - Double retVal = NullHandlingHelper.useDefaultValuesForNull() ? Double.NEGATIVE_INFINITY : null; + Double retVal = NullHandling.useDefaultValuesForNull() ? Double.NEGATIVE_INFINITY : null; while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java index 3ffa723244f9..9c47002e2ed8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -24,11 +24,11 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; -import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -76,7 +76,7 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - Double retVal = NullHandlingHelper.useDefaultValuesForNull() ? Double.POSITIVE_INFINITY : null; + Double retVal = NullHandling.useDefaultValuesForNull() ? Double.POSITIVE_INFINITY : null; while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java index e4fadad31de1..567969fb2b93 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -23,12 +23,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; -import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -76,7 +76,7 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - Long retVal = NullHandlingHelper.useDefaultValuesForNull() ? Long.MIN_VALUE : null; + Long retVal = NullHandling.useDefaultValuesForNull() ? Long.MIN_VALUE : null; while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java index 03a0b18e6bc0..1d6faaab9c28 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -24,11 +24,11 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; -import io.druid.segment.NullHandlingHelper; import java.util.Comparator; import java.util.Iterator; @@ -76,7 +76,7 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - Long retVal = NullHandlingHelper.useDefaultValuesForNull() ? Long.MAX_VALUE : null; + Long retVal = NullHandling.useDefaultValuesForNull() ? Long.MAX_VALUE : null; while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. diff --git a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java index fecb0e727e54..3fb3e852d676 100644 --- a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java @@ -23,11 +23,11 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; import io.druid.segment.IdLookup; -import io.druid.segment.NullHandlingHelper; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import javax.annotation.Nullable; @@ -106,7 +106,7 @@ private DimensionSelector filterWhiteList(DimensionSelector selector) } } else { for (int i = 0; i < selectorCardinality; i++) { - if (values.contains(NullHandlingHelper.nullToEmptyIfNeeded(selector.lookupName(i)))) { + if (values.contains(NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } @@ -137,7 +137,7 @@ public boolean apply(@Nullable String input) forwardMapping.defaultReturnValue(-1); final int[] reverseMapping = new int[maxPossibleFilteredCardinality]; for (int i = 0; i < selectorCardinality; i++) { - if (!values.contains(NullHandlingHelper.nullToEmptyIfNeeded(selector.lookupName(i)))) { + if (!values.contains(NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } diff --git a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java index f80faaed39f6..ce11703476be 100644 --- a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -22,10 +22,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; import it.unimi.dsi.fastutil.ints.Int2IntMap; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; @@ -76,7 +76,7 @@ public DimensionSelector decorate(final DimensionSelector selector) @Override public boolean apply(@Nullable String input) { - return compiledRegex.matcher(NullHandlingHelper.nullToEmptyIfNeeded(input)).matches(); + return compiledRegex.matcher(NullHandling.nullToEmptyIfNeeded(input)).matches(); } } ); @@ -86,7 +86,7 @@ public boolean apply(@Nullable String input) final Int2IntOpenHashMap forwardMapping = new Int2IntOpenHashMap(); forwardMapping.defaultReturnValue(-1); for (int i = 0; i < selectorCardinality; i++) { - String val = NullHandlingHelper.nullToEmptyIfNeeded(selector.lookupName(i)); + String val = NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)); if (val != null && compiledRegex.matcher(val).matches()) { forwardMapping.put(i, count++); } diff --git a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java index d848b0a83e31..d03ea8d13c7d 100644 --- a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java @@ -19,13 +19,13 @@ package io.druid.query.expression; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.ExprType; import io.druid.query.filter.LikeDimFilter; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nonnull; import java.util.List; @@ -63,7 +63,7 @@ public Expr apply(final List args) } final LikeDimFilter.LikeMatcher likeMatcher = LikeDimFilter.LikeMatcher.from( - NullHandlingHelper.nullToEmptyIfNeeded((String) patternExpr.getLiteralValue()), + NullHandling.nullToEmptyIfNeeded((String) patternExpr.getLiteralValue()), escapeChar ); diff --git a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java index 0516dc6a77d9..940630c250c6 100644 --- a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java @@ -19,11 +19,11 @@ package io.druid.query.expression; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; import io.druid.math.expr.ExprMacroTable; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nonnull; import java.util.List; @@ -64,9 +64,9 @@ class RegexpExtractExpr implements Expr public ExprEval eval(final ObjectBinding bindings) { String s = arg.eval(bindings).asString(); - final Matcher matcher = pattern.matcher(NullHandlingHelper.nullToEmptyIfNeeded(s)); + final Matcher matcher = pattern.matcher(NullHandling.nullToEmptyIfNeeded(s)); final String retVal = matcher.find() ? matcher.group(index) : null; - return ExprEval.of(NullHandlingHelper.emptyToNullIfNeeded(retVal)); + return ExprEval.of(NullHandling.emptyToNullIfNeeded(retVal)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java index 5b5931e8f1e4..1bacaab5ec52 100644 --- a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java +++ b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java @@ -21,7 +21,7 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import javax.annotation.Nullable; @@ -52,7 +52,7 @@ public FunctionalExtraction( ) { this.retainMissingValue = retainMissingValue; - this.replaceMissingValueWith = NullHandlingHelper.emptyToNullIfNeeded(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandling.emptyToNullIfNeeded(replaceMissingValueWith); Preconditions.checkArgument( !(this.retainMissingValue && !(this.replaceMissingValueWith == null)), "Cannot specify a [replaceMissingValueWith] and set [retainMissingValue] to true" @@ -69,7 +69,7 @@ public FunctionalExtraction( public String apply(@Nullable String dimValue) { final String retval = extractionFunction.apply(dimValue); - return NullHandlingHelper.isNullOrEquivalent(retval) ? NullHandlingHelper.emptyToNullIfNeeded(dimValue) : retval; + return NullHandling.isNullOrEquivalent(retval) ? NullHandling.emptyToNullIfNeeded(dimValue) : retval; } }; } else { @@ -79,7 +79,7 @@ public String apply(@Nullable String dimValue) @Override public String apply(@Nullable String dimValue) { - final String retval = NullHandlingHelper.emptyToNullIfNeeded(extractionFunction.apply(dimValue)); + final String retval = NullHandling.emptyToNullIfNeeded(extractionFunction.apply(dimValue)); return retval == null ? FunctionalExtraction.this.replaceMissingValueWith : retval; diff --git a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java index efe31a3040bc..0967bdfac0ad 100644 --- a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java @@ -19,7 +19,7 @@ package io.druid.query.extraction; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import javax.annotation.Nullable; @@ -42,14 +42,14 @@ public byte[] getCacheKey() @Nullable public String apply(@Nullable Object value) { - return value == null ? null : NullHandlingHelper.emptyToNullIfNeeded(value.toString()); + return value == null ? null : NullHandling.emptyToNullIfNeeded(value.toString()); } @Override @Nullable public String apply(@Nullable String value) { - return NullHandlingHelper.emptyToNullIfNeeded(value); + return NullHandling.emptyToNullIfNeeded(value); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java index 4f641da6435e..48ddeb2ac405 100644 --- a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java @@ -25,9 +25,9 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.js.JavaScriptConfig; -import io.druid.segment.NullHandlingHelper; import org.mozilla.javascript.Context; import org.mozilla.javascript.ContextFactory; import org.mozilla.javascript.ScriptableObject; @@ -139,7 +139,7 @@ private void checkAndCompileScript() @Nullable public String apply(@Nullable String value) { - return this.apply((Object) NullHandlingHelper.emptyToNullIfNeeded(value)); + return this.apply((Object) NullHandling.emptyToNullIfNeeded(value)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java index 59b9ff9c4f6c..6f60449e33d5 100644 --- a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -53,7 +53,7 @@ public LowerExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (NullHandlingHelper.isNullOrEquivalent(key)) { + if (NullHandling.isNullOrEquivalent(key)) { return null; } return key.toLowerCase(locale); diff --git a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java index de07a1fc04e2..5c1054afad91 100644 --- a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -62,7 +62,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (NullHandlingHelper.isNullOrEquivalent(dimValue)) { + if (NullHandling.isNullOrEquivalent(dimValue)) { // We'd return null whether or not the pattern matched return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java index 446d5773f4e8..feab409d4ba1 100644 --- a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -106,14 +106,14 @@ public byte[] getCacheKey() public String apply(@Nullable String dimValue) { final String retVal; - String val = NullHandlingHelper.nullToEmptyIfNeeded(dimValue); + String val = NullHandling.nullToEmptyIfNeeded(dimValue); final Matcher matcher = val == null ? null : pattern.matcher(val); if (matcher != null && matcher.find()) { retVal = matcher.group(index); } else { retVal = replaceMissingValue ? replaceMissingValueWith : dimValue; } - return NullHandlingHelper.emptyToNullIfNeeded(retVal); + return NullHandling.emptyToNullIfNeeded(retVal); } @JsonProperty("expr") diff --git a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java index ee9a04b47815..3e30daa6f5f6 100644 --- a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.common.config.NullHandling; import io.druid.query.search.SearchQuerySpec; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -64,7 +64,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - return searchQuerySpec.accept(dimValue) ? NullHandlingHelper.emptyToNullIfNeeded(dimValue) : null; + return searchQuerySpec.accept(dimValue) ? NullHandling.emptyToNullIfNeeded(dimValue) : null; } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java index 9528541a5b52..fa08a9741f69 100644 --- a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java @@ -20,7 +20,7 @@ package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import javax.annotation.Nullable; @@ -41,7 +41,7 @@ public static StrlenExtractionFn instance() @Override public String apply(@Nullable String value) { - if (!NullHandlingHelper.useDefaultValuesForNull() && value == null) { + if (!NullHandling.useDefaultValuesForNull() && value == null) { return null; } return String.valueOf(value == null ? 0 : value.length()); diff --git a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java index a282147db6c9..c980aaa920fb 100644 --- a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -63,7 +63,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (NullHandlingHelper.isNullOrEquivalent(dimValue)) { + if (NullHandling.isNullOrEquivalent(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java index 3c40596e7d2e..c69a21558a68 100644 --- a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.ibm.icu.text.SimpleDateFormat; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -77,7 +77,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (NullHandlingHelper.isNullOrEquivalent(dimValue)) { + if (NullHandling.isNullOrEquivalent(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java index 6ab50683dbfb..85f6358efd19 100644 --- a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -52,7 +52,7 @@ public UpperExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (NullHandlingHelper.isNullOrEquivalent(key)) { + if (NullHandling.isNullOrEquivalent(key)) { return null; } return key.toUpperCase(locale); diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index 94f6635061cb..d35b77997a86 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -31,13 +31,13 @@ import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.Comparators; import io.druid.query.extraction.ExtractionFn; import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.DimensionHandlerUtils; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.filter.InFilter; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; @@ -79,7 +79,7 @@ public InDimFilter( this.values = new TreeSet<>(Comparators.naturalNullsFirst()); for (String value : values) { - this.values.add(NullHandlingHelper.emptyToNullIfNeeded(value)); + this.values.add(NullHandling.emptyToNullIfNeeded(value)); } this.dimension = dimension; this.extractionFn = extractionFn; @@ -175,7 +175,7 @@ private InDimFilter optimizeLookup() // there may be row values that match the selector value but are not included // in the lookup map. Match on the selector value as well. // If the selector value is overwritten in the lookup map, don't add selector value to keys. - if (exFn.isRetainMissingValue() && NullHandlingHelper.isNullOrEquivalent(lookup.apply(convertedValue))) { + if (exFn.isRetainMissingValue() && NullHandling.isNullOrEquivalent(lookup.apply(convertedValue))) { keys.add(convertedValue); } } diff --git a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java index 256fdf2b399c..8d8dfb8476e2 100644 --- a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java @@ -27,9 +27,9 @@ import com.google.common.collect.RangeSet; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.Indexed; import io.druid.segment.filter.LikeFilter; @@ -152,7 +152,7 @@ private static void addPatternCharacter(final StringBuilder patternBuilder, fina public boolean matches(@Nullable final String s) { - String val = NullHandlingHelper.nullToEmptyIfNeeded(s); + String val = NullHandling.nullToEmptyIfNeeded(s); return val != null && pattern.matcher(val).matches(); } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 2749467877cc..b889cd39eda4 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -30,10 +30,10 @@ import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; +import io.druid.common.config.NullHandling; import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.filter.DimensionPredicateFilter; import io.druid.segment.filter.SelectorFilter; @@ -65,7 +65,7 @@ public SelectorDimFilter( Preconditions.checkArgument(dimension != null, "dimension must not be null"); this.dimension = dimension; - this.value = NullHandlingHelper.emptyToNullIfNeeded(value); + this.value = NullHandling.emptyToNullIfNeeded(value); this.extractionFn = extractionFn; } diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java index ff93c5562ba2..658c61631e87 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.Row; import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; @@ -33,7 +34,6 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.IdLookup; import io.druid.segment.LongColumnSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; @@ -227,7 +227,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (Objects.equals(NullHandlingHelper.emptyToNullIfNeeded(dimensionValue), value)) { + if (Objects.equals(NullHandling.emptyToNullIfNeeded(dimensionValue), value)) { return true; } } @@ -252,7 +252,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (Objects.equals(extractionFn.apply(NullHandlingHelper.emptyToNullIfNeeded(dimensionValue)), value)) { + if (Objects.equals(extractionFn.apply(NullHandling.emptyToNullIfNeeded(dimensionValue)), value)) { return true; } } @@ -285,7 +285,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (predicate.apply(NullHandlingHelper.emptyToNullIfNeeded(dimensionValue))) { + if (predicate.apply(NullHandling.emptyToNullIfNeeded(dimensionValue))) { return true; } } @@ -311,7 +311,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (predicate.apply(extractionFn.apply(NullHandlingHelper.emptyToNullIfNeeded(dimensionValue)))) { + if (predicate.apply(extractionFn.apply(NullHandling.emptyToNullIfNeeded(dimensionValue)))) { return true; } } @@ -337,7 +337,7 @@ public int getValueCardinality() @Override public String lookupName(int id) { - final String value = NullHandlingHelper.emptyToNullIfNeeded(row.get().getDimension(dimension).get(id)); + final String value = NullHandling.emptyToNullIfNeeded(row.get().getDimension(dimension).get(id)); return extractionFn == null ? value : extractionFn.apply(value); } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index ddfbb84028a9..62d9c74deb97 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -33,6 +33,7 @@ import com.google.common.primitives.Longs; import com.google.common.util.concurrent.ListeningExecutorService; import io.druid.collections.ResourceHolder; +import io.druid.common.config.NullHandling; import io.druid.common.utils.IntArrayUtils; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; @@ -64,7 +65,6 @@ import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; @@ -442,7 +442,7 @@ public Row apply(Grouper.Entry entry) Object dimVal = entry.getKey().getKey()[i]; theMap.put( query.getDimensions().get(i - dimStart).getOutputName(), - dimVal instanceof String ? NullHandlingHelper.emptyToNullIfNeeded((String) dimVal) : dimVal + dimVal instanceof String ? NullHandling.emptyToNullIfNeeded((String) dimVal) : dimVal ); } @@ -1588,7 +1588,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) protected Long readFromBuffer(ByteBuffer buffer, int initialOffset) { if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandlingHelper.nullToZeroIfNeeded((Long) null); + return NullHandling.nullToZeroIfNeeded((Long) null); } else { return buffer.getLong(initialOffset + keyBufferPosition + Byte.BYTES); } @@ -1656,7 +1656,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) protected Float readFromBuffer(ByteBuffer buffer, int initialOffset) { if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandlingHelper.nullToZeroIfNeeded((Float) null); + return NullHandling.nullToZeroIfNeeded((Float) null); } else { return buffer.getFloat(initialOffset + keyBufferPosition + Byte.BYTES); } @@ -1724,7 +1724,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) protected Double readFromBuffer(ByteBuffer buffer, int initialOffset) { if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandlingHelper.nullToZeroIfNeeded((Double) null); + return NullHandling.nullToZeroIfNeeded((Double) null); } else { return buffer.getDouble(initialOffset + keyBufferPosition + Byte.BYTES); } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java index 2e26d480ca6e..8ba0cc6dfa67 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java @@ -21,9 +21,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.ArrayBasedIndexedInts; import io.druid.segment.data.IndexedInts; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; @@ -60,7 +60,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, value ); } else { - resultMap.put(selectorPlus.getOutputName(), NullHandlingHelper.nullToEmptyIfNeeded((String) null)); + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultValue()); } } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java index 646167464fef..01697066a978 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java @@ -21,9 +21,9 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.IndexedInts; import java.nio.ByteBuffer; @@ -49,7 +49,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, ((DimensionSelector) selectorPlus.getSelector()).lookupName(id) ); } else { - resultMap.put(selectorPlus.getOutputName(), NullHandlingHelper.nullToEmptyIfNeeded((String) null)); + resultMap.put(selectorPlus.getOutputName(), NullHandling.nullToEmptyIfNeeded((String) null)); } } diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java index 15de6c5cb3f5..54ea60353bc1 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java @@ -24,10 +24,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.base.Throwables; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionCacheHelper; import io.druid.query.extraction.FunctionalExtraction; -import io.druid.segment.NullHandlingHelper; import javax.annotation.Nullable; import java.io.ByteArrayOutputStream; @@ -58,7 +58,7 @@ public LookupExtractionFn( @Override public String apply(@Nullable String input) { - return NullHandlingHelper.emptyToNullIfNeeded(lookup.apply(NullHandlingHelper.nullToEmptyIfNeeded(input))); + return NullHandling.emptyToNullIfNeeded(lookup.apply(NullHandling.nullToEmptyIfNeeded(input))); } }, retainMissingValue, diff --git a/processing/src/main/java/io/druid/query/search/SearchHit.java b/processing/src/main/java/io/druid/query/search/SearchHit.java index 314a5a3513be..4031d0df3d88 100644 --- a/processing/src/main/java/io/druid/query/search/SearchHit.java +++ b/processing/src/main/java/io/druid/query/search/SearchHit.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; /** */ @@ -40,7 +40,7 @@ public SearchHit( ) { this.dimension = Preconditions.checkNotNull(dimension); - this.value = NullHandlingHelper.nullToEmptyIfNeeded(value); + this.value = NullHandling.nullToEmptyIfNeeded(value); this.count = count; } diff --git a/processing/src/main/java/io/druid/segment/AggregatorNullHandling.java b/processing/src/main/java/io/druid/segment/AggregatorNullHandling.java new file mode 100644 index 000000000000..fe4aefaf3287 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/AggregatorNullHandling.java @@ -0,0 +1,41 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.google.common.base.Supplier; +import io.druid.common.config.NullHandling; + +public class AggregatorNullHandling +{ + + public static Supplier getNullableSupplier( + final Supplier supplier, + final BaseNullableColumnValueSelector selector + ) + { + return NullHandling.useDefaultValuesForNull() ? + supplier : () -> { + if (selector.isNull()) { + return null; + } + return supplier.get(); + }; + } +} diff --git a/processing/src/main/java/io/druid/segment/BaseNullableColumnValueSelector.java b/processing/src/main/java/io/druid/segment/BaseNullableColumnValueSelector.java index 6188732c14f4..3d2c6a5b990f 100644 --- a/processing/src/main/java/io/druid/segment/BaseNullableColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/BaseNullableColumnValueSelector.java @@ -21,13 +21,6 @@ import io.druid.guice.annotations.PublicApi; -/** - * Object value selecting polymorphic "part" of the {@link ColumnValueSelector} interface. Users of {@link - * ColumnValueSelector#getObject()} are encouraged to reduce the parameter/field/etc. type to - * BaseObjectColumnValueSelector to make it impossible to accidently call any method other than {@link #getObject()}. - *

- * All implementations of this interface MUST also implement {@link ColumnValueSelector}. - */ @PublicApi public interface BaseNullableColumnValueSelector { diff --git a/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java b/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java index b85545a8d51b..1afd14650da7 100644 --- a/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java @@ -31,7 +31,7 @@ * All implementations of this interface MUST also implement {@link ColumnValueSelector}. */ @PublicApi -public interface BaseObjectColumnValueSelector extends BaseNullableColumnValueSelector +public interface BaseObjectColumnValueSelector { @Nullable T getObject(); diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index b1641c5ed0f9..d333f5900700 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -22,6 +22,7 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.spatial.ImmutableRTree; +import io.druid.common.config.NullHandling; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BitmapIndex; @@ -209,7 +210,7 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) final Column column = index.getColumn(dimension); if (column == null || !columnSupportsFiltering(column)) { - if (NullHandlingHelper.isNullOrEquivalent(value)) { + if (NullHandling.isNullOrEquivalent(value)) { return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows()); } else { return bitmapFactory.makeEmptyImmutableBitmap(); diff --git a/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java b/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java index a733e7a11a80..5f05f8f9d6d5 100644 --- a/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java @@ -20,6 +20,7 @@ package io.druid.segment; import com.google.common.base.Predicate; +import io.druid.common.config.NullHandling; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.IndexedInts; @@ -36,7 +37,7 @@ public class ConstantDimensionSelector implements SingleValueHistoricalDimension public ConstantDimensionSelector(final String value) { - if (NullHandlingHelper.isNullOrEquivalent(value)) { + if (NullHandling.isNullOrEquivalent(value)) { // There's an optimized implementation for nulls that callers should use instead. throw new IllegalArgumentException("Use NullDimensionSelector or DimensionSelectorUtils.constantSelector"); } diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java index a7403be88d3b..7ee5746d86b5 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; +import io.druid.common.config.NullHandling; import io.druid.common.guava.GuavaUtils; import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; import io.druid.java.util.common.IAE; @@ -340,16 +341,16 @@ public static Long getExactLongFromDecimalString(String decimalStr) public static Number nullToZeroDouble(@Nullable Number number) { - return number == null ? NullHandlingHelper.ZERO_DOUBLE : number; + return number == null ? NullHandling.ZERO_DOUBLE : number; } public static Number nullToZeroLong(@Nullable Number number) { - return number == null ? NullHandlingHelper.ZERO_LONG : number; + return number == null ? NullHandling.ZERO_LONG : number; } public static Number nullToZeroFloat(@Nullable Number number) { - return number == null ? NullHandlingHelper.ZERO_FLOAT : number; + return number == null ? NullHandling.ZERO_FLOAT : number; } } diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java index 8e9e078105e3..e4b43e7fbf38 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java @@ -21,6 +21,7 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -99,7 +100,7 @@ class IndexerDoubleColumnSelector implements DoubleColumnSelector @Override public boolean isNull() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { return false; } final Object[] dims = currEntry.get().getDims(); diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index 8fc5e5439ac9..ede5f5ee4b93 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -29,6 +29,7 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.inject.ImplementedBy; +import io.druid.common.config.NullHandling; import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.ByteBufferUtils; import io.druid.java.util.common.ISE; @@ -456,7 +457,7 @@ class DictionaryMergeIterator implements CloseableIterator final PeekingIterator iter = Iterators.peekingIterator( Iterators.transform( indexed.iterator(), - input -> NullHandlingHelper.nullToEmptyIfNeeded(input) + input -> NullHandling.nullToEmptyIfNeeded(input) ) ); if (iter.hasNext()) { diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index a7b8489b796e..56ad73fcae1e 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -20,6 +20,7 @@ package io.druid.segment; import com.google.common.base.Predicate; +import io.druid.common.config.NullHandling; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.IndexedInts; @@ -102,7 +103,7 @@ public IdLookup idLookup() @Override public int lookupId(String name) { - return NullHandlingHelper.isNullOrEquivalent(name) ? 0 : -1; + return NullHandling.isNullOrEquivalent(name) ? 0 : -1; } @Nullable diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index fe18cdb7eafb..e47b39ffd99e 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -25,6 +25,7 @@ import com.google.common.primitives.Ints; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Comparators; @@ -58,7 +59,7 @@ public class StringDimensionIndexer implements DimensionIndexer { private static final Function EMPTY_TO_NULL_IF_NEEDED = o -> o != null - ? NullHandlingHelper.emptyToNullIfNeeded(o.toString()) + ? NullHandling.emptyToNullIfNeeded(o.toString()) : null; private static final int ABSENT_VALUE_ID = -1; diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 2f9b74faf0a9..41416e7c9799 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Ints; import io.druid.collections.ResourceHolder; +import io.druid.common.config.NullHandling; import io.druid.common.utils.SerializerUtils; import io.druid.io.Channels; import io.druid.java.util.common.IAE; @@ -31,7 +32,6 @@ import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.serde.MetaSerdeHelper; import io.druid.segment.serde.Serializer; import io.druid.segment.writeout.HeapByteBufferWriteOutBytes; @@ -105,13 +105,13 @@ public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) // nulBytes will be -1 for null values. return null; } - return NullHandlingHelper.emptyToNullIfNeeded(StringUtils.fromUtf8Nullable(buffer, numBytes)); + return NullHandling.emptyToNullIfNeeded(StringUtils.fromUtf8Nullable(buffer, numBytes)); } @Override public byte[] toBytes(String val) { - return StringUtils.toUtf8Nullable(NullHandlingHelper.nullToEmptyIfNeeded(val)); + return StringUtils.toUtf8Nullable(NullHandling.nullToEmptyIfNeeded(val)); } @Override diff --git a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java index 62cceaa0e032..c8115b804a39 100644 --- a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Supplier; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.Pair; import io.druid.query.BitmapResultFactory; import io.druid.query.extraction.ExtractionFn; @@ -37,7 +38,6 @@ import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.IntListUtils; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.BitmapIndex; import it.unimi.dsi.fastutil.ints.IntList; @@ -149,7 +149,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasLowerBound()) { startIndex = 0; } else { - final int found = bitmapIndex.getIndex(NullHandlingHelper.emptyToNullIfNeeded(boundDimFilter.getLower())); + final int found = bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(boundDimFilter.getLower())); if (found >= 0) { startIndex = boundDimFilter.isLowerStrict() ? found + 1 : found; } else { @@ -160,7 +160,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasUpperBound()) { endIndex = bitmapIndex.getCardinality(); } else { - final int found = bitmapIndex.getIndex(NullHandlingHelper.emptyToNullIfNeeded(boundDimFilter.getUpper())); + final int found = bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(boundDimFilter.getUpper())); if (found >= 0) { endIndex = boundDimFilter.isUpperStrict() ? found : found + 1; } else { @@ -250,10 +250,10 @@ private boolean doesMatch(String input) { if (input == null) { return (!boundDimFilter.hasLowerBound() - || (NullHandlingHelper.isNullOrEquivalent(boundDimFilter.getLower()) && !boundDimFilter.isLowerStrict())) + || (NullHandling.isNullOrEquivalent(boundDimFilter.getLower()) && !boundDimFilter.isLowerStrict())) // lower bound allows null && (!boundDimFilter.hasUpperBound() - || !NullHandlingHelper.isNullOrEquivalent(boundDimFilter.getUpper()) + || !NullHandling.isNullOrEquivalent(boundDimFilter.getUpper()) || !boundDimFilter.isUpperStrict()); // upper bound allows null } int lowerComparing = 1; diff --git a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java index ff546a23fbc1..c13d9edd3aa0 100644 --- a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import io.druid.common.config.NullHandling; import io.druid.math.expr.Evals; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -34,7 +35,6 @@ import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.virtual.ExpressionSelectors; import java.util.Set; @@ -110,7 +110,7 @@ public T getBitmapResult(final BitmapIndexSelector selector, final BitmapRes // There's only one binding, and it must be the single column, so it can safely be ignored in production. assert column.equals(identifierName); // convert null to Empty before passing to expressions if needed. - return NullHandlingHelper.nullToEmptyIfNeeded(value); + return NullHandling.nullToEmptyIfNeeded(value); }).asBoolean() ); } diff --git a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java index 1b92cb660fb1..3e2576586405 100644 --- a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java @@ -22,6 +22,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.query.BitmapResultFactory; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; @@ -30,7 +31,6 @@ import io.druid.query.filter.ValueMatcher; import io.druid.segment.ColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.BitmapIndex; import io.druid.segment.data.Indexed; import it.unimi.dsi.fastutil.ints.IntIterable; @@ -93,7 +93,7 @@ private Iterable getBitmapIterable(final BitmapIndexSelector se // Verify that dimension equals prefix. return ImmutableList.of(selector.getBitmapIndex( dimension, - NullHandlingHelper.emptyToNullIfNeeded(likeMatcher.getPrefix()) + NullHandling.emptyToNullIfNeeded(likeMatcher.getPrefix()) )); } else if (isSimplePrefix()) { // Verify that dimension startsWith prefix, and is accepted by likeMatcher.matchesSuffixOnly. diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java index ca97f69e23be..9e860db0cc32 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java @@ -24,16 +24,17 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; import io.druid.math.expr.Parser; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.AggregatorNullHandling; import io.druid.segment.BaseObjectColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; @@ -68,13 +69,13 @@ private static Expr.ObjectBinding createBindings(ColumnSelectorFactory columnSel final Supplier supplier; if (nativeType == ValueType.FLOAT) { final ColumnValueSelector columnValueSelector = columnSelectorFactory.makeColumnValueSelector(columnName); - supplier = NullHandlingHelper.getNullableSupplier(columnValueSelector::getFloat, columnValueSelector); + supplier = AggregatorNullHandling.getNullableSupplier(columnValueSelector::getFloat, columnValueSelector); } else if (nativeType == ValueType.LONG) { final ColumnValueSelector columnValueSelector = columnSelectorFactory.makeColumnValueSelector(columnName); - supplier = NullHandlingHelper.getNullableSupplier(columnValueSelector::getLong, columnValueSelector); + supplier = AggregatorNullHandling.getNullableSupplier(columnValueSelector::getLong, columnValueSelector); } else if (nativeType == ValueType.DOUBLE) { final ColumnValueSelector columnValueSelector = columnSelectorFactory.makeColumnValueSelector(columnName); - supplier = NullHandlingHelper.getNullableSupplier(columnValueSelector::getDouble, columnValueSelector); + supplier = AggregatorNullHandling.getNullableSupplier(columnValueSelector::getDouble, columnValueSelector); } else if (nativeType == ValueType.STRING) { supplier = supplierFromDimensionSelector( columnSelectorFactory.makeDimensionSelector(new DefaultDimensionSpec(columnName, columnName)) @@ -104,13 +105,13 @@ static Supplier supplierFromDimensionSelector(final DimensionSelector se final IndexedInts row = selector.getRow(); if (row.size() == 0) { // Treat empty multi-value rows as nulls. - return NullHandlingHelper.nullToEmptyIfNeeded((String) null); + return NullHandling.nullToEmptyIfNeeded((String) null); } else if (row.size() == 1) { - return NullHandlingHelper.nullToEmptyIfNeeded(selector.lookupName(row.get(0))); + return NullHandling.nullToEmptyIfNeeded(selector.lookupName(row.get(0))); } else { // Can't handle multi-value rows in expressions. // Treat them as nulls until we think of something better to do. - return NullHandlingHelper.nullToEmptyIfNeeded((String) null); + return NullHandling.nullToEmptyIfNeeded((String) null); } }; } @@ -121,7 +122,7 @@ static Supplier supplierFromObjectSelector(final BaseObjectColumnValueSe { if (selector == null) { // Missing column. - return Suppliers.ofInstance(NullHandlingHelper.nullToEmptyIfNeeded((String) null)); + return Suppliers.ofInstance(NullHandling.nullToEmptyIfNeeded((String) null)); } final Class clazz = selector.classOfObject(); @@ -133,11 +134,11 @@ static Supplier supplierFromObjectSelector(final BaseObjectColumnValueSe return () -> { final Object val = selector.getObject(); if (val instanceof String) { - return NullHandlingHelper.nullToEmptyIfNeeded((String) val); + return NullHandling.nullToEmptyIfNeeded((String) val); } else if (val instanceof Number) { return val; } else { - return NullHandlingHelper.nullToEmptyIfNeeded((String) null); + return NullHandling.nullToEmptyIfNeeded((String) null); } }; } else { diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 9700ef0c6235..5f8652643aa5 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -24,6 +24,7 @@ import com.google.common.base.Supplier; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; import io.druid.math.expr.Parser; @@ -36,7 +37,6 @@ import io.druid.segment.ColumnValueSelector; import io.druid.segment.ConstantColumnValueSelector; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.NilColumnValueSelector; import io.druid.segment.column.Column; @@ -204,7 +204,7 @@ class DefaultExpressionDimensionSelector extends BaseSingleValueDimensionSelecto @Override protected String getValue() { - return NullHandlingHelper.emptyToNullIfNeeded(baseSelector.getObject().asString()); + return NullHandling.emptyToNullIfNeeded(baseSelector.getObject().asString()); } @Override @@ -220,7 +220,7 @@ class ExtractionExpressionDimensionSelector extends BaseSingleValueDimensionSele @Override protected String getValue() { - return extractionFn.apply(NullHandlingHelper.emptyToNullIfNeeded(baseSelector.getObject().asString())); + return extractionFn.apply(NullHandling.emptyToNullIfNeeded(baseSelector.getObject().asString())); } @Override diff --git a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java index 1a1bfeac8769..f510db561ce7 100644 --- a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java +++ b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java @@ -20,7 +20,7 @@ package io.druid.guice; import com.google.inject.Injector; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -35,7 +35,7 @@ public void testNullHandlingHelperUseDefaultValues() try { System.setProperty(NULL_HANDLING_CONFIG_STRING, "true"); Injector injector = GuiceInjectors.makeStartupInjector(); - Assert.assertEquals(true, NullHandlingHelper.useDefaultValuesForNull()); + Assert.assertEquals(true, NullHandling.useDefaultValuesForNull()); } finally { if (prev != null) { @@ -51,7 +51,7 @@ public void testNullHandlingHelperNoDefaultValues() try { System.setProperty(NULL_HANDLING_CONFIG_STRING, "false"); Injector injector = GuiceInjectors.makeStartupInjector(); - Assert.assertEquals(false, NullHandlingHelper.useDefaultValuesForNull()); + Assert.assertEquals(false, NullHandling.useDefaultValuesForNull()); } finally { if (prev != null) { @@ -64,7 +64,7 @@ private void resetNullHandlingHelper(String prev) { System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); Injector injector = GuiceInjectors.makeStartupInjector(); - Assert.assertEquals(Boolean.valueOf(prev), NullHandlingHelper.useDefaultValuesForNull()); + Assert.assertEquals(Boolean.valueOf(prev), NullHandling.useDefaultValuesForNull()); } } diff --git a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java index 51a9b189047f..ca2d594b56f6 100644 --- a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java +++ b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.Maps; import com.google.common.io.Closeables; import com.google.common.util.concurrent.MoreExecutors; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.MapInputRowParser; @@ -48,7 +49,6 @@ import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; import io.druid.query.timeseries.TimeseriesResultValue; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexSegment; import io.druid.segment.TestHelper; @@ -288,7 +288,7 @@ public void testNumericEvolutionTimeseriesAggregation() ); // Only nonexistent(4) - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals( timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L, "d", 0.0)), runQuery(query, factory, ImmutableList.of(index4)) @@ -372,9 +372,9 @@ public void testNumericEvolutionFiltering() Assert.assertEquals( timeseriesResult(TestHelper.createExpectedMap( "a", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null, + NullHandling.useDefaultValuesForNull() ? 0L : null, "b", - NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : null, + NullHandling.useDefaultValuesForNull() ? 0.0 : null, "c", 0L )), diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java index 05760443724f..acd64376b59e 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java @@ -20,8 +20,8 @@ package io.druid.query.aggregation; import com.google.common.primitives.Doubles; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import org.easymock.EasyMock; import org.junit.Assert; @@ -70,7 +70,7 @@ public void testDoubleMaxAggregator() Assert.assertEquals(values[2], agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(Double.NEGATIVE_INFINITY, (Double) agg.get(), 0.0001); } else { Assert.assertNull(agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java index b0abe9c05fc5..8fad426663d0 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java @@ -20,8 +20,8 @@ package io.druid.query.aggregation; import com.google.common.primitives.Doubles; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import org.easymock.EasyMock; import org.junit.Assert; @@ -70,7 +70,7 @@ public void testDoubleMinAggregator() Assert.assertEquals(values[2], agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(Double.POSITIVE_INFINITY, (Double) agg.get(), 0.0001); } else { Assert.assertNull(agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 164764131231..96300eee0adc 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.js.JavaScriptConfig; import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; @@ -44,7 +45,6 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.IdLookup; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ValueType; @@ -223,9 +223,9 @@ public ColumnCapabilities getColumnCapabilities(String columnName) private void assertValues(FilteredAggregator agg, TestFloatColumnSelector selector, double... expectedVals) { - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 0.0d : null, agg.get()); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 0.0d : null, agg.get()); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 0.0d : null, agg.get()); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 0.0d : null, agg.get()); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 0.0d : null, agg.get()); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 0.0d : null, agg.get()); for (double expectedVal : expectedVals) { aggregate(selector, agg); Assert.assertEquals(expectedVal, agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java index fde1d1cee73c..3f779733ae4f 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java @@ -20,8 +20,8 @@ package io.druid.query.aggregation; import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import org.easymock.EasyMock; import org.junit.Assert; @@ -70,7 +70,7 @@ public void testLongMaxAggregator() Assert.assertEquals((float) values[2], agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(Long.MIN_VALUE, ((Long) agg.get()).longValue()); } else { Assert.assertNull(agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java index b54cfbb1f473..31a70ad0c890 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java @@ -20,8 +20,8 @@ package io.druid.query.aggregation; import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import org.easymock.EasyMock; import org.junit.Assert; @@ -70,7 +70,7 @@ public void testLongMinAggregator() Assert.assertEquals((float) values[2], agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(Long.MAX_VALUE, ((Long) agg.get()).longValue()); } else { Assert.assertNull(agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index 445487f9e178..9d2ed8941ae6 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.js.JavaScriptConfig; import io.druid.query.ColumnSelectorPlus; @@ -47,7 +48,6 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.IdLookup; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.data.IndexedInts; import org.junit.Assert; import org.junit.Test; @@ -436,8 +436,8 @@ public void testAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { aggregate(selectorList, agg); } - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.05); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get())); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.05); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get())); } @Test @@ -480,8 +480,8 @@ public void testBufferAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { bufferAggregate(selectorList, agg, buf, pos); } - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get(buf, pos))); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get(buf, pos))); } @Test @@ -560,11 +560,11 @@ public void testCombineValues() aggregate(selector2, agg2); } - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 4.0 : 3.0, (Double) valueAggregatorFactory.finalizeComputation(agg1.get()), 0.05); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg2.get()), 0.05); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 4.0 : 3.0, (Double) valueAggregatorFactory.finalizeComputation(agg1.get()), 0.05); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg2.get()), 0.05); Assert.assertEquals( - NullHandlingHelper.useDefaultValuesForNull() ? 7.0 : 6.0, + NullHandling.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) rowAggregatorFactory.finalizeComputation( rowAggregatorFactory.combine( agg1.get(), diff --git a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java index 85b40e6d23cb..76b00ca1ffd0 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.first; import io.druid.collections.SerializablePair; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; @@ -29,7 +30,6 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -89,7 +89,7 @@ public void testDoubleFirstAggregator() Assert.assertEquals(doubleValues[1], agg.getDouble(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } else { Assert.assertNull(agg.get()); @@ -145,7 +145,7 @@ public void testDoubleFirstCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getDouble(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } else { Assert.assertNull(agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java index 4bffcd2ca849..4c123ae73f8d 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.first; import io.druid.collections.SerializablePair; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; @@ -29,7 +30,6 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -89,7 +89,7 @@ public void testDoubleFirstAggregator() Assert.assertEquals(floats[1], agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } else { Assert.assertNull(agg.get()); @@ -145,7 +145,7 @@ public void testDoubleFirstCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } else { Assert.assertNull(agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java index e0c0d7ebc458..13a691a2ca16 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.first; import io.druid.collections.SerializablePair; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; @@ -28,7 +29,6 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -88,7 +88,7 @@ public void testLongFirstAggregator() Assert.assertEquals(longValues[3], agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } else { Assert.assertNull(agg.get()); @@ -144,7 +144,7 @@ public void testLongFirstCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } else { Assert.assertNull(agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java index e2bf0e44cc90..1f25c10d5cb7 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.last; import io.druid.collections.SerializablePair; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; @@ -29,7 +30,6 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -89,7 +89,7 @@ public void testDoubleLastAggregator() Assert.assertEquals(doubles[0], agg.getDouble(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } else { Assert.assertNull(agg.get()); @@ -145,7 +145,7 @@ public void testDoubleLastCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getDouble(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } } diff --git a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java index a17247a0c382..8252d5ca1c21 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.last; import io.druid.collections.SerializablePair; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; @@ -29,7 +30,6 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -89,7 +89,7 @@ public void testDoubleLastAggregator() Assert.assertEquals(floats[0], agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } else { Assert.assertNull(agg.get()); @@ -145,7 +145,7 @@ public void testDoubleLastCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getFloat(), 0.0001); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs, 0.0001); } else { Assert.assertNull(agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java index 70ce6a6d59b9..faa232abd136 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.last; import io.druid.collections.SerializablePair; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; @@ -28,7 +29,6 @@ import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.Column; import org.easymock.EasyMock; import org.junit.Assert; @@ -88,7 +88,7 @@ public void testLongLastAggregator() Assert.assertEquals(longValues[2], agg.getFloat(), 1); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs.longValue()); } else { Assert.assertNull(agg.get()); @@ -144,7 +144,7 @@ public void testLongLastCombiningAggregator() Assert.assertEquals(expected.rhs, agg.getFloat(), 1); agg.reset(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(0, ((Pair) agg.get()).rhs.longValue()); } else { Assert.assertNull(agg.get()); diff --git a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java index 9862d8c2558f..078940ccf93f 100644 --- a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java +++ b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java @@ -22,7 +22,7 @@ import com.google.common.base.Function; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -136,7 +136,7 @@ public void testRetainMissing() false ); final String out = fn.apply(in); - Assert.assertEquals(NullHandlingHelper.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandling.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); } @Test @@ -150,7 +150,7 @@ public void testRetainMissingButFound() false ); final String out = fn.apply(in); - Assert.assertEquals(NullHandlingHelper.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandling.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); } @Test @@ -164,8 +164,8 @@ public void testReplaceMissing() false ); final String out = fn.apply(in); - if (NullHandlingHelper.useDefaultValuesForNull()) { - Assert.assertEquals(NullHandlingHelper.isNullOrEquivalent(out) ? MISSING : out, exFn.apply(in)); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(NullHandling.isNullOrEquivalent(out) ? MISSING : out, exFn.apply(in)); } else { Assert.assertEquals(out == null ? MISSING : out, exFn.apply(in)); } @@ -183,7 +183,7 @@ public void testReplaceMissingBlank() false ); final String out = fn.apply(in); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); } else { Assert.assertEquals(out == null ? "" : out, exFn.apply(in)); @@ -201,7 +201,7 @@ public void testOnlyOneValuePresent() false ); final String out = fn.apply(in); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); } else { Assert.assertEquals(Strings.isNullOrEmpty(out) ? "" : out, exFn.apply(in)); @@ -217,7 +217,7 @@ public void testNullInputs() null, false ); - if (NullHandlingHelper.isNullOrEquivalent(fn.apply(null))) { + if (NullHandling.isNullOrEquivalent(fn.apply(null))) { Assert.assertEquals(null, exFn.apply(null)); } } diff --git a/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java index 302ff3f84152..3672adea16de 100644 --- a/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java @@ -23,10 +23,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.DateTimes; import io.druid.js.JavaScriptConfig; -import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -132,7 +132,7 @@ public void testJavascriptIsNull() Assert.assertEquals("yes", extractionFn.apply((String) null)); Assert.assertEquals("yes", extractionFn.apply((Object) null)); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals("yes", extractionFn.apply("")); } else { Assert.assertEquals("no", extractionFn.apply("")); diff --git a/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java index 4d0647ad2c6c..3bd8f5464dc3 100644 --- a/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java @@ -19,7 +19,7 @@ package io.druid.query.extraction; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -32,7 +32,7 @@ public class LowerExtractionFnTest public void testApply() { Assert.assertEquals("lower 1 string", extractionFn.apply("lOwER 1 String")); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); Assert.assertEquals(null, extractionFn.apply(null)); Assert.assertEquals(null, extractionFn.apply((Object) null)); Assert.assertEquals("1", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java index 86ced0dbd78f..839e9f5f838c 100644 --- a/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; -import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -76,7 +76,7 @@ public void testNullExtraction() Assert.assertNull(extractionFn.apply((Object) null)); Assert.assertNull(extractionFn.apply((String) null)); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply((String) "")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply((String) "")); } @Test diff --git a/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java index 6652802d5dd6..465036df30e1 100644 --- a/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; -import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -146,11 +146,11 @@ public void testNullAndEmpty() String regex = "(.*)/.*/.*"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, false, null); // no match, map empty input value to null - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); // null value, returns null Assert.assertEquals(null, extractionFn.apply(null)); // empty match, map empty result to null - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("/a/b")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("/a/b")); } @Test @@ -169,8 +169,8 @@ public void testMissingValueReplacementWhenPatternMatchesNull() { String regex = "^()$"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, true, "NO MATCH"); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "NO MATCH", extractionFn.apply(null)); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "NO MATCH", extractionFn.apply(null)); Assert.assertEquals("NO MATCH", extractionFn.apply("abc")); } @@ -179,10 +179,10 @@ public void testMissingValueReplacementToEmpty() { String regex = "(bob)"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, true, ""); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply(null)); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("abc")); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("123")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply(null)); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("abc")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("123")); Assert.assertEquals("bob", extractionFn.apply("bobby")); } diff --git a/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java index 60ff23b183d6..ac4ad42e630a 100644 --- a/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java @@ -20,8 +20,8 @@ package io.druid.query.extraction; import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; -import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -30,7 +30,7 @@ public class StrlenExtractionFnTest @Test public void testApply() { - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? "0" : null, StrlenExtractionFn.instance().apply(null)); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? "0" : null, StrlenExtractionFn.instance().apply(null)); Assert.assertEquals("0", StrlenExtractionFn.instance().apply("")); Assert.assertEquals("1", StrlenExtractionFn.instance().apply("x")); Assert.assertEquals("3", StrlenExtractionFn.instance().apply("foo")); diff --git a/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java index 3114f5d521e8..0be208d35083 100644 --- a/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; -import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -47,7 +47,7 @@ public void testEmptyAndNullExtraction() ExtractionFn extractionFn = new TimeDimExtractionFn("MM/dd/yyyy", "MM/yyyy"); Assert.assertNull(extractionFn.apply(null)); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertNull(extractionFn.apply("")); } else { Assert.assertEquals("", extractionFn.apply("")); diff --git a/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java index d8b8ce692524..8b852822d2d4 100644 --- a/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java @@ -19,7 +19,7 @@ package io.druid.query.extraction; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -32,7 +32,7 @@ public class UpperExtractionFnTest public void testApply() { Assert.assertEquals("UPPER", extractionFn.apply("uPpeR")); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); Assert.assertEquals(null, extractionFn.apply(null)); Assert.assertEquals(null, extractionFn.apply((Object) null)); Assert.assertEquals("1", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 380f84e480fd..1ff7951ce2eb 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -35,6 +35,7 @@ import io.druid.collections.DefaultBlockingPool; import io.druid.collections.NonBlockingPool; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.data.input.Row; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; @@ -122,7 +123,6 @@ import io.druid.query.ordering.StringComparators; import io.druid.query.search.ContainsSearchQuerySpec; import io.druid.query.spec.MultipleIntervalSegmentSpec; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; @@ -6955,7 +6955,7 @@ public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() List expectedResults; - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), @@ -7022,7 +7022,7 @@ public void testGroupByWithExtractionDimFilterKeyisNull() MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn; - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); extractionMap.put("", "REPLACED_VALUE"); } else { @@ -7113,7 +7113,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", @@ -7122,7 +7122,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", @@ -7131,7 +7131,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", @@ -7140,7 +7140,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), @@ -7151,7 +7151,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", @@ -7160,7 +7160,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", @@ -7169,7 +7169,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( @@ -7179,7 +7179,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", @@ -7188,7 +7188,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", @@ -7197,7 +7197,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", @@ -7206,7 +7206,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), @@ -7217,7 +7217,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", @@ -7226,7 +7226,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", @@ -7235,7 +7235,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() "rows", 0L, "idx", - NullHandlingHelper.useDefaultValuesForNull() ? 0L : null + NullHandling.useDefaultValuesForNull() ? 0L : null ) ); @@ -7292,7 +7292,7 @@ public void testGroupByWithExtractionDimFilterNullDims() MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn; - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { extractionMap.put("", "EMPTY"); lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); } else { @@ -8278,7 +8278,7 @@ public void testGroupByNumericStringsAsNumericWithDecoration() .build(); List expectedResults; - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { // "entertainment" rows are excluded by the decorated specs, they become empty rows expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( @@ -8353,7 +8353,7 @@ public void testGroupByDecorationOnNumerics() .setGranularity(QueryRunnerTestHelper.allGran) .build(); List expectedResults; - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java index b612fafd8dda..2439e4a49743 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java @@ -27,13 +27,13 @@ import com.google.common.collect.Ordering; import com.google.common.io.Files; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.ByteBufferUtils; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.segment.CloserRule; -import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -112,7 +112,7 @@ public void testGrowing() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 10000, 2); - final int expectedMaxSize = NullHandlingHelper.useDefaultValuesForNull() ? 219 : 210; + final int expectedMaxSize = NullHandling.useDefaultValuesForNull() ? 219 : 210; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -140,7 +140,7 @@ public void testGrowing2() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 2_000_000_000, 2); - final int expectedMaxSize = NullHandlingHelper.useDefaultValuesForNull() ? 40988516 : 39141224; + final int expectedMaxSize = NullHandling.useDefaultValuesForNull() ? 40988516 : 39141224; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -154,7 +154,7 @@ public void testGrowing3() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, Integer.MAX_VALUE, 2); - final int expectedMaxSize = NullHandlingHelper.useDefaultValuesForNull() ? 44938972 : 42955456; + final int expectedMaxSize = NullHandling.useDefaultValuesForNull() ? 44938972 : 42955456; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -168,7 +168,7 @@ public void testNoGrowing() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 10000, Integer.MAX_VALUE); - final int expectedMaxSize = NullHandlingHelper.useDefaultValuesForNull() ? 267 : 258; + final int expectedMaxSize = NullHandling.useDefaultValuesForNull() ? 267 : 258; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java index 5b35bd22f942..7caab84c4f69 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java @@ -22,12 +22,12 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.IAE; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; -import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -54,7 +54,7 @@ public void testLimitAndBufferSwapping() for (int i = 0; i < numRows; i++) { Assert.assertTrue(String.valueOf(i + keyBase), grouper.aggregate(i + keyBase).isOk()); } - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { // bucket size is hash(int) + key(int) + aggs(2 longs) + heap offset(int) = 28 bytes // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes @@ -95,7 +95,7 @@ public void testLimitAndBufferSwapping() Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk()); } - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { // we added another 1000 unique keys // previous size is 112, so next swap occurs after 62 rows // after that, there are 1000 - 62 = 938 rows, 938 / 74 = 12 additional swaps after the first, @@ -150,7 +150,7 @@ public void testMinBufferSize() } // With minimum buffer size, after the first swap, every new key added will result in a swap - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(224, grouper.getGrowthCount()); Assert.assertEquals(104, grouper.getSize()); Assert.assertEquals(209, grouper.getBuckets()); @@ -170,7 +170,7 @@ public void testMinBufferSize() for (int i = 0; i < numRows; i++) { Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk()); } - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(474, grouper.getGrowthCount()); Assert.assertEquals(104, grouper.getSize()); Assert.assertEquals(209, grouper.getBuckets()); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java index 8e3a5a93f400..00710351c0a7 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java @@ -25,13 +25,13 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.concurrent.Execs; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.query.groupby.epinephelinae.Grouper.Entry; -import io.druid.segment.NullHandlingHelper; import org.hamcrest.CoreMatchers; import org.junit.Assert; import org.junit.Rule; @@ -149,7 +149,7 @@ private void testStreamingAggregate(int bufferSize) throws ExecutionException, I public void testNotEnoughBuffer() { expectedException.expect(IllegalStateException.class); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { expectedException.expectMessage("Buffer[50] should be large enough to store at least three records[20]"); } else { expectedException.expectMessage("Buffer[50] should be large enough to store at least three records[21]"); diff --git a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java index e32e268227ff..4972e3c6130a 100644 --- a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java +++ b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java @@ -20,8 +20,8 @@ package io.druid.query.lookup; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.query.extraction.MapLookupExtractor; -import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Test; @@ -66,7 +66,7 @@ public void testNullKeyIsMappable() false, false ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals("bar", lookupExtractionFn.apply(null)); } else { Assert.assertEquals("REPLACE", lookupExtractionFn.apply(null)); diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 1e47cdd7067f..b3bdff5d8f82 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedInputRow; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; @@ -51,7 +52,6 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.ordering.StringComparators; import io.druid.query.spec.MultipleIntervalSegmentSpec; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.QueryableIndexSegment; import io.druid.segment.TestHelper; import io.druid.segment.TestIndex; @@ -753,7 +753,7 @@ public void testSearchWithNullValueInDimension() throws Exception QueryRunner runner = factory.createRunner(new QueryableIndexSegment("asdf", TestIndex.persistRealtimeAndLoadMMapped(index))); List expectedHits = Lists.newLinkedList(); expectedHits.add(new SearchHit("table", "table", 1)); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { expectedHits.add(new SearchHit("table", "", 1)); } else { expectedHits.add(new SearchHit("table", null, 1)); diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index b616a76d143c..37f4dc470d88 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.primitives.Doubles; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.StringUtils; @@ -60,7 +61,6 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.ordering.StringComparators; import io.druid.query.spec.MultipleIntervalSegmentSpec; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; @@ -149,8 +149,8 @@ public void testEmptyTimeseries() .build(); Map resultMap = Maps.newHashMap(); resultMap.put("rows", 0L); - resultMap.put("index", NullHandlingHelper.useDefaultValuesForNull() ? 0D : null); - resultMap.put("first", NullHandlingHelper.useDefaultValuesForNull() ? 0D : null); + resultMap.put("index", NullHandling.useDefaultValuesForNull() ? 0D : null); + resultMap.put("first", NullHandling.useDefaultValuesForNull() ? 0D : null); List> expectedResults = ImmutableList.of( new Result<>( DateTimes.of("2020-04-02"), @@ -234,7 +234,7 @@ public void testFullOnTimeseries() 0.02 ); } else { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals( result.toString(), 0.0D, @@ -700,7 +700,7 @@ public void testTimeseriesQueryZeroFilling() ); Map noRowsResult = Maps.newHashMap(); noRowsResult.put("rows", 0L); - noRowsResult.put("idx", NullHandlingHelper.useDefaultValuesForNull() ? 0L : null); + noRowsResult.put("idx", NullHandling.useDefaultValuesForNull() ? 0L : null); for (Interval interval : iterable) { lotsOfZeroes.add( new Result<>( @@ -1461,8 +1461,8 @@ public void testTimeseriesWithFilterOnNonExistentDimension() Map resultMap = Maps.newHashMap(); resultMap.put("rows", 0L); - resultMap.put("index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : null); - resultMap.put("addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 1.0 : null); + resultMap.put("index", NullHandling.useDefaultValuesForNull() ? 0.0 : null); + resultMap.put("addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 1.0 : null); resultMap.put("uniques", 0.0); List> expectedResults = Arrays.asList( @@ -1614,8 +1614,8 @@ public void testTimeseriesWithNonExistentFilter() .build(); Map resultMap = Maps.newHashMap(); resultMap.put("rows", 0L); - resultMap.put("index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : null); - resultMap.put("addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 1.0 : null); + resultMap.put("index", NullHandling.useDefaultValuesForNull() ? 0.0 : null); + resultMap.put("addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 1.0 : null); resultMap.put("uniques", 0.0); List> expectedResults = Arrays.asList( @@ -1658,8 +1658,8 @@ public void testTimeseriesWithNonExistentFilterAndMultiDim() .build(); Map resultMap = Maps.newHashMap(); resultMap.put("rows", 0L); - resultMap.put("index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : null); - resultMap.put("addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 1.0 : null); + resultMap.put("index", NullHandling.useDefaultValuesForNull() ? 0.0 : null); + resultMap.put("addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 1.0 : null); resultMap.put("uniques", 0.0); List> expectedResults = Arrays.asList( diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 14ca7fb122d2..71feada3ff46 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -30,6 +30,7 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; @@ -88,7 +89,6 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.ordering.StringComparators; import io.druid.query.spec.MultipleIntervalSegmentSpec; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.TestHelper; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; @@ -4235,7 +4235,7 @@ public void testTopNWithExtractionFilterAndFilteredAggregatorCaseNoExistingValue MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn; - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); extractionMap.put("", "NULL"); } else { @@ -4313,7 +4313,7 @@ public void testTopNWithExtractionFilterNoExistingValue() MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn; - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); extractionMap.put("", "NULL"); } else { diff --git a/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java b/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java index 15e47832a2bc..1202159ab7a7 100644 --- a/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.common.config.NullHandling; import io.druid.query.extraction.StringFormatExtractionFn; import io.druid.query.extraction.SubstringDimExtractionFn; import io.druid.segment.data.IndexedInts; @@ -68,7 +69,7 @@ public void testLookupName() throws Exception public void testLookupId() throws Exception { Assert.assertEquals(0, NULL_SELECTOR.idLookup().lookupId(null)); - Assert.assertEquals(NullHandlingHelper.useDefaultValuesForNull() ? 0 : -1, NULL_SELECTOR.idLookup().lookupId("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 0 : -1, NULL_SELECTOR.idLookup().lookupId("")); Assert.assertEquals(-1, NULL_SELECTOR.idLookup().lookupId("billy")); Assert.assertEquals(-1, NULL_SELECTOR.idLookup().lookupId("bob")); diff --git a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java index f40ace07d97e..4bfb68dad2cc 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMultiset; import com.google.common.collect.Sets; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedInputRow; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Comparators; @@ -90,7 +91,7 @@ public void testStringColumnNullHandling() throws Exception nullFlavors.add(mNull); nullFlavors.add(mListOfNull); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { nullFlavors.add(mEmptyString); nullFlavors.add(mListOfEmptyString); } else { @@ -185,7 +186,7 @@ public void testStringColumnNullHandling() throws Exception final List expectedNullRows = new ArrayList<>(); for (int i = 0; i < index.getNumRows(); i++) { final List row = getRow(dictionaryColumn, i); - if (row.isEmpty() || row.stream().anyMatch(NullHandlingHelper::isNullOrEquivalent)) { + if (row.isEmpty() || row.stream().anyMatch(NullHandling::isNullOrEquivalent)) { expectedNullRows.add(i); } } @@ -222,14 +223,14 @@ private static List normalize(final Object value, final boolean hasMulti if (value == null) { retVal.add(null); } else if (value instanceof String) { - retVal.add(NullHandlingHelper.emptyToNullIfNeeded(((String) value))); + retVal.add(NullHandling.emptyToNullIfNeeded(((String) value))); } else if (value instanceof List) { final List list = (List) value; if (list.isEmpty() && !hasMultipleValues) { // empty lists become nulls in single valued columns - retVal.add(NullHandlingHelper.emptyToNullIfNeeded(null)); + retVal.add(NullHandling.emptyToNullIfNeeded(null)); } else { - retVal.addAll(list.stream().map(NullHandlingHelper::emptyToNullIfNeeded).collect(Collectors.toList())); + retVal.addAll(list.stream().map(NullHandling::emptyToNullIfNeeded).collect(Collectors.toList())); } } else { throw new ISE("didn't expect class[%s]", value.getClass()); diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index 6a68941a3847..cc69d5b38eb1 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -28,6 +28,7 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.DimensionSchema; @@ -41,7 +42,6 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; -import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; @@ -58,6 +58,7 @@ import io.druid.segment.incremental.IncrementalIndexAdapter; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.incremental.IndexSizeExceededException; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import it.unimi.dsi.fastutil.ints.IntIterator; import org.joda.time.Interval; import org.junit.Assert; @@ -1251,7 +1252,7 @@ public void testJointDimMerge() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals( ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), ImmutableList.copyOf(adapter.getDimensionNames()) @@ -1416,7 +1417,7 @@ public void testNoRollupMergeWithoutDuplicateRow() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals( ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), ImmutableList.copyOf(adapter.getDimensionNames()) @@ -1428,7 +1429,7 @@ public void testNoRollupMergeWithoutDuplicateRow() throws Exception ); } Assert.assertEquals(4, boatList.size()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); @@ -1584,7 +1585,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals( ImmutableList.of("d3", "d6", "d8", "d9"), ImmutableList.copyOf(adapter.getDimensionNames()) @@ -2265,7 +2266,7 @@ public void testPersistNullColumnSkipping() throws Exception ) ) ); - List expectedColumnNames = NullHandlingHelper.useDefaultValuesForNull() + List expectedColumnNames = NullHandling.useDefaultValuesForNull() ? Arrays.asList("A", "d1") : Arrays.asList("A", "d1", "d2"); List actualColumnNames = Lists.newArrayList(index.getColumnNames()); @@ -2274,14 +2275,14 @@ public void testPersistNullColumnSkipping() throws Exception Assert.assertEquals(expectedColumnNames, actualColumnNames); SmooshedFileMapper sfm = closer.closeLater(SmooshedFileMapper.load(tempDir)); - List expectedFilenames = NullHandlingHelper.useDefaultValuesForNull() ? Arrays.asList( + List expectedFilenames = NullHandling.useDefaultValuesForNull() ? Arrays.asList( "A", "__time", "d1", "index.drd", "metadata.drd" ) - : Arrays.asList( + : Arrays.asList( "A", "__time", "d1", diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java index 7c909de99272..a5dbe9444919 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.Pair; @@ -30,9 +31,6 @@ import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.granularity.Granularity; import io.druid.java.util.common.guava.Sequences; -import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; -import io.druid.segment.writeout.SegmentWriteOutMediumFactory; -import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.QueryPlus; import io.druid.query.QueryRunner; @@ -61,6 +59,9 @@ import io.druid.query.topn.TopNQuery; import io.druid.query.topn.TopNQueryBuilder; import io.druid.query.topn.TopNResultValue; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; +import io.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -406,7 +407,7 @@ public void testNonIntersectingSchemas() .put("addRowsIndexConstant", 103.0D) .put("uniques", UNIQUES_1) .put("maxIndex", 100.0D) - .put("minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : 100.0D) + .put("minIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : 100.0D) .build() ) ) @@ -753,7 +754,7 @@ public void testValueAndEmptySchemas() .put("addRowsIndexConstant", 103.0D) .put("uniques", UNIQUES_1) .put("maxIndex", 100.0D) - .put("minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : 100.0D) + .put("minIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : 100.0D) .build() ) ) @@ -873,11 +874,11 @@ public void testEmptySchemas() new TimeseriesResultValue( TestHelper.createExpectedMap( "rows", 1L, - "index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null, - "addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 2.0D : null, + "index", NullHandling.useDefaultValuesForNull() ? 0.0D : null, + "addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 2.0D : null, "uniques", 0.0D, - "maxIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null, - "minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null + "maxIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : null, + "minIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : null )) ) ); @@ -888,11 +889,11 @@ public void testEmptySchemas() new TimeseriesResultValue( TestHelper.createExpectedMap( "rows", 0L, - "index", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : null, - "addRowsIndexConstant", NullHandlingHelper.useDefaultValuesForNull() ? 1.0D : null, + "index", NullHandling.useDefaultValuesForNull() ? 0.0D : null, + "addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 1.0D : null, "uniques", 0.0D, - "maxIndex", NullHandlingHelper.useDefaultValuesForNull() ? Double.NEGATIVE_INFINITY : null, - "minIndex", NullHandlingHelper.useDefaultValuesForNull() ? Double.POSITIVE_INFINITY : null + "maxIndex", NullHandling.useDefaultValuesForNull() ? Double.NEGATIVE_INFINITY : null, + "minIndex", NullHandling.useDefaultValuesForNull() ? Double.POSITIVE_INFINITY : null ) ) ) @@ -1192,7 +1193,7 @@ public void testDifferentMetrics() .put("addRowsIndexConstant", 912.0D) .put("uniques", UNIQUES_1) .put("maxIndex", 100.0D) - .put("minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0D : 100.0D) + .put("minIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : 100.0D) .build() ) ) diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java index 3f16b1602c58..36e60487ff78 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java @@ -22,11 +22,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.granularity.Granularity; -import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.QueryPlus; import io.druid.query.QueryRunner; @@ -56,6 +56,7 @@ import io.druid.query.topn.TopNQueryBuilder; import io.druid.query.topn.TopNResultValue; import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -151,7 +152,7 @@ public void testFullOnTimeseries() .put("addRowsIndexConstant", 912.0) .put("uniques", 2.000977198748901D) .put("maxIndex", 100.0) - .put("minIndex", NullHandlingHelper.useDefaultValuesForNull() ? 0.0 : 100.0) + .put("minIndex", NullHandling.useDefaultValuesForNull() ? 0.0 : 100.0) .build() ) ) diff --git a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java index 37ac72fdaaf1..ccf0ab066e83 100644 --- a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java @@ -20,7 +20,7 @@ package io.druid.segment.data; import com.google.common.collect.Maps; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -149,7 +149,7 @@ public void testNullSerializationWithoutCache() throws Exception ) ); Assert.assertNull(deserialized.get(0)); - Assert.assertEquals(deserialized.get(1), NullHandlingHelper.emptyToNullIfNeeded("")); + Assert.assertEquals(deserialized.get(1), NullHandling.emptyToNullIfNeeded("")); } @Test @@ -166,7 +166,7 @@ public void testNullSerialization() throws Exception unCached, CachingIndexed.INITIAL_CACHE_CAPACITY); Assert.assertNull(deserialized.get(0)); - Assert.assertEquals(deserialized.get(1), NullHandlingHelper.emptyToNullIfNeeded("")); + Assert.assertEquals(deserialized.get(1), NullHandling.emptyToNullIfNeeded("")); } diff --git a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java index 3ade65b1572c..b34d52e6e08d 100644 --- a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -36,7 +37,6 @@ import io.druid.query.filter.BoundDimFilter; import io.druid.query.ordering.StringComparators; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -111,7 +111,7 @@ public void testLexicographicMatchWithEmptyString() new BoundDimFilter("dim2", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), new BoundDimFilter("dim3", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC) ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { for (BoundDimFilter filter : filters) { assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); } @@ -134,7 +134,7 @@ public void testLexicographicMatchNull() new BoundDimFilter("dim1", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0") ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("1", "2", "5") @@ -150,7 +150,7 @@ public void testLexicographicMatchNull() @Test public void testLexicographicMatchMissingColumn() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") @@ -274,7 +274,7 @@ public void testAlphaNumericMatchNull() new BoundDimFilter("dim1", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), ImmutableList.of("0") ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), ImmutableList.of("1", "2", "5") @@ -383,7 +383,7 @@ public void testNumericMatchNull() new BoundDimFilter("dim1", "", "", false, false, false, null, StringComparators.NUMERIC), ImmutableList.of("0") ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), ImmutableList.of("1", "2", "5") @@ -500,7 +500,7 @@ public void testMatchWithExtractionFn() String nullJsFn = "function(str) { return null; }"; ExtractionFn makeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") @@ -527,7 +527,7 @@ public void testMatchWithExtractionFn() ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new BoundDimFilter( "dim2", diff --git a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java index d1de40c05d4c..99728789002d 100644 --- a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -38,7 +39,6 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -124,7 +124,7 @@ public void testMissingColumnNotSpecifiedInDimensionList() DefaultDimensionSpec.of("dim6"), DefaultDimensionSpec.of("dim7") )), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( DefaultDimensionSpec.of("dim1"), DefaultDimensionSpec.of("dim6") diff --git a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java index f81aa913761f..302ec6db9678 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.FloatDimensionSchema; @@ -36,7 +37,6 @@ import io.druid.query.expression.TestExprMacroTable; import io.druid.query.filter.ExpressionDimFilter; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import io.druid.segment.incremental.IncrementalIndexSchema; import org.junit.AfterClass; @@ -131,7 +131,7 @@ public void testOneMultiValuedStringColumn() { // Expressions currently treat multi-valued arrays as nulls. // This test is just documenting the current behavior, not necessarily saying it makes sense. - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("0", "1", "2", "4", "5", "6", "7", "8")); } else { assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("2")); @@ -194,7 +194,7 @@ public void testCompareColumns() @Test public void testMissingColumn() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(EDF("missing == ''"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); } else { // AS per SQL standard null == null returns false. @@ -202,7 +202,7 @@ public void testMissingColumn() } assertFilterMatches(EDF("missing == '1'"), ImmutableList.of()); assertFilterMatches(EDF("missing == 2"), ImmutableList.of()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { // missing equivaluent to 0 assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); assertFilterMatches(EDF("missing < 2"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); diff --git a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java index b3942fe2883c..f126eb447ffd 100644 --- a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java @@ -24,6 +24,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -46,7 +47,6 @@ import io.druid.query.filter.OrDimFilter; import io.druid.query.filter.SelectorDimFilter; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Assert; @@ -200,7 +200,7 @@ public static void tearDown() throws Exception @Test public void testSinglePreFilterWithNulls() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); } else { assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of()); @@ -217,7 +217,7 @@ public void testSinglePreFilterWithNulls() @Test public void testSinglePostFilterWithNulls() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of("0")); } else { assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of()); @@ -230,7 +230,7 @@ public void testSinglePostFilterWithNulls() assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "abc", null), ImmutableList.of("5", "8")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "ab", null), ImmutableList.of()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); } else { assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-", JS_EXTRACTION_FN), ImmutableList.of("0")); @@ -246,7 +246,7 @@ public void testSinglePostFilterWithNulls() @Test public void testBasicPreAndPostFilterWithNulls() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new AndDimFilter(Arrays.asList( new SelectorDimFilter("dim2", "a", null), @@ -296,7 +296,7 @@ public void testBasicPreAndPostFilterWithNulls() ImmutableList.of() ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new AndDimFilter(Arrays.asList( new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), @@ -378,7 +378,7 @@ public void testOrPostFilterWithNulls() ImmutableList.of("0", "3") ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new OrDimFilter(Arrays.asList( new SelectorDimFilter("dim1", "abc", null), @@ -444,7 +444,7 @@ public void testOrPostFilterWithNulls() ImmutableList.of("0", "3") ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new OrDimFilter(Arrays.asList( new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), @@ -512,7 +512,7 @@ public void testOrPostFilterWithNulls() public void testMissingColumnSpecifiedInDimensionList() { assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new NoBitmapSelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") @@ -569,7 +569,7 @@ public void testMissingColumnSpecifiedInDimensionList() public void testMissingColumnNotSpecifiedInDimensionList() { assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new NoBitmapSelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") diff --git a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java index f2bc0dcf7c6a..ed1d61efe081 100644 --- a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -40,7 +41,6 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -126,7 +126,7 @@ public void testSingleValueStringColumnWithNulls() ImmutableList.of("a") ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( toInFilter("dim1", null, "10", "abc"), ImmutableList.of("a", "b", "f") @@ -147,7 +147,7 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( toInFilter("dim2", null), ImmutableList.of("b", "c", "f") @@ -207,7 +207,7 @@ public void testMissingColumn() ImmutableList.of("a", "b", "c", "d", "e", "f") ); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( toInFilter("dim3", ""), ImmutableList.of("a", "b", "c", "d", "e", "f") @@ -249,7 +249,7 @@ public void testMatchWithExtractionFn() String nullJsFn = "function(str) { if (str === null) { return 'YES'; } else { return 'NO';} }"; ExtractionFn yesNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), ImmutableList.of("a", "b", "c", "d", "f") diff --git a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java index 00480e8e2feb..240cc92e43cd 100644 --- a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -37,7 +38,6 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -110,7 +110,7 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of("0")); } else { assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of()); @@ -128,7 +128,7 @@ public void testSingleValueStringColumnWithNulls() public void testMultiValueStringColumn() { // multi-val null...... - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "2", "5")); } else { assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "5")); diff --git a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java index dd2c2c8504f2..fb989db93fb8 100644 --- a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -33,7 +34,6 @@ import io.druid.query.extraction.SubstringDimExtractionFn; import io.druid.query.filter.LikeDimFilter; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -157,7 +157,7 @@ public void testMatchEmptyString() @Test public void testMatchEmptyStringWithExtractionFn() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), ImmutableList.of("0", "1", "2", "3", "4", "5") diff --git a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java index 0e441b74c6b0..98cd861d51f5 100644 --- a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -35,7 +36,6 @@ import io.druid.query.extraction.JavaScriptExtractionFn; import io.druid.query.filter.RegexDimFilter; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -100,7 +100,7 @@ public void testSingleValueStringColumnWithoutNulls() public void testSingleValueStringColumnWithNulls() { // RegexFilter always returns false for null row values. - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("1", "2", "3", "4", "5")); } else { assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); @@ -116,7 +116,7 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "3", "4")); } else { assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "2", "3", "4")); @@ -150,7 +150,7 @@ public void testRegexWithExtractionFn() { String nullJsFn = "function(str) { if (str === null) { return 'NOT_NULL_ANYMORE'; } else { return str;} }"; ExtractionFn changeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of("0")); assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "2", "5")); } else { diff --git a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java index 4e04df39895f..16a885a8d0ce 100644 --- a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -37,7 +38,6 @@ import io.druid.query.search.ContainsSearchQuerySpec; import io.druid.query.search.SearchQuerySpec; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Test; @@ -106,7 +106,7 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { // SearchQueryFilter always returns false for null row values. assertFilterMatches( new SearchQueryDimFilter("dim1", specForValue(""), null), @@ -129,7 +129,7 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue(""), null), ImmutableList.of("0", "3", "4")); } else { assertFilterMatches( @@ -169,7 +169,7 @@ public void testSearchQueryWithExtractionFn() String nullJsFn = "function(str) { if (str === null) { return 'NOT_NULL_ANYMORE'; } else { return str;} }"; ExtractionFn changeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("0") diff --git a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java index 835623c6e529..05a3e104d2d2 100644 --- a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -38,7 +39,6 @@ import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.segment.IndexBuilder; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import org.junit.AfterClass; import org.junit.Assert; @@ -114,7 +114,7 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); } else { @@ -132,7 +132,7 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "2", "5")); assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("1", "2", "5")); } else { @@ -149,7 +149,7 @@ public void testMultiValueStringColumn() public void testMissingColumnSpecifiedInDimensionList() { assertFilterMatches(new SelectorDimFilter("dim3", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); } else { assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of()); @@ -163,7 +163,7 @@ public void testMissingColumnSpecifiedInDimensionList() public void testMissingColumnNotSpecifiedInDimensionList() { assertFilterMatches(new SelectorDimFilter("dim4", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); } else { assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of()); @@ -219,7 +219,7 @@ public void testSelectorWithLookupExtractionFn() ); LookupExtractor mapExtractor3 = new MapLookupExtractor(stringMap3, false); LookupExtractionFn lookupFn3 = new LookupExtractionFn(mapExtractor3, false, null, false, true); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { // Nulls and empty strings are considered equivalent assertFilterMatches( new SelectorDimFilter("dim0", null, lookupFn3), @@ -277,7 +277,7 @@ public void testSelectorWithLookupExtractionFn() assertFilterMatches(optFilter1, ImmutableList.of("0", "1", "2", "5")); assertFilterMatches(optFilter2, ImmutableList.of("2", "5")); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { // Null and Empty strings are same assertFilterMatches(optFilter3, ImmutableList.of("0", "1", "2", "3", "4", "5")); } else { @@ -291,7 +291,7 @@ public void testSelectorWithLookupExtractionFn() // remove these when ExtractionDimFilter is removed. assertFilterMatches(new ExtractionDimFilter("dim1", "UNKNOWN", lookupFn, null), ImmutableList.of("0", "1", "2", "5")); assertFilterMatches(new ExtractionDimFilter("dim0", "5", lookupFn2, null), ImmutableList.of("2", "5")); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { assertFilterMatches( new ExtractionDimFilter("dim0", null, lookupFn3, null), ImmutableList.of("0", "1", "2", "3", "4", "5") diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index 6930c3d32068..3427a40ecece 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; @@ -50,7 +51,6 @@ import io.druid.query.topn.TopNResultValue; import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.StorageAdapter; import io.druid.segment.VirtualColumns; import io.druid.segment.data.IndexedInts; @@ -601,7 +601,7 @@ public Object apply(Cursor cursor) // no null id, so should get empty dims array Assert.assertEquals(0, rowD.size()); IndexedInts rowE = dimSelector3E.getRow(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(1, rowE.size()); // the null id Assert.assertEquals(0, rowE.get(0)); diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java index 4ceab59b4c28..3ba15dad45e1 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.Row; import io.druid.data.input.impl.DimensionSchema; @@ -37,7 +38,6 @@ import io.druid.query.aggregation.FilteredAggregatorFactory; import io.druid.query.filter.SelectorDimFilter; import io.druid.segment.CloserRule; -import io.druid.segment.NullHandlingHelper; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -225,7 +225,7 @@ public void testNullDimensionTransform() throws IndexSizeExceededException Row row = index.iterator().next(); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(Arrays.asList(null, null, "A"), row.getRaw("string")); Assert.assertEquals( Arrays.asList(null, null, String.valueOf(Float.POSITIVE_INFINITY)), diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java index d2dcd1e904b1..bc31c22605ff 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.Row; @@ -38,7 +39,6 @@ import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; import io.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -114,7 +114,7 @@ public void testObjectSelector() Assert.assertEquals(null, selector.getObject()); CURRENT_ROW.set(ROW1); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(4.0d, selector.getObject()); } else { // y is null for row1 @@ -137,7 +137,7 @@ public void testLongSelector() Assert.assertEquals(0L, selector.getLong()); CURRENT_ROW.set(ROW1); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(4L, selector.getLong()); } else { // y is null for row1 @@ -160,7 +160,7 @@ public void testLongSelectorUsingStringFunction() Assert.assertEquals(0L, selector.getLong()); CURRENT_ROW.set(ROW1); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(4L, selector.getLong()); } else { // y is null for row1 @@ -183,7 +183,7 @@ public void testFloatSelector() Assert.assertEquals(0.0f, selector.getFloat(), 0.0f); CURRENT_ROW.set(ROW1); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(4.0f, selector.getFloat(), 0.0f); } else { // y is null for row1 @@ -216,7 +216,7 @@ public void testDimensionSelector() Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); CURRENT_ROW.set(ROW1); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(false, nullMatcher.matches()); Assert.assertEquals(false, fiveMatcher.matches()); Assert.assertEquals(true, nonNullMatcher.matches()); @@ -259,7 +259,7 @@ public void testDimensionSelectorUsingStringFunction() CURRENT_ROW.set(ROW1); Assert.assertEquals(1, selector.getRow().size()); Assert.assertEquals( - NullHandlingHelper.useDefaultValuesForNull() ? "4" : null, + NullHandling.useDefaultValuesForNull() ? "4" : null, selector.lookupName(selector.getRow().get(0)) ); @@ -291,7 +291,7 @@ public void testDimensionSelectorWithExtraction() Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); CURRENT_ROW.set(ROW1); - if (NullHandlingHelper.useDefaultValuesForNull()) { + if (NullHandling.useDefaultValuesForNull()) { Assert.assertEquals(false, nullMatcher.matches()); Assert.assertEquals(false, fiveMatcher.matches()); Assert.assertEquals(true, nonNullMatcher.matches()); diff --git a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java index 637fa77d51a0..d6d1ea1601bc 100644 --- a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java +++ b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -31,7 +31,7 @@ import io.druid.query.lookup.LookupExtractor; import io.druid.query.lookup.LookupReferencesManager; import io.druid.segment.DimensionSelector; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.segment.column.ValueType; import javax.annotation.Nullable; @@ -78,7 +78,7 @@ public LookupDimensionSpec( { this.retainMissingValue = retainMissingValue; this.optimize = optimize == null ? true : optimize; - this.replaceMissingValueWith = NullHandlingHelper.emptyToNullIfNeeded(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandling.emptyToNullIfNeeded(replaceMissingValueWith); this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be Null"); this.outputName = Preconditions.checkNotNull(outputName, "outputName can not be Null"); this.lookupReferencesManager = lookupReferencesManager; diff --git a/server/src/test/java/io/druid/query/expression/ExprMacroTest.java b/server/src/test/java/io/druid/query/expression/ExprMacroTest.java index b0279164d664..8a1f9b632745 100644 --- a/server/src/test/java/io/druid/query/expression/ExprMacroTest.java +++ b/server/src/test/java/io/druid/query/expression/ExprMacroTest.java @@ -23,7 +23,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.math.expr.Expr; import io.druid.math.expr.Parser; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -44,7 +44,7 @@ public class ExprMacroTest .build() ); - public static String emptyStringResult = NullHandlingHelper.useDefaultValuesForNull() ? null : ""; + public static String emptyStringResult = NullHandling.useDefaultValuesForNull() ? null : ""; @Rule diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java index af99862f5e84..e3c899d85d8a 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java @@ -41,7 +41,7 @@ import io.druid.query.filter.SelectorDimFilter; import io.druid.query.ordering.StringComparator; import io.druid.query.ordering.StringComparators; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; import io.druid.sql.calcite.filtration.BoundRefKey; @@ -311,7 +311,7 @@ private static DimFilter toSimpleLeafFilter( final DimFilter equalFilter = new SelectorDimFilter( druidExpression.getSimpleExtraction().getColumn(), - NullHandlingHelper.nullToEmptyIfNeeded((String) null), + NullHandling.nullToEmptyIfNeeded((String) null), druidExpression.getSimpleExtraction().getExtractionFn() ); diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java index 6260b6bc719a..46554fd4cea5 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java @@ -45,7 +45,7 @@ import io.druid.query.topn.TopNQuery; import io.druid.query.topn.TopNResultValue; import io.druid.segment.DimensionHandlerUtils; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.server.QueryLifecycleFactory; import io.druid.server.security.AuthenticationResult; @@ -398,7 +398,7 @@ private Object coerce(final Object value, final SqlTypeName sqlType) if (SqlTypeName.CHAR_TYPES.contains(sqlType)) { if (value == null || value instanceof String) { - coercedValue = NullHandlingHelper.nullToEmptyIfNeeded((String) value); + coercedValue = NullHandling.nullToEmptyIfNeeded((String) value); } else if (value instanceof NlsString) { coercedValue = ((NlsString) value).getValue(); } else if (value instanceof Number) { diff --git a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java index bfbd4575d336..b5653cdebee3 100644 --- a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java @@ -23,7 +23,7 @@ import com.google.common.collect.Lists; import io.druid.java.util.common.DateTimes; import io.druid.math.expr.ExprMacroTable; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.server.security.AllowAllAuthenticator; import io.druid.server.security.NoopEscalator; import io.druid.server.security.AuthConfig; @@ -143,7 +143,7 @@ public void testSelectAllInFirstFrame() throws Exception Meta.Frame.create( 0, true, - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", "", 2.0f}, @@ -179,7 +179,7 @@ public void testSelectSplitOverTwoFrames() throws Exception Meta.Frame.create( 0, false, - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", "", 2.0f} @@ -199,7 +199,7 @@ public void testSelectSplitOverTwoFrames() throws Exception Meta.Frame.create( 2, true, - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? Lists.newArrayList( new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 2559c051151f..ce86be2a0463 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -83,7 +83,7 @@ import io.druid.query.topn.InvertedTopNMetricSpec; import io.druid.query.topn.NumericTopNMetricSpec; import io.druid.query.topn.TopNQueryBuilder; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; import io.druid.segment.virtual.ExpressionVirtualColumn; @@ -480,7 +480,7 @@ public void testExplainInformationSchemaColumns() throws Exception @Test public void testSelectStar() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT * FROM druid.foo", ImmutableList.of( @@ -567,7 +567,7 @@ public void testExplainSelectStar() throws Exception @Test public void testSelectStarWithLimit() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT * FROM druid.foo LIMIT 2", @@ -591,7 +591,7 @@ public void testSelectStarWithLimit() throws Exception @Test public void testSelectWithProjection() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT SUBSTRING(dim2, 1, 1) FROM druid.foo LIMIT 2", ImmutableList.of( @@ -617,7 +617,7 @@ public void testSelectWithProjection() throws Exception @Test public void testSelectStarWithLimitTimeDescending() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT * FROM druid.foo ORDER BY __time DESC LIMIT 2", @@ -643,7 +643,7 @@ public void testSelectStarWithLimitTimeDescending() throws Exception @Test public void testSelectStarWithoutLimitTimeAscending() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT * FROM druid.foo ORDER BY __time", ImmutableList.of( @@ -688,7 +688,7 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception @Test public void testSelectSingleColumnTwice() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT dim2 x, dim2 y FROM druid.foo LIMIT 2", ImmutableList.of( @@ -809,7 +809,7 @@ public void testSelfJoinWithFallback() throws Exception @Test public void testExplainSelfJoinWithFallback() throws Exception { - String emptyStringEq = NullHandlingHelper.useDefaultValuesForNull() ? null : "\"\""; + String emptyStringEq = NullHandling.useDefaultValuesForNull() ? null : "\"\""; final String explanation = "BindableJoin(condition=[=($0, $2)], joinType=[inner])\n" + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":" @@ -1161,7 +1161,7 @@ public void testHavingOnApproximateCountDistinct() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 3L}, new Object[]{"a", 2L} @@ -1218,7 +1218,7 @@ public void testHavingOnExactCountDistinct() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 3L}, new Object[]{"a", 2L} @@ -1341,7 +1341,7 @@ public void testHavingOnRatio() throws Exception @Test public void testGroupByWithSelectProjections() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" + " dim1," @@ -1374,7 +1374,7 @@ public void testGroupByWithSelectProjections() throws Exception @Test public void testGroupByWithSelectAndOrderByProjections() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" + " dim1," @@ -1424,7 +1424,7 @@ public void testGroupByWithSelectAndOrderByProjections() throws Exception @Test public void testTopNWithSelectProjections() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" @@ -1461,7 +1461,7 @@ public void testTopNWithSelectProjections() throws Exception @Test public void testTopNWithSelectAndOrderByProjections() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" @@ -1581,7 +1581,7 @@ public void testPruneDeadAggregatorsThroughHaving() throws Exception @Test public void testGroupByCaseWhen() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" + " CASE EXTRACT(DAY FROM __time)\n" @@ -1647,7 +1647,7 @@ public void testIsNullString() throws Exception .build() ), ImmutableList.of( - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? // Matches everything but "abc" new Object[]{5L} : // match only null values @@ -1669,14 +1669,14 @@ public void testEmptyStringEquality() throws Exception .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," - + (NullHandlingHelper.useDefaultValuesForNull() ? "1" : "0") + + (NullHandling.useDefaultValuesForNull() ? "1" : "0") + ",(\"dim2\" == ''))")) .aggregators(AGGS(new CountAggregatorFactory("a0"))) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), ImmutableList.of( - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? // Matches everything but "abc" new Object[]{5L} : // match only empty string @@ -1698,13 +1698,13 @@ public void testNullStringEquality() throws Exception .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," - + (NullHandlingHelper.useDefaultValuesForNull() ? "1" : "0") + + (NullHandling.useDefaultValuesForNull() ? "1" : "0") + ",(\"dim2\" == null))")) .aggregators(AGGS(new CountAggregatorFactory("a0"))) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? // Matches everything but "abc" ImmutableList.of(new Object[]{5L}) : // null is not eqaual to null or any other value @@ -1738,7 +1738,7 @@ public void testCoalesceColumns() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"10.1", 1L}, new Object[]{"2", 1L}, @@ -1773,7 +1773,7 @@ public void testColumnIsNull() throws Exception .build() ), ImmutableList.of( - new Object[]{NullHandlingHelper.useDefaultValuesForNull() ? 3L : 2L} + new Object[]{NullHandling.useDefaultValuesForNull() ? 3L : 2L} ) ); } @@ -1997,7 +1997,7 @@ public void testCountNullableColumn() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L} ) : @@ -2353,7 +2353,7 @@ public void testSimpleAggregations() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 6L, 5L, 1L, 6L, 8L, 3L} ) : @@ -2576,7 +2576,7 @@ public void testFilteredAggregations() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{1L, 5L, 1L, 2L, 5L, 5L, 2L, 1L, 5L, 1L, 5L} ) : @@ -3522,7 +3522,7 @@ public void testSelectDistinctWithLimit() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{""}, new Object[]{"a"}, @@ -3553,7 +3553,7 @@ public void testSelectDistinctWithSortAsOuterQuery() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{""}, new Object[]{"a"}, @@ -3584,7 +3584,7 @@ public void testSelectDistinctWithSortAsOuterQuery2() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{""}, new Object[]{"a"}, @@ -3627,7 +3627,7 @@ public void testSelectDistinctWithSortAsOuterQuery4() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{""}, new Object[]{"abc"}, @@ -3759,7 +3759,7 @@ public void testExactCountDistinct() throws Exception .build() ), ImmutableList.of( - new Object[]{NullHandlingHelper.useDefaultValuesForNull() ? 2L : 3L} + new Object[]{NullHandling.useDefaultValuesForNull() ? 2L : 3L} ) ); } @@ -3837,7 +3837,7 @@ public void testExactCountDistinctWithGroupingAndOtherAggregators() throws Excep .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 3L, 3L}, new Object[]{"a", 2L, 1L}, @@ -3919,7 +3919,7 @@ public void testApproxCountDistinct() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 3L, 2L, 2L, 2L, 6L} ) : @@ -4037,7 +4037,7 @@ public void testDoubleNestedGroupBy() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 3L} ) : @@ -4107,7 +4107,7 @@ public void testExactCountDistinctUsingSubquery() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 3L} ) : @@ -4120,9 +4120,9 @@ public void testExactCountDistinctUsingSubquery() throws Exception @Test public void testTopNFilterJoin() throws Exception { - DimFilter filter = NullHandlingHelper.useDefaultValuesForNull() ? + DimFilter filter = NullHandling.useDefaultValuesForNull() ? IN("dim2", Arrays.asList(null, "a"), null) - : SELECTOR("dim2", "a", null); + : SELECTOR("dim2", "a", null); // Filters on top N values of some dimension by using an inner join. testQuery( "SELECT t1.dim1, SUM(t1.cnt)\n" @@ -4171,7 +4171,7 @@ public void testTopNFilterJoin() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 1L}, new Object[]{"1", 1L}, @@ -4411,7 +4411,7 @@ public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws Except .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L, 2L} ) : @@ -4450,7 +4450,7 @@ public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws Except .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L, 2L} ) : @@ -4494,7 +4494,7 @@ public void testExactCountDistinctUsingSubqueryWithWhereToOuterFilter() throws E .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L, 1L} ) : @@ -4585,7 +4585,7 @@ public void testHistogramUsingSubquery() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"1", 1L}, new Object[]{"2", 1L}, @@ -4640,7 +4640,7 @@ public void testHistogramUsingSubqueryWithSort() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"1", 1L}, new Object[]{"2", 1L} @@ -4801,7 +4801,7 @@ public void testSillyQuarters() throws Exception @Test public void testRegexpExtract() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT DISTINCT\n" + " REGEXP_EXTRACT(dim1, '^.'),\n" @@ -4850,7 +4850,7 @@ public void testRegexpExtract() throws Exception @Test public void testGroupBySortPushDown() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT dim2, dim1, SUM(cnt) FROM druid.foo GROUP BY dim2, dim1 ORDER BY dim1 LIMIT 4", ImmutableList.of( @@ -4892,7 +4892,7 @@ public void testGroupBySortPushDown() throws Exception @Test public void testGroupByLimitPushDownWithHavingOnLong() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT dim1, dim2, SUM(cnt) AS thecnt " + "FROM druid.foo " @@ -4928,7 +4928,7 @@ public void testGroupByLimitPushDownWithHavingOnLong() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"10.1", "", 1L}, new Object[]{"2", "", 1L}, @@ -5345,7 +5345,7 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Excepti .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{T("2000"), "", 2L}, new Object[]{T("2000"), "a", 1L}, @@ -5397,7 +5397,7 @@ public void testGroupByStringLength() throws Exception @Test public void testFilterAndGroupByLookup() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; final RegisteredLookupExtractionFn extractionFn = new RegisteredLookupExtractionFn( null, "lookyloo", @@ -5480,7 +5480,7 @@ public void testCountDistinctOfLookup() throws Exception .build() ), ImmutableList.of( - new Object[]{NullHandlingHelper.useDefaultValuesForNull() ? 2L : 1L} + new Object[]{NullHandling.useDefaultValuesForNull() ? 2L : 1L} ) ); } @@ -5816,7 +5816,7 @@ public void testTimeseriesLosAngelesUsingTimeFloorConnectionLosAngeles() throws public void testTimeseriesDontSkipEmptyBuckets() throws Exception { // Tests that query context parameters are passed through to the underlying query engine. - Long defaultVal = NullHandlingHelper.useDefaultValuesForNull() ? 0L : null; + Long defaultVal = NullHandling.useDefaultValuesForNull() ? 0L : null; testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_DONT_SKIP_EMPTY_BUCKETS, @@ -6265,7 +6265,7 @@ public void testGroupByTimeAndOtherDimension() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", T("2000-01-01"), 2L}, new Object[]{"", T("2001-01-01"), 1L}, @@ -6430,7 +6430,7 @@ public void testUsingSubqueryAsFilterOnTwoColumns() throws Exception @Test public void testUsingSubqueryAsFilterWithInnerSort() throws Exception { - String nullValue = NullHandlingHelper.useDefaultValuesForNull() ? "" : null; + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; // Regression test for https://github.com/druid-io/druid/issues/4208 testQuery( @@ -6469,7 +6469,7 @@ public void testUsingSubqueryAsFilterWithInnerSort() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", "a"}, new Object[]{"10.1", nullValue}, diff --git a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java index f53ac2d93f01..38d237e62e24 100644 --- a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java @@ -30,7 +30,7 @@ import io.druid.math.expr.ExprMacroTable; import io.druid.query.QueryInterruptedException; import io.druid.query.ResourceLimitExceededException; -import io.druid.segment.NullHandlingHelper; +import io.druid.common.config.NullHandling; import io.druid.server.security.AllowAllAuthenticator; import io.druid.server.security.NoopEscalator; import io.druid.server.security.AuthConfig; @@ -217,7 +217,7 @@ public void testFieldAliasingGroupBy() throws Exception ).rhs; Assert.assertEquals( - NullHandlingHelper.useDefaultValuesForNull() ? + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( ImmutableMap.of("x", "", "y", ""), ImmutableMap.of("x", "a", "y", "a"), From 9afc302a07a0f118a58e13718b175051f78ff7b5 Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 18 Dec 2017 02:30:55 +0530 Subject: [PATCH 41/50] fix compilation after merge --- .../distinctcount/DistinctCountBufferAggregator.java | 1 - .../druid/segment/serde/DoubleGenericColumnPartSerdeTest.java | 2 +- .../io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java | 2 +- .../io/druid/segment/serde/LongGenericColumnPartSerdeTest.java | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index 5f3d6acbb9f8..841e5681ffdf 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -25,7 +25,6 @@ import io.druid.query.aggregation.BufferAggregator; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionSelector; -import io.druid.common.config.NullHandling; import io.druid.segment.data.IndexedInts; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; diff --git a/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java index f15b73896989..da65317a26c6 100644 --- a/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java +++ b/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java @@ -30,7 +30,7 @@ public class DoubleGenericColumnPartSerdeTest { - private static ObjectMapper jsonMapper = TestHelper.getJsonMapper(); + private static ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); @Test public void testSerdeV2() throws IOException diff --git a/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java index 41680d3b5b8e..4abad1d4688a 100644 --- a/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java +++ b/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java @@ -30,7 +30,7 @@ public class FloatGenericColumnPartSerdeTest { - private static ObjectMapper jsonMapper = TestHelper.getJsonMapper(); + private static ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); @Test public void testSerdeV2() throws IOException diff --git a/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java index 63c155995284..4712aa99b241 100644 --- a/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java +++ b/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java @@ -30,7 +30,7 @@ public class LongGenericColumnPartSerdeTest { - private static ObjectMapper jsonMapper = TestHelper.getJsonMapper(); + private static ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); @Test public void testSerdeV2() throws IOException From df861ced5292e67127ca9420e70395cbe6a57f80 Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 18 Dec 2017 18:28:32 +0530 Subject: [PATCH 42/50] Add more info to debug travis failure --- sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index ce86be2a0463..205274c1ec84 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -6662,7 +6662,7 @@ private void verifyResults( Assert.assertEquals(StringUtils.format("result count: %s", sql), expectedResults.size(), results.size()); for (int i = 0; i < results.size(); i++) { Assert.assertArrayEquals( - StringUtils.format("result #%d: %s", i + 1, sql), + StringUtils.format("result #%d: %s obtainedResults: %s", i + 1, sql, results), expectedResults.get(i), results.get(i) ); From 2470d3b12350359aa324c4924075eddf53ad0a2d Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 20 Dec 2017 17:27:12 +0530 Subject: [PATCH 43/50] Review comments --- .../io/druid/common/config/NullHandling.java | 14 +++++++++--- .../java/io/druid/math/expr/Function.java | 8 +++---- .../DistinctCountAggregator.java | 22 +++++++++++++++---- .../DistinctCountBufferAggregator.java | 20 ++++++++++++++--- .../common/task/RealtimeIndexTaskTest.java | 6 ++--- .../query/aggregation/AggregatorUtil.java | 3 ++- .../NullableAggregateCombiner.java | 10 ++++++--- .../NullableAggregatorFactory.java | 4 +++- .../aggregation/NullableBufferAggregator.java | 1 - ...ngStringGroupByColumnSelectorStrategy.java | 2 +- 10 files changed, 66 insertions(+), 24 deletions(-) diff --git a/common/src/main/java/io/druid/common/config/NullHandling.java b/common/src/main/java/io/druid/common/config/NullHandling.java index 643eeafb7b58..40156202b1ec 100644 --- a/common/src/main/java/io/druid/common/config/NullHandling.java +++ b/common/src/main/java/io/druid/common/config/NullHandling.java @@ -28,8 +28,11 @@ public class NullHandling { private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; - // use these values to ensure that convertObjectToLong(), convertObjectToDouble() and convertObjectToFloat() - // return the same boxed object when returning a constant zero. + /** + * use these values to ensure that {@link NullHandling#nullToZeroIfNeeded(Long)}, + * {@link NullHandling#nullToZeroIfNeeded(Double)}, {@link NullHandling#nullToZeroIfNeeded(Float)} + * return the same boxed object when returning a constant zero + */ public static final Double ZERO_DOUBLE = 0.0d; public static final Float ZERO_FLOAT = 0.0f; public static final Long ZERO_LONG = 0L; @@ -62,11 +65,16 @@ public static String emptyToNullIfNeeded(@Nullable String value) return useDefaultValuesForNull() ? Strings.emptyToNull(value) : value; } - public static String defaultValue() + public static String defaultStringValue() { return useDefaultValuesForNull() ? "" : null; } + public static Long defaultLongValue() + { + return useDefaultValuesForNull() ? ZERO_LONG : null; + } + public static boolean isNullOrEquivalent(@Nullable String value) { return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index 80f45e98655d..c94c6364c268 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -992,7 +992,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } else { // If starting index of substring is greater then the length of string, the result will be a zero length string. // e.g. 'select substring("abc", 4,5) as c;' will return an empty string - return ExprEval.of(NullHandling.defaultValue()); + return ExprEval.of(NullHandling.defaultStringValue()); } } } @@ -1016,7 +1016,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String pattern = args.get(1).eval(bindings).asString(); final String replacement = args.get(2).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandling.defaultValue()); + return ExprEval.of(NullHandling.defaultStringValue()); } return ExprEval.of( arg.replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) @@ -1041,7 +1041,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandling.defaultValue()); + return ExprEval.of(NullHandling.defaultStringValue()); } return ExprEval.of(StringUtils.toLowerCase(arg)); } @@ -1064,7 +1064,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); if (arg == null) { - return ExprEval.of(NullHandling.defaultValue()); + return ExprEval.of(NullHandling.defaultStringValue()); } return ExprEval.of(StringUtils.toUpperCase(arg)); } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java index 657feac74099..5945b24481bd 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java @@ -19,17 +19,19 @@ package io.druid.query.aggregation.distinctcount; +import com.google.common.base.Preconditions; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.query.aggregation.Aggregator; import io.druid.segment.DimensionSelector; -import io.druid.common.config.NullHandling; import io.druid.segment.data.IndexedInts; public class DistinctCountAggregator implements Aggregator { + private static int UNKNOWN = -1; private final DimensionSelector selector; private final MutableBitmap mutableBitmap; - private final int idForNull; + private int idForNull; public DistinctCountAggregator( DimensionSelector selector, @@ -38,7 +40,12 @@ public DistinctCountAggregator( { this.selector = selector; this.mutableBitmap = mutableBitmap; - this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : -1; + Preconditions.checkArgument( + selector.nameLookupPossibleInAdvance() + || selector.getValueCardinality() != DimensionSelector.CARDINALITY_UNKNOWN, + "DistinctCountAggregator not supported for selector" + ); + this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : UNKNOWN; } @Override @@ -55,7 +62,14 @@ public void aggregate() private boolean isNotNull(int index) { - return selector.nameLookupPossibleInAdvance() ? index != idForNull : selector.lookupName(index) != null; + if (idForNull == UNKNOWN) { + String value = selector.lookupName(index); + if (value == null) { + idForNull = index; + return false; + } + } + return index != idForNull; } @Override diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index 841e5681ffdf..aae412138548 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation.distinctcount; +import com.google.common.base.Preconditions; import io.druid.collections.bitmap.MutableBitmap; import io.druid.collections.bitmap.WrappedRoaringBitmap; import io.druid.common.config.NullHandling; @@ -33,9 +34,10 @@ public class DistinctCountBufferAggregator implements BufferAggregator { + private static int UNKNOWN = -1; private final DimensionSelector selector; private final Int2ObjectMap mutableBitmapCollection = new Int2ObjectOpenHashMap<>(); - private final int idForNull; + private int idForNull; public DistinctCountBufferAggregator( @@ -43,7 +45,12 @@ public DistinctCountBufferAggregator( ) { this.selector = selector; - this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : -1; + Preconditions.checkArgument( + selector.nameLookupPossibleInAdvance() + || selector.getValueCardinality() != DimensionSelector.CARDINALITY_UNKNOWN, + "DistinctCountBufferAggregator not supported for selector" + ); + this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : UNKNOWN; } @Override @@ -78,7 +85,14 @@ private MutableBitmap getMutableBitmap(int position) private boolean isNotNull(int index) { - return selector.nameLookupPossibleInAdvance() ? index != idForNull : selector.lookupName(index) != null; + if (idForNull == UNKNOWN) { + String value = selector.lookupName(index); + if (value == null) { + idForNull = index; + return false; + } + } + return index != idForNull; } @Override diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index f16575abdacb..64677e13ce7b 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -120,6 +120,7 @@ import io.druid.server.coordination.DataSegmentServerAnnouncer; import io.druid.server.coordination.ServerType; import io.druid.timeline.DataSegment; +import io.druid.utils.Runnables; import org.easymock.EasyMock; import org.hamcrest.CoreMatchers; import org.joda.time.DateTime; @@ -209,8 +210,7 @@ public InputRow nextRow() @Override public Runnable commit() { - return () -> { - }; + return Runnables.getNoopRunnable(); } @Override @@ -1121,7 +1121,7 @@ public Long sumMetric(final Task task, final DimFilter filter, final String metr Lists.>newArrayList() ); if (results.isEmpty()) { - return NullHandling.nullToZeroIfNeeded((Long) null); + return NullHandling.defaultLongValue(); } return results.get(0).getValue().getLongMetric(metric); } diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index a67a24b7bbb7..a8078aefb0a6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -37,6 +37,7 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.virtual.ExpressionSelectors; +import javax.annotation.Nullable; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -141,7 +142,7 @@ public static BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefau final ExprMacroTable macroTable, final String fieldName, final String fieldExpression, - final Float nullValue + @Nullable final Float nullValue ) { if (fieldName != null && fieldExpression == null) { diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java index c261da320c5d..a4cdf25a6add 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -44,8 +44,12 @@ public NullableAggregateCombiner(AggregateCombiner delegate) @Override public void reset(ColumnValueSelector selector) { - isNullResult = true; - delegate.reset(selector); + if(selector.isNull()){ + isNullResult = true; + } else { + isNullResult = false; + delegate.reset(selector); + } } @Override @@ -81,7 +85,7 @@ public long getLong() @Override public boolean isNull() { - return isNullResult; + return isNullResult || delegate.isNull(); } @Nullable diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java index 698d3afa0157..6dd116b747c8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -26,10 +26,12 @@ import io.druid.segment.ColumnSelectorFactory; /** + * abstract class with functionality to wrap aggregator/bufferAggregator/combiner to make them Nullable. + * Implementations of {@link AggregatorFactory} which needs to Support Nullable Aggregations are encouraged + * to extend this class. */ public abstract class NullableAggregatorFactory extends AggregatorFactory { - @Override final public Aggregator factorize(ColumnSelectorFactory metricFactory) { diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 5e3f6d894f3a..943d82b64e0c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -30,7 +30,6 @@ * Note that the delegate aggregator is not required to perform check for isNull on the columnValueSelector as only non-null values * will be passed to the delegate aggregator. */ - public class NullableBufferAggregator implements BufferAggregator { private static final byte IS_NULL_BYTE = (byte) 1; diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java index 8ba0cc6dfa67..5ba86423687a 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java @@ -60,7 +60,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, value ); } else { - resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultValue()); + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultStringValue()); } } From a3001dc58db531f09eac201b59f562f171d433b2 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 20 Dec 2017 17:45:57 +0530 Subject: [PATCH 44/50] fix unused imports --- .../io/druid/segment/MapVirtualColumn.java | 24 +++++-------------- .../NullableAggregateCombiner.java | 3 +-- .../query/aggregation/NullableAggregator.java | 7 ------ .../aggregation/DoubleMaxAggregationTest.java | 2 -- .../aggregation/DoubleMinAggregationTest.java | 1 - .../aggregation/LongMaxAggregationTest.java | 1 - .../aggregation/LongMinAggregationTest.java | 1 - .../first/DoubleFirstAggregationTest.java | 1 - .../first/FloatFirstAggregationTest.java | 1 - .../first/LongFirstAggregationTest.java | 1 - .../last/DoubleLastAggregationTest.java | 1 - .../last/FloatLastAggregationTest.java | 1 - .../last/LongLastAggregationTest.java | 1 - 13 files changed, 7 insertions(+), 38 deletions(-) diff --git a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java index 4566ed8553a8..80fbf4c201c5 100644 --- a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java +++ b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java @@ -83,12 +83,6 @@ public ColumnValueSelector makeColumnValueSelector(String columnName, ColumnS if (subColumnName == null) { return new MapVirtualColumnValueSelector(keySelector, valueSelector) { - @Override - public boolean isNull() - { - return false; - } - @Override public Class classOfObject() { @@ -121,12 +115,6 @@ public Map getObject() } return new MapVirtualColumnValueSelector(keySelector, valueSelector) { - @Override - public boolean isNull() - { - return getObject() == null; - } - @Override public Class classOfObject() { @@ -151,12 +139,6 @@ public String getObject() } else { return new MapVirtualColumnValueSelector(keySelector, valueSelector) { - @Override - public boolean isNull() - { - return getObject() == null; - } - @Override public Class classOfObject() { @@ -210,6 +192,12 @@ public long getLong() return 0L; } + @Override + public boolean isNull() + { + return false; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java index a4cdf25a6add..3db92d47c8ee 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -29,7 +29,6 @@ * Note that the delegate combiner is not required to perform check for isNull on the columnValueSelector as only non-null values * will be passed to the delegate combiner. */ - public class NullableAggregateCombiner implements AggregateCombiner { private boolean isNullResult = true; @@ -44,7 +43,7 @@ public NullableAggregateCombiner(AggregateCombiner delegate) @Override public void reset(ColumnValueSelector selector) { - if(selector.isNull()){ + if (selector.isNull()) { isNullResult = true; } else { isNullResult = false; diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index 4fbfcd01aa23..5182b6462368 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -53,13 +53,6 @@ public void aggregate() } } - @Override - public void reset() - { - isNullResult = true; - delegate.reset(); - } - @Override @Nullable public Object get() diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java index 847eec78264b..3d7c5a6a393d 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation; import com.google.common.primitives.Doubles; -import io.druid.common.config.NullHandling; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.TestHelper; import org.easymock.EasyMock; @@ -68,7 +67,6 @@ public void testDoubleMaxAggregator() Assert.assertEquals(values[2], ((Double) agg.get()).doubleValue(), 0.0001); Assert.assertEquals((long) values[2], agg.getLong()); Assert.assertEquals(values[2], agg.getFloat(), 0.0001); - } @Test diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java index 0b70075e144a..f5337d3363f0 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation; import com.google.common.primitives.Doubles; -import io.druid.common.config.NullHandling; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.TestHelper; import org.easymock.EasyMock; diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java index cf043b863483..e025a4c126a7 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation; import com.google.common.primitives.Longs; -import io.druid.common.config.NullHandling; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.TestHelper; import org.easymock.EasyMock; diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java index 8586ae50738c..f8ed6b1b09ed 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation; import com.google.common.primitives.Longs; -import io.druid.common.config.NullHandling; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.TestHelper; import org.easymock.EasyMock; diff --git a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java index 5d61073bea67..15d419e06d19 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation.first; import io.druid.collections.SerializablePair; -import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; diff --git a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java index ee5afc5cba96..719c16980dd6 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation.first; import io.druid.collections.SerializablePair; -import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; diff --git a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java index 551b6ce2478c..6c48dbfd5ac1 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation.first; import io.druid.collections.SerializablePair; -import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; diff --git a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java index 261c8a5fc676..423472df77b1 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation.last; import io.druid.collections.SerializablePair; -import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; diff --git a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java index 8cbbb9abfa2e..2179f8feb809 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation.last; import io.druid.collections.SerializablePair; -import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; diff --git a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java index aa8743597cc9..5106749298c8 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java @@ -20,7 +20,6 @@ package io.druid.query.aggregation.last; import io.druid.collections.SerializablePair; -import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; import io.druid.query.aggregation.Aggregator; From f965efe6b1711cbb9b256d6933e49b5e9eb6a66d Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 27 Dec 2017 21:15:31 +0530 Subject: [PATCH 45/50] fix compilation --- .../segment/serde/DoubleGenericColumnPartSerdeV2.java | 9 ++++----- .../segment/serde/FloatGenericColumnPartSerdeV2.java | 7 +++---- .../segment/serde/LongGenericColumnPartSerdeV2.java | 7 +++---- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java index d39c0d347b8c..d768eae1a15e 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -29,9 +29,8 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; -import io.druid.segment.data.ByteBufferSerializer; -import io.druid.segment.data.CompressedDoublesIndexedSupplier; -import io.druid.segment.data.IndexedDoubles; +import io.druid.segment.data.ColumnarDoubles; +import io.druid.segment.data.CompressedColumnarDoublesSuppliers; import javax.annotation.Nullable; import java.io.IOException; @@ -92,7 +91,7 @@ public Deserializer getDeserializer() return (buffer, builder, columnConfig) -> { int offset = buffer.getInt(); int initialPos = buffer.position(); - final Supplier column = CompressedDoublesIndexedSupplier.fromByteBuffer( + final Supplier column = CompressedColumnarDoublesSuppliers.fromByteBuffer( buffer, byteOrder ); @@ -100,7 +99,7 @@ public Deserializer getDeserializer() buffer.position(initialPos + offset); final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { - bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); + bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java index 089efd183804..bae4747d3ff8 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -27,8 +27,7 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; -import io.druid.segment.data.ByteBufferSerializer; -import io.druid.segment.data.CompressedFloatsIndexedSupplier; +import io.druid.segment.data.CompressedColumnarFloatsSupplier; import javax.annotation.Nullable; import java.io.IOException; @@ -143,14 +142,14 @@ public Deserializer getDeserializer() return (buffer, builder, columnConfig) -> { int offset = buffer.getInt(); int initialPos = buffer.position(); - final CompressedFloatsIndexedSupplier column = CompressedFloatsIndexedSupplier.fromByteBuffer( + final CompressedColumnarFloatsSupplier column = CompressedColumnarFloatsSupplier.fromByteBuffer( buffer, byteOrder ); buffer.position(initialPos + offset); final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { - bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); + bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java index 1846102c8e49..dfd37dde4afb 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -27,8 +27,7 @@ import io.druid.segment.column.ValueType; import io.druid.segment.data.BitmapSerde; import io.druid.segment.data.BitmapSerdeFactory; -import io.druid.segment.data.ByteBufferSerializer; -import io.druid.segment.data.CompressedLongsIndexedSupplier; +import io.druid.segment.data.CompressedColumnarLongsSupplier; import javax.annotation.Nullable; import java.io.IOException; @@ -141,14 +140,14 @@ public Deserializer getDeserializer() return (buffer, builder, columnConfig) -> { int offset = buffer.getInt(); int initialPos = buffer.position(); - final CompressedLongsIndexedSupplier column = CompressedLongsIndexedSupplier.fromByteBuffer( + final CompressedColumnarLongsSupplier column = CompressedColumnarLongsSupplier.fromByteBuffer( buffer, byteOrder ); buffer.position(initialPos + offset); final ImmutableBitmap bitmap; if (buffer.hasRemaining()) { - bitmap = ByteBufferSerializer.read(buffer, bitmapSerdeFactory.getObjectStrategy()); + bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); } else { bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); } From cff7d474acea3e54abad4046e3799b2d66f06c95 Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 28 Dec 2017 10:49:58 +0530 Subject: [PATCH 46/50] review comments --- .../io/druid/common/config/NullHandling.java | 25 ++++++++----------- .../io/druid/tests/indexer/ITKafkaTest.java | 2 +- .../epinephelinae/RowBasedGrouperHelper.java | 6 ++--- .../StringGroupByColumnSelectorStrategy.java | 2 +- .../virtual/ExpressionObjectSelector.java | 8 +++--- .../sql/calcite/expression/Expressions.java | 2 +- 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/common/src/main/java/io/druid/common/config/NullHandling.java b/common/src/main/java/io/druid/common/config/NullHandling.java index 40156202b1ec..2631af6ca4c7 100644 --- a/common/src/main/java/io/druid/common/config/NullHandling.java +++ b/common/src/main/java/io/druid/common/config/NullHandling.java @@ -29,8 +29,8 @@ public class NullHandling private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; /** - * use these values to ensure that {@link NullHandling#nullToZeroIfNeeded(Long)}, - * {@link NullHandling#nullToZeroIfNeeded(Double)}, {@link NullHandling#nullToZeroIfNeeded(Float)} + * use these values to ensure that {@link NullHandling#defaultDoubleValue()}, + * {@link NullHandling#defaultFloatValue()} , {@link NullHandling#defaultFloatValue()} * return the same boxed object when returning a constant zero */ public static final Double ZERO_DOUBLE = 0.0d; @@ -65,36 +65,33 @@ public static String emptyToNullIfNeeded(@Nullable String value) return useDefaultValuesForNull() ? Strings.emptyToNull(value) : value; } + @Nullable public static String defaultStringValue() { return useDefaultValuesForNull() ? "" : null; } + @Nullable public static Long defaultLongValue() { return useDefaultValuesForNull() ? ZERO_LONG : null; } - public static boolean isNullOrEquivalent(@Nullable String value) - { - return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; - } - @Nullable - public static Long nullToZeroIfNeeded(@Nullable Long value) + public static Float defaultFloatValue() { - return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_LONG : value; + return useDefaultValuesForNull() ? ZERO_FLOAT : null; } @Nullable - public static Double nullToZeroIfNeeded(@Nullable Double value) + public static Double defaultDoubleValue() { - return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_DOUBLE : value; + return useDefaultValuesForNull() ? ZERO_DOUBLE : null; } - @Nullable - public static Float nullToZeroIfNeeded(@Nullable Float value) + public static boolean isNullOrEquivalent(@Nullable String value) { - return INSTANCE.isUseDefaultValuesForNull() && value == null ? ZERO_FLOAT : value; + return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; } + } diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITKafkaTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITKafkaTest.java index e28e804be5fd..c4db5f6932cc 100644 --- a/integration-tests/src/test/java/io/druid/tests/indexer/ITKafkaTest.java +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITKafkaTest.java @@ -106,7 +106,7 @@ public class ITKafkaTest extends AbstractIndexerTest private IntegrationTestingConfig config; @Test - public void testKafka()Double + public void testKafka() { LOG.info("Starting test: ITKafkaTest"); diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index 62d9c74deb97..1b25a37c43a0 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -1588,7 +1588,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) protected Long readFromBuffer(ByteBuffer buffer, int initialOffset) { if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandling.nullToZeroIfNeeded((Long) null); + return NullHandling.defaultLongValue(); } else { return buffer.getLong(initialOffset + keyBufferPosition + Byte.BYTES); } @@ -1656,7 +1656,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) protected Float readFromBuffer(ByteBuffer buffer, int initialOffset) { if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandling.nullToZeroIfNeeded((Float) null); + return NullHandling.defaultFloatValue(); } else { return buffer.getFloat(initialOffset + keyBufferPosition + Byte.BYTES); } @@ -1724,7 +1724,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) protected Double readFromBuffer(ByteBuffer buffer, int initialOffset) { if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandling.nullToZeroIfNeeded((Double) null); + return NullHandling.defaultDoubleValue(); } else { return buffer.getDouble(initialOffset + keyBufferPosition + Byte.BYTES); } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java index 01697066a978..a5261236e59c 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java @@ -49,7 +49,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, ((DimensionSelector) selectorPlus.getSelector()).lookupName(id) ); } else { - resultMap.put(selectorPlus.getOutputName(), NullHandling.nullToEmptyIfNeeded((String) null)); + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultStringValue()); } } diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java index 9e860db0cc32..6c132273ee81 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java @@ -105,13 +105,13 @@ static Supplier supplierFromDimensionSelector(final DimensionSelector se final IndexedInts row = selector.getRow(); if (row.size() == 0) { // Treat empty multi-value rows as nulls. - return NullHandling.nullToEmptyIfNeeded((String) null); + return NullHandling.defaultStringValue(); } else if (row.size() == 1) { return NullHandling.nullToEmptyIfNeeded(selector.lookupName(row.get(0))); } else { // Can't handle multi-value rows in expressions. // Treat them as nulls until we think of something better to do. - return NullHandling.nullToEmptyIfNeeded((String) null); + return NullHandling.defaultStringValue(); } }; } @@ -122,7 +122,7 @@ static Supplier supplierFromObjectSelector(final BaseObjectColumnValueSe { if (selector == null) { // Missing column. - return Suppliers.ofInstance(NullHandling.nullToEmptyIfNeeded((String) null)); + return Suppliers.ofInstance(NullHandling.defaultStringValue()); } final Class clazz = selector.classOfObject(); @@ -138,7 +138,7 @@ static Supplier supplierFromObjectSelector(final BaseObjectColumnValueSe } else if (val instanceof Number) { return val; } else { - return NullHandling.nullToEmptyIfNeeded((String) null); + return NullHandling.defaultStringValue(); } }; } else { diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java index e3c899d85d8a..b87d052574f4 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java @@ -311,7 +311,7 @@ private static DimFilter toSimpleLeafFilter( final DimFilter equalFilter = new SelectorDimFilter( druidExpression.getSimpleExtraction().getColumn(), - NullHandling.nullToEmptyIfNeeded((String) null), + NullHandling.defaultStringValue(), druidExpression.getSimpleExtraction().getExtractionFn() ); From 7603c8e0fb1d66d9af309429e2255e4948cf56b2 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 16 Jan 2018 23:22:07 +0530 Subject: [PATCH 47/50] review comments --- .../benchmark/FilterPartitionBenchmark.java | 3 +- codestyle/checkstyle.xml | 10 +++ .../io/druid/common/config/NullHandling.java | 9 ++ .../java/io/druid/math/expr/ExprEval.java | 4 +- .../java/io/druid/math/expr/Function.java | 24 ++++-- .../DistinctCountAggregator.java | 8 ++ .../DistinctCountBufferAggregator.java | 8 ++ .../namespace/UriExtractionNamespace.java | 2 + .../cache/JdbcExtractionNamespaceTest.java | 4 + .../io/druid/server/lookup/LoadingLookup.java | 6 +- .../io/druid/server/lookup/PollingLookup.java | 4 +- .../server/lookup/jdbc/JdbcDataFetcher.java | 4 +- .../server/lookup/PollingLookupTest.java | 6 +- .../main/java/io/druid/indexer/JobHelper.java | 2 + .../io/druid/indexer/path/StaticPathSpec.java | 2 + .../java/util/common/parsers/ParserUtils.java | 2 + .../io/druid/query/DefaultQueryMetrics.java | 2 + .../query/aggregation/AggregatorFactory.java | 2 + .../aggregation/LongMaxAggregatorFactory.java | 3 +- .../aggregation/LongMinAggregatorFactory.java | 3 +- .../aggregation/LongSumAggregatorFactory.java | 4 +- .../NullableAggregateCombiner.java | 9 ++ .../query/aggregation/NullableAggregator.java | 11 ++- .../NullableAggregatorFactory.java | 8 +- .../aggregation/NullableBufferAggregator.java | 11 ++- .../SimpleDoubleAggregatorFactory.java | 4 +- .../SimpleFloatAggregatorFactory.java | 4 +- ...alityAggregatorColumnSelectorStrategy.java | 19 +++-- ...alityAggregatorColumnSelectorStrategy.java | 6 ++ ...alityAggregatorColumnSelectorStrategy.java | 18 ++-- ...alityAggregatorColumnSelectorStrategy.java | 9 ++ .../first/DoubleFirstAggregatorFactory.java | 84 +++++++++--------- .../first/FloatFirstAggregatorFactory.java | 83 +++++++++--------- .../first/LongFirstAggregatorFactory.java | 82 +++++++++--------- .../last/DoubleLastAggregatorFactory.java | 83 +++++++++--------- .../last/FloatLastAggregatorFactory.java | 82 +++++++++--------- .../last/LongLastAggregatorFactory.java | 85 ++++++++++--------- .../post/ArithmeticPostAggregator.java | 3 +- .../extraction/JavaScriptExtractionFn.java | 3 +- .../query/extraction/LowerExtractionFn.java | 2 + .../query/extraction/MapLookupExtractor.java | 2 + .../extraction/StringFormatExtractionFn.java | 2 + .../query/extraction/UpperExtractionFn.java | 2 + .../io/druid/query/filter/InDimFilter.java | 10 ++- .../io/druid/query/filter/LikeDimFilter.java | 3 +- .../druid/query/filter/SelectorDimFilter.java | 2 + .../epinephelinae/GroupByQueryEngineV2.java | 6 +- .../io/druid/query/lookup/LookupConfig.java | 2 + .../druid/query/metadata/SegmentAnalyzer.java | 6 +- .../segment/StringDimensionMergerV9.java | 4 +- .../io/druid/segment/filter/LikeFilter.java | 21 +++-- .../query/lookup/LookupExtractionFnTest.java | 6 ++ .../segment/filter/FilterPartitionTest.java | 2 + .../query/dimension/LookupDimensionSpec.java | 3 +- .../query/expression/LookupExprMacro.java | 4 +- .../io/druid/query/lookup/LookupModule.java | 2 + .../java/io/druid/server/QueryLifecycle.java | 4 + .../druid/server/emitter/EmitterModule.java | 2 + .../announcer/ListeningAnnouncerConfig.java | 2 + .../dimension/LookupDimensionSpecTest.java | 2 + .../main/java/io/druid/cli/DumpSegment.java | 26 +++--- 61 files changed, 513 insertions(+), 318 deletions(-) diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java index bd7e8415495e..67864e161e24 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java @@ -610,8 +610,9 @@ public Filter toFilter() if (extractionFn == null) { return new NoBitmapSelectorFilter(dimension, value); } else { + //CHECKSTYLE.OFF: Regexp final String valueOrNull = Strings.emptyToNull(value); - + //CHECKSTYLE.ON: Regexp final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml index a38d63c4e305..da2092ed470c 100644 --- a/codestyle/checkstyle.xml +++ b/codestyle/checkstyle.xml @@ -156,5 +156,15 @@ + + + + + + + + + + diff --git a/common/src/main/java/io/druid/common/config/NullHandling.java b/common/src/main/java/io/druid/common/config/NullHandling.java index 2631af6ca4c7..75df6e980e73 100644 --- a/common/src/main/java/io/druid/common/config/NullHandling.java +++ b/common/src/main/java/io/druid/common/config/NullHandling.java @@ -24,6 +24,11 @@ import javax.annotation.Nullable; +/** + * Helper class for NullHandling. This class is used to switch between SQL compatible Null Handling behavior + * introduced as part of {@link https://github.com/druid-io/druid/issues/4349} and the old druid behavior + * where null values are replaced with default values e.g Null Strings are replaced with empty values. + */ public class NullHandling { private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; @@ -56,13 +61,17 @@ public static boolean useDefaultValuesForNull() @Nullable public static String nullToEmptyIfNeeded(@Nullable String value) { + //CHECKSTYLE.OFF: Regexp return useDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; + //CHECKSTYLE.ON: Regexp } @Nullable public static String emptyToNullIfNeeded(@Nullable String value) { + //CHECKSTYLE.OFF: Regexp return useDefaultValuesForNull() ? Strings.emptyToNull(value) : value; + //CHECKSTYLE.ON: Regexp } @Nullable diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 4233df42254b..2aa724c6081b 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -52,7 +52,7 @@ public static ExprEval of(double doubleValue) return new DoubleExprEval(doubleValue); } - public static ExprEval of(String stringValue) + public static ExprEval of(@Nullable String stringValue) { return new StringExprEval(stringValue); } @@ -231,7 +231,7 @@ public Expr toExpr() private static class StringExprEval extends ExprEval { - private StringExprEval(String value) + private StringExprEval(@Nullable String value) { super(NullHandling.emptyToNullIfNeeded(value)); } diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index c94c6364c268..9a589ebc9fd3 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -19,7 +19,6 @@ package io.druid.math.expr; -import com.google.common.base.Strings; import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; @@ -900,15 +899,21 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) return ExprEval.of(null); } else { // Pass first argument in to the constructor to provide StringBuilder a little extra sizing hint. - final StringBuilder builder = new StringBuilder(Strings.nullToEmpty(args.get(0).eval(bindings).asString())); + String first = NullHandling.nullToEmptyIfNeeded(args.get(0).eval(bindings).asString()); + if (first == null) { + // Result of concatenation is null if any of the Values is null. + // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. + return ExprEval.of(null); + } + final StringBuilder builder = new StringBuilder(first); for (int i = 1; i < args.size(); i++) { - final String s = args.get(i).eval(bindings).asString(); - if (!NullHandling.useDefaultValuesForNull() && s == null) { + final String s = NullHandling.nullToEmptyIfNeeded(args.get(i).eval(bindings).asString()); + if (s == null) { // Result of concatenation is null if any of the Values is null. // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. return ExprEval.of(null); } else { - builder.append(NullHandling.nullToEmptyIfNeeded(s)); + builder.append(s); } } return ExprEval.of(builder.toString()); @@ -951,9 +956,12 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) throw new IAE("Function[%s] needs 2 arguments", name()); } - final String haystack = Strings.nullToEmpty(args.get(0).eval(bindings).asString()); - final String needle = Strings.nullToEmpty(args.get(1).eval(bindings).asString()); + final String haystack = NullHandling.nullToEmptyIfNeeded(args.get(0).eval(bindings).asString()); + final String needle = NullHandling.nullToEmptyIfNeeded(args.get(1).eval(bindings).asString()); + if (haystack == null || needle == null) { + return ExprEval.of(null); + } return ExprEval.of(haystack.indexOf(needle)); } } @@ -1019,7 +1027,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) return ExprEval.of(NullHandling.defaultStringValue()); } return ExprEval.of( - arg.replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) + arg.replace(NullHandling.nullToEmptyIfNeeded(pattern), NullHandling.nullToEmptyIfNeeded(replacement)) ); } } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java index d2edf96ff748..3c66cac3860b 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java @@ -26,6 +26,12 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop. + */ public class DistinctCountAggregator implements Aggregator { private static int UNKNOWN = -1; @@ -67,6 +73,8 @@ private boolean isNotNull(int index) if (value == null) { idForNull = index; return false; + } else { + return true; } } return index != idForNull; diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index aae412138548..5f00dca9fe2c 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -32,6 +32,12 @@ import java.nio.ByteBuffer; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop. + */ public class DistinctCountBufferAggregator implements BufferAggregator { private static int UNKNOWN = -1; @@ -90,6 +96,8 @@ private boolean isNotNull(int index) if (value == null) { idForNull = index; return false; + } else { + return true; } } return index != idForNull; diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java index ff0a93c4da2c..c75b345d1358 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java @@ -396,8 +396,10 @@ public TSVFlatDataParser( "Must specify more than one column to have a key value pair" ); final DelimitedParser delegate = new DelimitedParser( + //CHECKSTYLE.OFF: Regexp Strings.emptyToNull(delimiter), Strings.emptyToNull(listDelimiter), + //CHECKSTYLE.ON: Regexp hasHeaderRow, skipHeaderRows ); diff --git a/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java b/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java index 08cc4ee6ee20..8635110ae161 100644 --- a/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java +++ b/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java @@ -384,7 +384,9 @@ public void testMappingWithoutFilter() String key = e.getKey(); String[] val = e.getValue(); String field = val[0]; + //CHECKSTYLE.OFF: Regexp Assert.assertEquals("non-null check", Strings.emptyToNull(field), Strings.emptyToNull(map.get(key))); + //CHECKSTYLE.ON: Regexp } Assert.assertEquals("null check", null, map.get("baz")); } @@ -415,9 +417,11 @@ public void testMappingWithFilter() String filterVal = val[1]; if (filterVal.equals("1")) { + //CHECKSTYLE.OFF: Regexp Assert.assertEquals("non-null check", Strings.emptyToNull(field), Strings.emptyToNull(map.get(key))); } else { Assert.assertEquals("non-null check", null, Strings.emptyToNull(map.get(key))); + //CHECKSTYLE.ON: Regexp } } } diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java index 3534b322db3a..7f1b64310e61 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java @@ -21,8 +21,8 @@ import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.logger.Logger; import io.druid.query.lookup.LookupExtractor; import io.druid.server.lookup.cache.loading.LoadingCache; @@ -71,7 +71,7 @@ public String apply(final String key) final String presentVal; try { presentVal = loadingCache.get(key, new ApplyCallable(key)); - return Strings.emptyToNull(presentVal); + return NullHandling.emptyToNullIfNeeded(presentVal); } catch (ExecutionException e) { LOGGER.debug("value not found for key [%s]", key); @@ -133,7 +133,7 @@ public ApplyCallable(String key) public String call() throws Exception { // avoid returning null and return an empty string to cache it. - return Strings.nullToEmpty(dataFetcher.fetch(key)); + return NullHandling.nullToEmptyIfNeeded(dataFetcher.fetch(key)); } } diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java index 0abc23b8e226..3a09ae9b248a 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java @@ -20,11 +20,11 @@ package io.druid.server.lookup; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; @@ -119,7 +119,7 @@ public String apply(@NotNull String key) // it must've been closed after swapping while I was getting it. Try again. return this.apply(key); } - return Strings.emptyToNull((String) cache.get(key)); + return NullHandling.emptyToNullIfNeeded((String) cache.get(key)); } finally { if (cacheRefKeeper != null && cache != null) { diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java index 6d0af4e821fb..e1ac561d4ff1 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java @@ -21,9 +21,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.druid.metadata.MetadataStorageConnectorConfig; @@ -132,7 +132,7 @@ public List inTransaction(Handle handle, TransactionStatus status) throw if (pairs.isEmpty()) { return null; } - return Strings.nullToEmpty(pairs.get(0)); + return NullHandling.nullToEmptyIfNeeded(pairs.get(0)); } @Override diff --git a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java index dee07f60332e..4afaed01feae 100644 --- a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java +++ b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java @@ -21,11 +21,11 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.ISE; import io.druid.query.lookup.LookupExtractor; import io.druid.server.lookup.cache.polling.OffHeapPollingCache; @@ -191,7 +191,7 @@ public void testBulkApply() public String apply(String input) { //make sure to rewrite null strings as empty. - return Strings.nullToEmpty(input); + return NullHandling.nullToEmptyIfNeeded(input); } })); } @@ -208,7 +208,7 @@ private void assertMapLookup(Map map, LookupExtractor lookup) for (Map.Entry entry : map.entrySet()) { String key = entry.getKey(); String val = entry.getValue(); - Assert.assertEquals("non-null check", Strings.emptyToNull(val), lookup.apply(key)); + Assert.assertEquals("non-null check", NullHandling.emptyToNullIfNeeded(val), lookup.apply(key)); } } } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java index 16809561cbbc..be2aa2bbda9b 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java @@ -302,8 +302,10 @@ public static void injectSystemProperties(Job job) public static void injectDruidProperties(Configuration configuration, List listOfAllowedPrefix) { + //CHECKSTYLE.OFF: Regexp String mapJavaOpts = Strings.nullToEmpty(configuration.get(MRJobConfig.MAP_JAVA_OPTS)); String reduceJavaOpts = Strings.nullToEmpty(configuration.get(MRJobConfig.REDUCE_JAVA_OPTS)); + //CHECKSTYLE.ON: Regexp for (String propName : System.getProperties().stringPropertyNames()) { for (String prefix : listOfAllowedPrefix) { diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java index 37519fd67e4b..82e305157ff8 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java @@ -128,7 +128,9 @@ public static void addToMultipleInputs( private static void addInputPath(Job job, Iterable pathStrings, Class inputFormatClass) { Configuration conf = job.getConfiguration(); + //CHECKSTYLE.OFF: Regexp StringBuilder inputFormats = new StringBuilder(Strings.nullToEmpty(conf.get(MultipleInputs.DIR_FORMATS))); + //CHECKSTYLE.ON: Regexp String[] paths = Iterables.toArray(pathStrings, String.class); for (int i = 0; i < paths.length - 1; i++) { diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java b/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java index e3975ca15d44..6e2d7a004c5e 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java @@ -52,6 +52,7 @@ public class ParserUtils } } + //CHECKSTYLE.OFF: Regexp public static Function getMultiValueFunction( final String listDelimiter, final Splitter listSplitter @@ -67,6 +68,7 @@ public static Function getMultiValueFunction( } }; } + //CHECKSTYLE.ON: Regexp public static ArrayList generateFieldNames(int length) { diff --git a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java index 7d830fedac00..0fbaeb5fdec8 100644 --- a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java +++ b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java @@ -118,7 +118,9 @@ public void duration(QueryType query) @Override public void queryId(QueryType query) { + //CHECKSTYLE.OFF: Regexp setDimension(DruidMetrics.ID, Strings.nullToEmpty(query.getId())); + //CHECKSTYLE.ON: Regexp } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java index a93f88f4838f..cadde722e50b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java @@ -36,6 +36,8 @@ * AggregatorFactory is a strategy (in the terms of Design Patterns) that represents column aggregation, e. g. min, * max, sum of metric columns, or cardinality of dimension columns (see {@link * io.druid.query.aggregation.cardinality.CardinalityAggregatorFactory}). + * Implementations of {@link AggregatorFactory} which needs to Support Nullable Aggregations are encouraged + * to extend {@link io.druid.query.aggregation.NullableAggregatorFactory}. */ @ExtensionPoint public abstract class AggregatorFactory implements Cacheable diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index 07699519f29e..9487c5000354 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -178,7 +178,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 6fc331210b22..7ea18903e91e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -181,7 +181,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 219911bc06bf..22eda3a1296d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -32,6 +32,7 @@ import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -154,7 +155,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java index 3db92d47c8ee..e33b7b458a4f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -66,18 +66,27 @@ public void fold(ColumnValueSelector selector) @Override public float getFloat() { + if (isNullResult) { + throw new IllegalStateException("Cannot return primitive float for Null Value"); + } return delegate.getFloat(); } @Override public double getDouble() { + if (isNullResult) { + throw new IllegalStateException("Cannot return double for Null Value"); + } return delegate.getDouble(); } @Override public long getLong() { + if (isNullResult) { + throw new IllegalStateException("Cannot return long for Null Value"); + } return delegate.getLong(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index 5182b6462368..1eac57edc34b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -66,25 +66,34 @@ public Object get() @Override public float getFloat() { + if (isNullResult) { + throw new IllegalStateException("Cannot return float for Null Value"); + } return delegate.getFloat(); } @Override public long getLong() { + if (isNullResult) { + throw new IllegalStateException("Cannot return long for Null Value"); + } return delegate.getLong(); } @Override public double getDouble() { + if (isNullResult) { + throw new IllegalStateException("Cannot return double for Null Value"); + } return delegate.getDouble(); } @Override public boolean isNull() { - return isNullResult; + return isNullResult || delegate.isNull(); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java index 6dd116b747c8..7d69ce9f9903 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -33,7 +33,7 @@ public abstract class NullableAggregatorFactory extends AggregatorFactory { @Override - final public Aggregator factorize(ColumnSelectorFactory metricFactory) + public final Aggregator factorize(ColumnSelectorFactory metricFactory) { Pair pair = factorize2( metricFactory); @@ -43,7 +43,7 @@ final public Aggregator factorize(ColumnSelectorFactory metricFactory) protected abstract Pair factorize2(ColumnSelectorFactory metricfactory); @Override - final public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public final BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { Pair pair = factorizeBuffered2( metricFactory); @@ -54,7 +54,7 @@ final public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFact @Override - final public AggregateCombiner makeAggregateCombiner() + public final AggregateCombiner makeAggregateCombiner() { AggregateCombiner combiner = makeAggregateCombiner2(); return NullHandling.useDefaultValuesForNull() ? combiner : new NullableAggregateCombiner(combiner); @@ -63,7 +63,7 @@ final public AggregateCombiner makeAggregateCombiner() protected abstract AggregateCombiner makeAggregateCombiner2(); @Override - final public int getMaxIntermediateSize() + public final int getMaxIntermediateSize() { return getMaxIntermediateSize2() + (NullHandling.useDefaultValuesForNull() ? 0 : Byte.BYTES); } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 943d82b64e0c..cd27bc6039d0 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -76,25 +76,34 @@ public Object get(ByteBuffer buf, int position) @Override public float getFloat(ByteBuffer buf, int position) { + if (isNull(buf, position)) { + throw new IllegalStateException("Cannot return float for Null Value"); + } return delegate.getFloat(buf, position + Byte.BYTES); } @Override public long getLong(ByteBuffer buf, int position) { + if (isNull(buf, position)) { + throw new IllegalStateException("Cannot return long for Null Value"); + } return delegate.getLong(buf, position + Byte.BYTES); } @Override public double getDouble(ByteBuffer buf, int position) { + if (isNull(buf, position)) { + throw new IllegalStateException("Cannot return double for Null Value"); + } return delegate.getDouble(buf, position + Byte.BYTES); } @Override public boolean isNull(ByteBuffer buf, int position) { - return buf.get(position) == IS_NULL_BYTE; + return buf.get(position) == IS_NULL_BYTE || delegate.isNull(buf, position + Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index fff7a299042e..6a809a441695 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -28,6 +28,7 @@ import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -152,7 +153,8 @@ public List requiredFields() } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java index b920301828dc..cdc797658577 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -26,6 +26,7 @@ import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -99,7 +100,8 @@ public Comparator getComparator() } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java index 457e471739f4..7d37d5f2da1b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java @@ -25,23 +25,28 @@ import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseDoubleColumnValueSelector; - +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop + */ public class DoubleCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy { @Override - public void hashRow(BaseDoubleColumnValueSelector dimSelector, Hasher hasher) + public void hashRow(BaseDoubleColumnValueSelector selector, Hasher hasher) { - if (NullHandling.useDefaultValuesForNull() || !dimSelector.isNull()) { - hasher.putDouble(dimSelector.getDouble()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + hasher.putDouble(selector.getDouble()); } } @Override - public void hashValues(BaseDoubleColumnValueSelector dimSelector, HyperLogLogCollector collector) + public void hashValues(BaseDoubleColumnValueSelector selector, HyperLogLogCollector collector) { - if (NullHandling.useDefaultValuesForNull() || !dimSelector.isNull()) { - collector.add(CardinalityAggregator.hashFn.hashLong(Double.doubleToLongBits(dimSelector.getDouble())).asBytes()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashLong(Double.doubleToLongBits(selector.getDouble())).asBytes()); } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java index 1761cab9b6d9..33b66019618b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java @@ -25,6 +25,12 @@ import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseFloatColumnValueSelector; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop + */ public class FloatCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy { diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java index dd836d16be68..82879bba81ea 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java @@ -25,22 +25,28 @@ import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseLongColumnValueSelector; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop + */ public class LongCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy { @Override - public void hashRow(BaseLongColumnValueSelector dimSelector, Hasher hasher) + public void hashRow(BaseLongColumnValueSelector selector, Hasher hasher) { - if (NullHandling.useDefaultValuesForNull() || !dimSelector.isNull()) { - hasher.putLong(dimSelector.getLong()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + hasher.putLong(selector.getLong()); } } @Override - public void hashValues(BaseLongColumnValueSelector dimSelector, HyperLogLogCollector collector) + public void hashValues(BaseLongColumnValueSelector selector, HyperLogLogCollector collector) { - if (NullHandling.useDefaultValuesForNull() || !dimSelector.isNull()) { - collector.add(CardinalityAggregator.hashFn.hashLong(dimSelector.getLong()).asBytes()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashLong(selector.getLong()).asBytes()); } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java index f604a59401d4..7e98ef7b96a3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java @@ -50,11 +50,17 @@ public void hashRow(DimensionSelector dimSelector, Hasher hasher) final String[] values = new String[size]; for (int i = 0; i < size; ++i) { final String value = dimSelector.lookupName(row.get(i)); + // SQL standard spec does not count null values, + // Skip counting null values when we are not replacing null with default value. + // A special value for null in case null handling is configured to use empty string for null. if (!NullHandling.useDefaultValuesForNull() && !hasNonNullValue && value != null) { hasNonNullValue = true; } values[i] = nullToSpecial(value); } + // SQL standard spec does not count null values, + // Skip counting null values when we are not replacing null with default value. + // A special value for null in case null handling is configured to use empty string for null. if (NullHandling.useDefaultValuesForNull() || hasNonNullValue) { // Values need to be sorted to ensure consistent multi-value ordering across different segments Arrays.sort(values); @@ -75,6 +81,9 @@ public void hashValues(DimensionSelector dimSelector, HyperLogLogCollector colle for (int i = 0; i < row.size(); i++) { int index = row.get(i); final String value = dimSelector.lookupName(index); + // SQL standard spec does not count null values, + // Skip counting null values when we are not replacing null with default value. + // A special value for null in case null handling is configured to use empty string for null. if (NullHandling.useDefaultValuesForNull() || value != null) { collector.add(CardinalityAggregator.hashFn.hashUnencodedChars(nullToSpecial(value)).asBytes()); } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index f79f3059df9c..73f3f40a9964 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -83,20 +83,24 @@ public DoubleFirstAggregatorFactory( public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new DoubleFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - columnValueSelector - ), columnValueSelector); + return Pair.of( + new DoubleFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector + ); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new DoubleFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - columnValueSelector - ), columnValueSelector); + return Pair.of( + new DoubleFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector + ); } @Override @@ -133,43 +137,45 @@ public AggregatorFactory getCombiningFactory() public Pair factorize2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new DoubleFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }, selector); + return Pair.of( + new DoubleFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }, selector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new DoubleFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putDouble(position + Longs.BYTES, pair.rhs); - } - } + return Pair.of( + new DoubleFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putDouble(position + Longs.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -189,7 +195,7 @@ public Object deserialize(Object object) @Override @Nullable - public Object finalizeComputation(Object object) + public Object finalizeComputation(@Nullable Object object) { return object == null ? null : ((SerializablePair) object).rhs; } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 53ac58d3e8fd..3360f0feac1d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -80,20 +80,22 @@ public FloatFirstAggregatorFactory( public Pair factorize2(ColumnSelectorFactory metricFactory) { ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new FloatFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - columnValueSelector - ), columnValueSelector); + return Pair.of( + new FloatFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new FloatFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - columnValueSelector - ), columnValueSelector); + return Pair.of( + new FloatFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector); } @Override @@ -130,43 +132,45 @@ public AggregatorFactory getCombiningFactory() public Pair factorize2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new FloatFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }, selector); + return Pair.of( + new FloatFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }, selector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new FloatFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putFloat(position + Longs.BYTES, pair.rhs); - } - } + return Pair.of( + new FloatFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putFloat(position + Longs.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -185,7 +189,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object == null ? object : ((SerializablePair) object).rhs; } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 984951ef29c4..7c09ddd37097 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -74,20 +74,22 @@ public LongFirstAggregatorFactory( public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new LongFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - longColumnSelector - ), longColumnSelector); + return Pair.of( + new LongFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new LongFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - longColumnSelector - ), longColumnSelector); + return Pair.of( + new LongFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override @@ -124,43 +126,45 @@ public AggregatorFactory getCombiningFactory() public Pair factorize2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new LongFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }, selector); + return Pair.of( + new LongFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }, selector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new LongFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putLong(position + Long.BYTES, pair.rhs); - } - } + return Pair.of( + new LongFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putLong(position + Long.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -180,7 +184,7 @@ public Object deserialize(Object object) @Override @Nullable - public Object finalizeComputation(Object object) + public Object finalizeComputation(@Nullable Object object) { return object == null ? object : ((SerializablePair) object).rhs; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 5b5b76f4d667..a0db56a6b4b3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -74,20 +74,22 @@ public DoubleLastAggregatorFactory( public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new DoubleLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - doubleColumnSelector - ), doubleColumnSelector); + return Pair.of( + new DoubleLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + doubleColumnSelector + ), doubleColumnSelector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new DoubleLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - doubleColumnSelector - ), doubleColumnSelector); + return Pair.of( + new DoubleLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + doubleColumnSelector + ), doubleColumnSelector); } @Override @@ -124,43 +126,45 @@ public AggregatorFactory getCombiningFactory() public Pair factorize2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new DoubleLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }, selector); + return Pair.of( + new DoubleLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }, selector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new DoubleLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putDouble(position + Longs.BYTES, pair.rhs); - } - } + return Pair.of( + new DoubleLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putDouble(position + Longs.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -179,7 +183,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object == null ? null : ((SerializablePair) object).rhs; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 7864d0e30dba..93e2efa716b8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -72,20 +72,22 @@ public FloatLastAggregatorFactory( public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new FloatLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - floatColumnSelector - ), floatColumnSelector); + return Pair.of( + new FloatLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + floatColumnSelector + ), floatColumnSelector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new FloatLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - floatColumnSelector - ), floatColumnSelector); + return Pair.of( + new FloatLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + floatColumnSelector + ), floatColumnSelector); } @Override @@ -122,43 +124,45 @@ public AggregatorFactory getCombiningFactory() public Pair factorize2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new FloatLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }, selector); + return Pair.of( + new FloatLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }, selector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new FloatLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putFloat(position + Longs.BYTES, pair.rhs); - } - } + return Pair.of( + new FloatLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putFloat(position + Longs.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -178,7 +182,7 @@ public Object deserialize(Object object) @Override @Nullable - public Object finalizeComputation(Object object) + public Object finalizeComputation(@Nullable Object object) { return object == null ? null : ((SerializablePair) object).rhs; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 38380072ff62..d2cf47bba0e4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -70,20 +70,22 @@ public LongLastAggregatorFactory( public Pair factorize2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new LongLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - longColumnSelector - ), longColumnSelector); + return Pair.of( + new LongLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of(new LongLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - longColumnSelector - ), longColumnSelector); + return Pair.of( + new LongLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override @@ -120,43 +122,45 @@ public AggregatorFactory getCombiningFactory() public Pair factorize2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new LongLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }, selector); + return Pair.of( + new LongLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }, selector); } @Override public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of(new LongLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putLong(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + return Pair.of( + new LongLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putLong(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -175,7 +179,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object == null ? null : ((SerializablePair) object).rhs; } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java index 87b25a63021e..ecb70da4bf16 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -30,7 +30,6 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.cache.CacheKeyBuilder; -import io.druid.segment.DimensionHandlerUtils; import java.util.Comparator; import java.util.Iterator; @@ -111,7 +110,7 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - Double retVal = NullHandling.useDefaultValuesForNull() ? DimensionHandlerUtils.ZERO_DOUBLE : null; + Double retVal = NullHandling.defaultDoubleValue(); if (fieldsIter.hasNext()) { Number nextVal = (Number) fieldsIter.next().compute(values); if (nextVal == null) { diff --git a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java index 48ddeb2ac405..5aec937fba34 100644 --- a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.js.JavaScriptConfig; @@ -114,7 +113,7 @@ public byte[] getCacheKey() public String apply(@Nullable Object value) { checkAndCompileScript(); - return Strings.emptyToNull(fn.apply(value)); + return NullHandling.emptyToNullIfNeeded(fn.apply(value)); } /** diff --git a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java index 6f60449e33d5..b0dddf121b73 100644 --- a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java @@ -74,7 +74,9 @@ public ExtractionType getExtractionType() @Override public byte[] getCacheKey() { + //CHECKSTYLE.OFF: Regexp byte[] localeBytes = StringUtils.toUtf8(Strings.nullToEmpty(localeString)); + //CHECKSTYLE.ON: Regexp return ByteBuffer.allocate(2 + localeBytes.length) .put(ExtractionCacheHelper.CACHE_TYPE_ID_LOWER) .put((byte) 0XFF) diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index b818bd8cd50c..ccdae8f0932f 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -76,7 +76,9 @@ public List unapply(final String value) { @Override public boolean apply(@Nullable String key) { + //CHECKSTYLE.OFF: Regexp return map.get(key).equals(Strings.nullToEmpty(value)); + //CHECKSTYLE.ON: Regexp } }).keySet()); diff --git a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java index 168cad01ea24..ba21754774fe 100644 --- a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java @@ -107,7 +107,9 @@ public String apply(@Nullable String value) value = ""; } } + //CHECKSTYLE.OFF: Regexp return Strings.emptyToNull(StringUtils.format(format, value)); + //CHECKSTYLE.ON: Regexp } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java index 85f6358efd19..ec2b914d3806 100644 --- a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java @@ -73,7 +73,9 @@ public ExtractionType getExtractionType() @Override public byte[] getCacheKey() { + //CHECKSTYLE.OFF: Regexp byte[] localeBytes = StringUtils.toUtf8(Strings.nullToEmpty(localeString)); + //CHECKSTYLE.ON: Regexp return ByteBuffer.allocate(2 + localeBytes.length) .put(ExtractionCacheHelper.CACHE_TYPE_ID_UPPER) .put((byte) 0XFF) diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index d35b77997a86..4e56d7eda92c 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -118,7 +118,9 @@ public byte[] getCacheKey() if (value == null) { hasNull = true; } + //CHECKSTYLE.OFF: Regexp valuesBytes[index] = StringUtils.toUtf8(Strings.nullToEmpty(value)); + //CHECKSTYLE.ON: Regexp valuesBytesSize += valuesBytes[index].length + 1; ++index; } @@ -165,7 +167,9 @@ private InDimFilter optimizeLookup() // We cannot do an unapply()-based optimization if the selector value // and the replaceMissingValuesWith value are the same, since we have to match on // all values that are not present in the lookup. + //CHECKSTYLE.OFF: Regexp final String convertedValue = Strings.emptyToNull(value); + //CHECKSTYLE.ON: Regexp if (!exFn.isRetainMissingValue() && Objects.equals(convertedValue, exFn.getReplaceMissingValueWith())) { return this; } @@ -210,7 +214,9 @@ public RangeSet getDimensionRangeSet(String dimension) } RangeSet retSet = TreeRangeSet.create(); for (String value : values) { + //CHECKSTYLE.OFF: Regexp retSet.add(Range.singleton(Strings.nullToEmpty(value))); + //CHECKSTYLE.ON: Regexp } return retSet; } @@ -260,11 +266,11 @@ public String toString() if (extractionFn != null) { builder.append(")"); } - + //CHECKSTYLE.OFF: Regexp builder.append(" IN (") .append(Joiner.on(", ").join(Iterables.transform(values, input -> Strings.nullToEmpty(input)))) .append(")"); - + //CHECKSTYLE.ON: Regexp return builder.toString(); } diff --git a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java index 8d8dfb8476e2..8a55ffbd4a68 100644 --- a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.collect.RangeSet; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; @@ -99,7 +98,7 @@ private LikeMatcher( ) { this.suffixMatch = Preconditions.checkNotNull(suffixMatch, "suffixMatch"); - this.prefix = Strings.nullToEmpty(prefix); + this.prefix = NullHandling.nullToEmptyIfNeeded(prefix); this.pattern = Preconditions.checkNotNull(pattern, "pattern"); } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index b889cd39eda4..8c18a99ccbba 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -190,7 +190,9 @@ public RangeSet getDimensionRangeSet(String dimension) return null; } RangeSet retSet = TreeRangeSet.create(); + //CHECKSTYLE.OFF: Regexp retSet.add(Range.singleton(Strings.nullToEmpty(value))); + //CHECKSTYLE.ON: Regexp return retSet; } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java index 456fa92a48ba..bc4abfa8d480 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java @@ -20,12 +20,12 @@ package io.druid.query.groupby.epinephelinae; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import io.druid.collections.NonBlockingPool; import io.druid.collections.ResourceHolder; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; import io.druid.java.util.common.DateTimes; @@ -124,8 +124,8 @@ public static Sequence process( final ResourceHolder bufferHolder = intermediateResultsBufferPool.take(); - final String fudgeTimestampString = Strings.emptyToNull( - query.getContextValue(GroupByStrategyV2.CTX_KEY_FUDGE_TIMESTAMP, "") + final String fudgeTimestampString = NullHandling.emptyToNullIfNeeded( + query.getContextValue(GroupByStrategyV2.CTX_KEY_FUDGE_TIMESTAMP, null) ); final DateTime fudgeTimestamp = fudgeTimestampString == null diff --git a/processing/src/main/java/io/druid/query/lookup/LookupConfig.java b/processing/src/main/java/io/druid/query/lookup/LookupConfig.java index 810afa9d3d2e..f5fef048e94f 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupConfig.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupConfig.java @@ -56,7 +56,9 @@ public LookupConfig( @JsonProperty("snapshotWorkingDir") String snapshotWorkingDir ) { + //CHECKSTYLE.OFF: Regexp this.snapshotWorkingDir = Strings.nullToEmpty(snapshotWorkingDir); + //CHECKSTYLE.ON: Regexp } public String getSnapshotWorkingDir() diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java index 36c36d256b46..8743d106a900 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java @@ -21,12 +21,12 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Accumulator; @@ -218,8 +218,8 @@ private ColumnAnalysis analyzeStringColumn( } if (analyzingMinMax() && cardinality > 0) { - min = Strings.nullToEmpty(bitmapIndex.getValue(0)); - max = Strings.nullToEmpty(bitmapIndex.getValue(cardinality - 1)); + min = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(0)); + max = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(cardinality - 1)); } return new ColumnAnalysis( diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index 13758ec2bfd8..3e767bbb1edd 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -20,7 +20,6 @@ package io.druid.segment; import com.google.common.base.Splitter; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; @@ -28,6 +27,7 @@ import io.druid.collections.spatial.ImmutableRTree; import io.druid.collections.spatial.RTree; import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; @@ -187,7 +187,7 @@ private void writeDictionary(Iterable dictionaryValues) throws IOExcepti { for (String value : dictionaryValues) { dictionaryWriter.write(value); - value = Strings.emptyToNull(value); + value = NullHandling.emptyToNullIfNeeded(value); if (dictionarySize == 0) { firstDictionaryValue = value; } diff --git a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java index 3e2576586405..99b953c6270f 100644 --- a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java @@ -19,7 +19,6 @@ package io.druid.segment.filter; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.common.config.NullHandling; @@ -144,16 +143,24 @@ private IntIterable getDimValueIndexIterableForPrefixMatch( final Indexed dimValues ) { - final String lower = Strings.nullToEmpty(likeMatcher.getPrefix()); - final String upper = Strings.nullToEmpty(likeMatcher.getPrefix()) + Character.MAX_VALUE; + + final String lower = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()); + final String upper = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()) + Character.MAX_VALUE; + final int startIndex; // inclusive final int endIndex; // exclusive - final int lowerFound = bitmapIndex.getIndex(lower); - startIndex = lowerFound >= 0 ? lowerFound : -(lowerFound + 1); + if (lower == null) { + // For Null values + startIndex = bitmapIndex.getIndex(null); + endIndex = startIndex + 1; + } else { + final int lowerFound = bitmapIndex.getIndex(lower); + startIndex = lowerFound >= 0 ? lowerFound : -(lowerFound + 1); - final int upperFound = bitmapIndex.getIndex(upper); - endIndex = upperFound >= 0 ? upperFound + 1 : -(upperFound + 1); + final int upperFound = bitmapIndex.getIndex(upper); + endIndex = upperFound >= 0 ? upperFound + 1 : -(upperFound + 1); + } return new IntIterable() { diff --git a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java index d60f728764e9..c052e3d461fd 100644 --- a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java @@ -74,7 +74,9 @@ public Object[] apply(List input) public LookupExtractionFnTest(boolean retainMissing, String replaceMissing, boolean injective) { + //CHECKSTYLE.OFF: Regexp this.replaceMissing = Strings.emptyToNull(replaceMissing); + //CHECKSTYLE.ON: Regexp this.retainMissing = retainMissing; this.injective = injective; } @@ -82,7 +84,9 @@ public LookupExtractionFnTest(boolean retainMissing, String replaceMissing, bool @Test public void testEqualsAndHash() { + //CHECKSTYLE.OFF: Regexp if (retainMissing && !Strings.isNullOrEmpty(replaceMissing)) { + //CHECKSTYLE.ON: Regexp // skip return; } @@ -154,7 +158,9 @@ public void testIllegalArgs() final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), retainMissing, + //CHECKSTYLE.OFF: Regexp Strings.emptyToNull(replaceMissing), + //CHECKSTYLE.ON: Regexp injective, false ); diff --git a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java index f126eb447ffd..0b742ec7c2c2 100644 --- a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java @@ -117,7 +117,9 @@ public Filter toFilter() if (extractionFn == null) { return new NoBitmapSelectorFilter(dimension, value); } else { + //CHECKSTYLE.OFF: Regexp final String valueOrNull = Strings.emptyToNull(value); + //CHECKSTYLE.ON: Regexp final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override diff --git a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java index d6d1ea1601bc..953d5ff199a6 100644 --- a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java +++ b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -168,12 +168,13 @@ public boolean mustDecorate() public byte[] getCacheKey() { byte[] dimensionBytes = StringUtils.toUtf8(dimension); + //CHECKSTYLE.OFF: Regexp byte[] dimExtractionFnBytes = Strings.isNullOrEmpty(name) ? getLookup().getCacheKey() : StringUtils.toUtf8(name); byte[] outputNameBytes = StringUtils.toUtf8(outputName); byte[] replaceWithBytes = StringUtils.toUtf8(Strings.nullToEmpty(replaceMissingValueWith)); - + //CHECKSTYLE.ON: Regexp return ByteBuffer.allocate(7 + dimensionBytes.length diff --git a/server/src/main/java/io/druid/query/expression/LookupExprMacro.java b/server/src/main/java/io/druid/query/expression/LookupExprMacro.java index 77dc88c184bd..e24ad3ff8c71 100644 --- a/server/src/main/java/io/druid/query/expression/LookupExprMacro.java +++ b/server/src/main/java/io/druid/query/expression/LookupExprMacro.java @@ -19,8 +19,8 @@ package io.druid.query.expression; -import com.google.common.base.Strings; import com.google.inject.Inject; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -77,7 +77,7 @@ class LookupExpr implements Expr @Override public ExprEval eval(final ObjectBinding bindings) { - return ExprEval.of(extractionFn.apply(Strings.emptyToNull(arg.eval(bindings).asString()))); + return ExprEval.of(extractionFn.apply(NullHandling.emptyToNullIfNeeded(arg.eval(bindings).asString()))); } @Override diff --git a/server/src/main/java/io/druid/query/lookup/LookupModule.java b/server/src/main/java/io/druid/query/lookup/LookupModule.java index a1d387cf7225..e855aa320022 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupModule.java +++ b/server/src/main/java/io/druid/query/lookup/LookupModule.java @@ -261,7 +261,9 @@ public String getLookupTier() ); final String lookupTier = lookupTierIsDatasource ? dataSourceTaskIdHolder.getDataSource() : this.lookupTier; return Preconditions.checkNotNull( + //CHECKSTYLE.OFF: Regexp lookupTier == null ? DEFAULT_TIER : Strings.emptyToNull(lookupTier), + //CHECKSTYLE.ON: Regexp "Cannot have empty lookup tier from %s", lookupTierIsDatasource ? "bound value" : LookupModule.PROPERTY_BASE ); diff --git a/server/src/main/java/io/druid/server/QueryLifecycle.java b/server/src/main/java/io/druid/server/QueryLifecycle.java index 1f25f370e279..959367778f4e 100644 --- a/server/src/main/java/io/druid/server/QueryLifecycle.java +++ b/server/src/main/java/io/druid/server/QueryLifecycle.java @@ -293,7 +293,9 @@ public void emitLogsAndMetrics( queryMetricsFactory, toolChest, queryPlus.getQuery(), + //CHECKSTYLE.OFF: Regexp Strings.nullToEmpty(remoteAddress) + //CHECKSTYLE.ON: Regexp ); queryMetrics.success(success); queryMetrics.reportQueryTime(queryTimeNs); @@ -325,7 +327,9 @@ public void emitLogsAndMetrics( requestLogger.log( new RequestLogLine( DateTimes.utc(startMs), + //CHECKSTYLE.OFF: Regexp Strings.nullToEmpty(remoteAddress), + //CHECKSTYLE.ON: Regexp queryPlus.getQuery(), new QueryStats(statsMap) ) diff --git a/server/src/main/java/io/druid/server/emitter/EmitterModule.java b/server/src/main/java/io/druid/server/emitter/EmitterModule.java index a0e08225ee1a..fbb614b6728f 100644 --- a/server/src/main/java/io/druid/server/emitter/EmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/EmitterModule.java @@ -88,7 +88,9 @@ public void configure(Binder binder) String version = getClass().getPackage().getImplementationVersion(); extraServiceDimensions .addBinding("version") + //CHECKSTYLE.OFF: Regexp .toInstance(Strings.nullToEmpty(version)); // Version is null during `mvn test`. + //CHECKSTYLE.ON: Regexp } @Provides diff --git a/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java b/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java index 35288b6bc3f9..71d55e2d07fa 100644 --- a/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java +++ b/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java @@ -93,7 +93,9 @@ public String getAnnouncementPath(String listenerName) { return ZKPaths.makePath( getListenersPath(), Preconditions.checkNotNull( + //CHECKSTYLE.OFF: Regexp Strings.emptyToNull(listenerName), "Listener name cannot be null" + //CHECKSTYLE.ON: Regexp ) ); } diff --git a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java index 7e1ef423f959..ddd2108f4b25 100644 --- a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java +++ b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java @@ -160,7 +160,9 @@ public Object[] parametersForTestApply() public void testApply(DimensionSpec dimensionSpec, Map map) { for (Map.Entry entry : map.entrySet()) { + //CHECKSTYLE.OFF: Regexp Assert.assertEquals(Strings.emptyToNull(entry.getValue()), dimensionSpec.getExtractionFn().apply(entry.getKey())); + //CHECKSTYLE.ON: Regexp } } diff --git a/services/src/main/java/io/druid/cli/DumpSegment.java b/services/src/main/java/io/druid/cli/DumpSegment.java index e68c871b026f..3d82fbf8d3ce 100644 --- a/services/src/main/java/io/druid/cli/DumpSegment.java +++ b/services/src/main/java/io/druid/cli/DumpSegment.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -41,6 +40,7 @@ import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.common.config.NullHandling; import io.druid.guice.DruidProcessingModule; import io.druid.guice.QueryRunnerFactoryModule; import io.druid.guice.QueryableModule; @@ -364,18 +364,20 @@ public Object apply(final OutputStream out) jg.writeFieldName(columnName); jg.writeStartObject(); for (int i = 0; i < bitmapIndex.getCardinality(); i++) { - jg.writeFieldName(Strings.nullToEmpty(bitmapIndex.getValue(i))); - final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); - if (decompressBitmaps) { - jg.writeStartArray(); - final IntIterator iterator = bitmap.iterator(); - while (iterator.hasNext()) { - final int rowNum = iterator.next(); - jg.writeNumber(rowNum); + String val = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(i)); + if (val != null) { + final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); + if (decompressBitmaps) { + jg.writeStartArray(); + final IntIterator iterator = bitmap.iterator(); + while (iterator.hasNext()) { + final int rowNum = iterator.next(); + jg.writeNumber(rowNum); + } + jg.writeEndArray(); + } else { + jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap)); } - jg.writeEndArray(); - } else { - jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap)); } } jg.writeEndObject(); From bad3ed54a610cdc3dcd9698e39ccc1f67be2de68 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 16 Jan 2018 23:32:31 +0530 Subject: [PATCH 48/50] review comment --- .../java/io/druid/query/aggregation/Aggregator.java | 9 ++++++--- .../druid/query/aggregation/BufferAggregator.java | 13 ++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java index 02ab6d084b95..a131351eef89 100644 --- a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java @@ -56,9 +56,12 @@ default double getDouble() } /** - * returns true if the Aggregator supports returning null values and the aggregated value is Null. - * The default implementation always return false to enable smooth backward compatibility, re-implement if your aggregator is nullable. - * For backwards compatibility, isNull() may return false even if {@link Aggregator#get()} returns null. Users of this method should account for this case. + * returns true if aggregator's output type is primitive long/double/float and aggregated value is null, + * but when aggregated output type is Object, this method always returns false, + * and users are advised to check nullability for the object returned by {@link #get()} + * method. + * The default implementation always return false to enable smooth backward compatibility, + * re-implement if your aggregator is nullable. */ default boolean isNull() { diff --git a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java index c1dba19228c4..6385cb3405b7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java @@ -183,13 +183,12 @@ default void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, By } /** - * Returns true if the aggregator is nullable and the aggregated value is null - *

- * Implementations must not change the position, limit or mark of the given buffer - * - * The default implementation always returns false. - * This default method is added to enable smooth backward compatibility, please re-implement it if your aggregators - * support null values + * returns true if aggregator's output type is primitive long/double/float and aggregated value is null, + * but when aggregated output type is Object, this method always returns false, + * and users are advised to check nullability for the object returned by {@link #get()} + * method. + * The default implementation always return false to enable smooth backward compatibility, + * re-implement if your aggregator is nullable. * * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the aggregate value is stored From 72f90b490dde44db2b9c19ba71a98130376f5b08 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 16 Jan 2018 23:35:15 +0530 Subject: [PATCH 49/50] review comment --- .../main/java/io/druid/query/aggregation/AggregatorUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index a8078aefb0a6..f1b2438fe02b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -169,7 +169,8 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public boolean isNull() { - return baseSelector.getObject().isNull(); + final ExprEval exprEval = baseSelector.getObject(); + return exprEval.isNull(); } } return new ExpressionFloatColumnSelector(); From bc7155308a920d99aaf434a6ff9c5dd36c5bd00d Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 22 Jan 2018 00:42:26 +0530 Subject: [PATCH 50/50] fix failing tests --- .../sql/QuantileSqlAggregatorTest.java | 12 +++++++++--- .../groupby/strategy/GroupByStrategyV2.java | 17 +++++++++-------- .../io/druid/segment/DoubleColumnSelector.java | 3 +++ .../io/druid/sql/calcite/CalciteQueryTest.java | 9 +++++++-- .../sql/calcite/expression/ExpressionsTest.java | 3 ++- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index 8fae927cef8b..0cb2608b0a8e 100644 --- a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.granularity.Granularities; import io.druid.query.Druids; import io.druid.query.QueryDataSource; @@ -308,9 +309,14 @@ public void testQuantileOnInnerQuery() throws Exception // Verify results final List results = plannerResult.run().toList(); - final List expectedResults = ImmutableList.of( - new Object[]{7.0, 8.26386833190918} - ); + final List expectedResults = + NullHandling.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{7.0, 8.26386833190918} + ) : + ImmutableList.of( + new Object[]{5.25, 6.59091854095459} + ); Assert.assertEquals(expectedResults.size(), results.size()); for (int i = 0; i < expectedResults.size(); i++) { Assert.assertArrayEquals(expectedResults.get(i), results.get(i)); diff --git a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java index 5ad75004b760..539fb7b39d63 100644 --- a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java +++ b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java @@ -228,6 +228,14 @@ protected BinaryFn createMergeFn(Query queryParam) // Fudge timestamp, maybe. final DateTime fudgeTimestamp = getUniversalTimestamp(query); + ImmutableMap.Builder context = ImmutableMap.builder(); + context.put("finalize", false); + context.put(GroupByQueryConfig.CTX_KEY_STRATEGY, GroupByStrategySelector.STRATEGY_V2); + if (fudgeTimestamp != null) { + context.put(CTX_KEY_FUDGE_TIMESTAMP, String.valueOf(fudgeTimestamp.getMillis())); + } + context.put(CTX_KEY_OUTERMOST, false); + context.put(GroupByQueryConfig.CTX_KEY_APPLY_LIMIT_PUSH_DOWN, query.isApplyLimitPushDown()); final GroupByQuery newQuery = new GroupByQuery( query.getDataSource(), @@ -243,14 +251,7 @@ protected BinaryFn createMergeFn(Query queryParam) query.getLimitSpec(), query.getContext() ).withOverriddenContext( - ImmutableMap.of( - "finalize", false, - GroupByQueryConfig.CTX_KEY_STRATEGY, GroupByStrategySelector.STRATEGY_V2, - CTX_KEY_FUDGE_TIMESTAMP, fudgeTimestamp == null ? "" : String.valueOf(fudgeTimestamp.getMillis()), - CTX_KEY_OUTERMOST, false, - // the having spec shouldn't be passed down, so we need to convey the existing limit push down status - GroupByQueryConfig.CTX_KEY_APPLY_LIMIT_PUSH_DOWN, query.isApplyLimitPushDown() - ) + context.build() ); Sequence rowSequence = Sequences.map( diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java b/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java index 5f9207f31890..4e5a3bbd9b5a 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java @@ -61,6 +61,9 @@ default long getLong() @Override default Double getObject() { + if (isNull()) { + return null; + } return getDouble(); } diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 8de1d1bb59b2..9ded67f3077a 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -504,6 +504,7 @@ public void testSelectStar() throws Exception @Test public void testSelectStarOnForbiddenTable() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; assertQueryIsForbidden( "SELECT * FROM druid.forbiddenDatasource", CalciteTests.REGULAR_USER_AUTH_RESULT @@ -523,7 +524,7 @@ public void testSelectStarOnForbiddenTable() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "forbidden", "abcd", 9999.0f, 0.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-01"), 1L, "forbidden", "abcd", 9999.0f, nullValue, HLLCV1.class.getName()} ) ); } @@ -2647,8 +2648,12 @@ public void testFilteredAggregationWithNotIn() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{5L, 2L} + ) : + ImmutableList.of( + new Object[]{5L, 3L} ) ); } @@ -3989,7 +3994,7 @@ public void testApproxCountDistinct() throws Exception new Object[]{6L, 3L, 2L, 2L, 2L, 6L} ) : ImmutableList.of( - new Object[]{6L, 3L, 2L, 1L, 2L, 6L} + new Object[]{6L, 3L, 2L, 1L, 1L, 6L} ) ); } diff --git a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java index 15abe5590b37..90f40f932125 100644 --- a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.math.expr.ExprEval; import io.druid.math.expr.Parser; @@ -194,7 +195,7 @@ public void testStrpos() rexBuilder.makeLiteral("ax") ), DruidExpression.fromExpression("(strpos(" + DruidExpression.nullLiteral() + ",'ax') + 1)"), - 0L + NullHandling.useDefaultValuesForNull() ? 0L : null ); }